IEMAll.cpp@ 96407

Last change on this file since 96407 was 96407, checked in by vboxsync, 3 years ago
scm copyright and license note update
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 456.1 KB

Line
1	/* $Id: IEMAll.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/** @page pg_iem IEM - Interpreted Execution Manager
30	*
31	* The interpreted exeuction manager (IEM) is for executing short guest code
32	* sequences that are causing too many exits / virtualization traps. It will
33	* also be used to interpret single instructions, thus replacing the selective
34	* interpreters in EM and IOM.
35	*
36	* Design goals:
37	* - Relatively small footprint, although we favour speed and correctness
38	* over size.
39	* - Reasonably fast.
40	* - Correctly handle lock prefixed instructions.
41	* - Complete instruction set - eventually.
42	* - Refactorable into a recompiler, maybe.
43	* - Replace EMInterpret*.
44	*
45	* Using the existing disassembler has been considered, however this is thought
46	* to conflict with speed as the disassembler chews things a bit too much while
47	* leaving us with a somewhat complicated state to interpret afterwards.
48	*
49	*
50	* The current code is very much work in progress. You've been warned!
51	*
52	*
53	* @section sec_iem_fpu_instr FPU Instructions
54	*
55	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
56	* same or equivalent instructions on the host FPU. To make life easy, we also
57	* let the FPU prioritize the unmasked exceptions for us. This however, only
58	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
59	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
60	* can trigger spurious FPU exceptions.
61	*
62	* The guest FPU state is not loaded into the host CPU and kept there till we
63	* leave IEM because the calling conventions have declared an all year open
64	* season on much of the FPU state. For instance an innocent looking call to
65	* memcpy might end up using a whole bunch of XMM or MM registers if the
66	* particular implementation finds it worthwhile.
67	*
68	*
69	* @section sec_iem_logging Logging
70	*
71	* The IEM code uses the \"IEM\" log group for the main logging. The different
72	* logging levels/flags are generally used for the following purposes:
73	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
74	* - Flow (LogFlow) : Basic enter/exit IEM state info.
75	* - Level 2 (Log2) : ?
76	* - Level 3 (Log3) : More detailed enter/exit IEM state info.
77	* - Level 4 (Log4) : Decoding mnemonics w/ EIP.
78	* - Level 5 (Log5) : Decoding details.
79	* - Level 6 (Log6) : Enables/disables the lockstep comparison with REM.
80	* - Level 7 (Log7) : iret++ execution logging.
81	* - Level 8 (Log8) : Memory writes.
82	* - Level 9 (Log9) : Memory reads.
83	* - Level 10 (Log10): TLBs.
84	* - Level 11 (Log11): Unmasked FPU exceptions.
85	*/
86
87	/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
88	#ifdef _MSC_VER
89	# pragma warning(disable:4505)
90	#endif
91
92
93	/*********************************************************************************************************************************
94	* Header Files *
95	*********************************************************************************************************************************/
96	#define LOG_GROUP LOG_GROUP_IEM
97	#define VMCPU_INCL_CPUM_GST_CTX
98	#include <VBox/vmm/iem.h>
99	#include <VBox/vmm/cpum.h>
100	#include <VBox/vmm/apic.h>
101	#include <VBox/vmm/pdm.h>
102	#include <VBox/vmm/pgm.h>
103	#include <VBox/vmm/iom.h>
104	#include <VBox/vmm/em.h>
105	#include <VBox/vmm/hm.h>
106	#include <VBox/vmm/nem.h>
107	#include <VBox/vmm/gim.h>
108	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
109	# include <VBox/vmm/em.h>
110	# include <VBox/vmm/hm_svm.h>
111	#endif
112	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
113	# include <VBox/vmm/hmvmxinline.h>
114	#endif
115	#include <VBox/vmm/tm.h>
116	#include <VBox/vmm/dbgf.h>
117	#include <VBox/vmm/dbgftrace.h>
118	#include "IEMInternal.h"
119	#include <VBox/vmm/vmcc.h>
120	#include <VBox/log.h>
121	#include <VBox/err.h>
122	#include <VBox/param.h>
123	#include <VBox/dis.h>
124	#include <VBox/disopcode.h>
125	#include <iprt/asm-math.h>
126	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
127	# include <iprt/asm-amd64-x86.h>
128	#elif defined(RT_ARCH_ARM64) \|\| defined(RT_ARCH_ARM32)
129	# include <iprt/asm-arm.h>
130	#endif
131	#include <iprt/assert.h>
132	#include <iprt/string.h>
133	#include <iprt/x86.h>
134
135	#include "IEMInline.h"
136
137
138	/*********************************************************************************************************************************
139	* Structures and Typedefs *
140	*********************************************************************************************************************************/
141	/**
142	* CPU exception classes.
143	*/
144	typedef enum IEMXCPTCLASS
145	{
146	IEMXCPTCLASS_BENIGN,
147	IEMXCPTCLASS_CONTRIBUTORY,
148	IEMXCPTCLASS_PAGE_FAULT,
149	IEMXCPTCLASS_DOUBLE_FAULT
150	} IEMXCPTCLASS;
151
152
153	/*********************************************************************************************************************************
154	* Global Variables *
155	*********************************************************************************************************************************/
156	#if defined(IEM_LOG_MEMORY_WRITES)
157	/** What IEM just wrote. */
158	uint8_t g_abIemWrote[256];
159	/** How much IEM just wrote. */
160	size_t g_cbIemWrote;
161	#endif
162
163
164	/*********************************************************************************************************************************
165	* Internal Functions *
166	*********************************************************************************************************************************/
167	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
168	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT;
169
170
171	/**
172	* Initializes the decoder state.
173	*
174	* iemReInitDecoder is mostly a copy of this function.
175	*
176	* @param pVCpu The cross context virtual CPU structure of the
177	* calling thread.
178	* @param fBypassHandlers Whether to bypass access handlers.
179	* @param fDisregardLock Whether to disregard the LOCK prefix.
180	*/
181	DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
182	{
183	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
184	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
185	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
186	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
187	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
188	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
189	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
190	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
191	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
192	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
193
194	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
195	IEMMODE enmMode = iemCalcCpuMode(pVCpu);
196	pVCpu->iem.s.enmCpuMode = enmMode;
197	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
198	pVCpu->iem.s.enmEffAddrMode = enmMode;
199	if (enmMode != IEMMODE_64BIT)
200	{
201	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
202	pVCpu->iem.s.enmEffOpSize = enmMode;
203	}
204	else
205	{
206	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
207	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
208	}
209	pVCpu->iem.s.fPrefixes = 0;
210	pVCpu->iem.s.uRexReg = 0;
211	pVCpu->iem.s.uRexB = 0;
212	pVCpu->iem.s.uRexIndex = 0;
213	pVCpu->iem.s.idxPrefix = 0;
214	pVCpu->iem.s.uVex3rdReg = 0;
215	pVCpu->iem.s.uVexLength = 0;
216	pVCpu->iem.s.fEvexStuff = 0;
217	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
218	#ifdef IEM_WITH_CODE_TLB
219	pVCpu->iem.s.pbInstrBuf = NULL;
220	pVCpu->iem.s.offInstrNextByte = 0;
221	pVCpu->iem.s.offCurInstrStart = 0;
222	# ifdef VBOX_STRICT
223	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
224	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
225	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
226	# endif
227	#else
228	pVCpu->iem.s.offOpcode = 0;
229	pVCpu->iem.s.cbOpcode = 0;
230	#endif
231	pVCpu->iem.s.offModRm = 0;
232	pVCpu->iem.s.cActiveMappings = 0;
233	pVCpu->iem.s.iNextMapping = 0;
234	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
235	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
236	pVCpu->iem.s.fDisregardLock = fDisregardLock;
237
238	#ifdef DBGFTRACE_ENABLED
239	switch (enmMode)
240	{
241	case IEMMODE_64BIT:
242	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
243	break;
244	case IEMMODE_32BIT:
245	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
246	break;
247	case IEMMODE_16BIT:
248	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
249	break;
250	}
251	#endif
252	}
253
254
255	/**
256	* Reinitializes the decoder state 2nd+ loop of IEMExecLots.
257	*
258	* This is mostly a copy of iemInitDecoder.
259	*
260	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
261	*/
262	DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
263	{
264	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
265	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
268	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
269	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
270	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
271	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
272	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
273
274	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
275	IEMMODE enmMode = iemCalcCpuMode(pVCpu);
276	pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
277	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
278	pVCpu->iem.s.enmEffAddrMode = enmMode;
279	if (enmMode != IEMMODE_64BIT)
280	{
281	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
282	pVCpu->iem.s.enmEffOpSize = enmMode;
283	}
284	else
285	{
286	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
287	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
288	}
289	pVCpu->iem.s.fPrefixes = 0;
290	pVCpu->iem.s.uRexReg = 0;
291	pVCpu->iem.s.uRexB = 0;
292	pVCpu->iem.s.uRexIndex = 0;
293	pVCpu->iem.s.idxPrefix = 0;
294	pVCpu->iem.s.uVex3rdReg = 0;
295	pVCpu->iem.s.uVexLength = 0;
296	pVCpu->iem.s.fEvexStuff = 0;
297	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
298	#ifdef IEM_WITH_CODE_TLB
299	if (pVCpu->iem.s.pbInstrBuf)
300	{
301	uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
302	- pVCpu->iem.s.uInstrBufPc;
303	if (off < pVCpu->iem.s.cbInstrBufTotal)
304	{
305	pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
306	pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
307	if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
308	pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
309	else
310	pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
311	}
312	else
313	{
314	pVCpu->iem.s.pbInstrBuf = NULL;
315	pVCpu->iem.s.offInstrNextByte = 0;
316	pVCpu->iem.s.offCurInstrStart = 0;
317	pVCpu->iem.s.cbInstrBuf = 0;
318	pVCpu->iem.s.cbInstrBufTotal = 0;
319	}
320	}
321	else
322	{
323	pVCpu->iem.s.offInstrNextByte = 0;
324	pVCpu->iem.s.offCurInstrStart = 0;
325	pVCpu->iem.s.cbInstrBuf = 0;
326	pVCpu->iem.s.cbInstrBufTotal = 0;
327	}
328	#else
329	pVCpu->iem.s.cbOpcode = 0;
330	pVCpu->iem.s.offOpcode = 0;
331	#endif
332	pVCpu->iem.s.offModRm = 0;
333	Assert(pVCpu->iem.s.cActiveMappings == 0);
334	pVCpu->iem.s.iNextMapping = 0;
335	Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
336	Assert(pVCpu->iem.s.fBypassHandlers == false);
337
338	#ifdef DBGFTRACE_ENABLED
339	switch (enmMode)
340	{
341	case IEMMODE_64BIT:
342	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
343	break;
344	case IEMMODE_32BIT:
345	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
346	break;
347	case IEMMODE_16BIT:
348	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
349	break;
350	}
351	#endif
352	}
353
354
355
356	/**
357	* Prefetch opcodes the first time when starting executing.
358	*
359	* @returns Strict VBox status code.
360	* @param pVCpu The cross context virtual CPU structure of the
361	* calling thread.
362	* @param fBypassHandlers Whether to bypass access handlers.
363	* @param fDisregardLock Whether to disregard LOCK prefixes.
364	*
365	* @todo Combine fDisregardLock and fBypassHandlers into a flag parameter and
366	* store them as such.
367	*/
368	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) RT_NOEXCEPT
369	{
370	iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock);
371
372	#ifdef IEM_WITH_CODE_TLB
373	/** @todo Do ITLB lookup here. */
374
375	#else /* !IEM_WITH_CODE_TLB */
376
377	/*
378	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
379	*
380	* First translate CS:rIP to a physical address.
381	*/
382	uint32_t cbToTryRead;
383	RTGCPTR GCPtrPC;
384	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
385	{
386	cbToTryRead = GUEST_PAGE_SIZE;
387	GCPtrPC = pVCpu->cpum.GstCtx.rip;
388	if (IEM_IS_CANONICAL(GCPtrPC))
389	cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
390	else
391	return iemRaiseGeneralProtectionFault0(pVCpu);
392	}
393	else
394	{
395	uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
396	AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
397	if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
398	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
399	else
400	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
401	if (cbToTryRead) { /* likely */ }
402	else /* overflowed */
403	{
404	Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
405	cbToTryRead = UINT32_MAX;
406	}
407	GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
408	Assert(GCPtrPC <= UINT32_MAX);
409	}
410
411	PGMPTWALK Walk;
412	int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
413	if (RT_SUCCESS(rc))
414	Assert(Walk.fSucceeded); /* probable. */
415	else
416	{
417	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
418	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
419	if (Walk.fFailed & PGM_WALKFAIL_EPT)
420	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
421	#endif
422	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
423	}
424	if ((Walk.fEffective & X86_PTE_US) \|\| pVCpu->iem.s.uCpl != 3) { /* likely */ }
425	else
426	{
427	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
428	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
429	if (Walk.fFailed & PGM_WALKFAIL_EPT)
430	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
431	#endif
432	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
433	}
434	if (!(Walk.fEffective & X86_PTE_PAE_NX) \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
435	else
436	{
437	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
439	if (Walk.fFailed & PGM_WALKFAIL_EPT)
440	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
441	#endif
442	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
443	}
444	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
445	/** @todo Check reserved bits and such stuff. PGM is better at doing
446	* that, so do it when implementing the guest virtual address
447	* TLB... */
448
449	/*
450	* Read the bytes at this address.
451	*/
452	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
453	if (cbToTryRead > cbLeftOnPage)
454	cbToTryRead = cbLeftOnPage;
455	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
456	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
457
458	if (!pVCpu->iem.s.fBypassHandlers)
459	{
460	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
461	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
462	{ /* likely */ }
463	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
464	{
465	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
466	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
467	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
468	}
469	else
470	{
471	Log((RT_SUCCESS(rcStrict)
472	? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
473	: "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
474	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
475	return rcStrict;
476	}
477	}
478	else
479	{
480	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
481	if (RT_SUCCESS(rc))
482	{ /* likely */ }
483	else
484	{
485	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
486	GCPtrPC, GCPhys, rc, cbToTryRead));
487	return rc;
488	}
489	}
490	pVCpu->iem.s.cbOpcode = cbToTryRead;
491	#endif /* !IEM_WITH_CODE_TLB */
492	return VINF_SUCCESS;
493	}
494
495
496	/**
497	* Invalidates the IEM TLBs.
498	*
499	* This is called internally as well as by PGM when moving GC mappings.
500	*
501	* @returns
502	* @param pVCpu The cross context virtual CPU structure of the calling
503	* thread.
504	*/
505	VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)
506	{
507	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
508	Log10(("IEMTlbInvalidateAll\n"));
509	# ifdef IEM_WITH_CODE_TLB
510	pVCpu->iem.s.cbInstrBufTotal = 0;
511	pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
512	if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
513	{ /* very likely */ }
514	else
515	{
516	pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
517	unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
518	while (i-- > 0)
519	pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
520	}
521	# endif
522
523	# ifdef IEM_WITH_DATA_TLB
524	pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
525	if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
526	{ /* very likely */ }
527	else
528	{
529	pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
530	unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
531	while (i-- > 0)
532	pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
533	}
534	# endif
535	#else
536	RT_NOREF(pVCpu);
537	#endif
538	}
539
540
541	/**
542	* Invalidates a page in the TLBs.
543	*
544	* @param pVCpu The cross context virtual CPU structure of the calling
545	* thread.
546	* @param GCPtr The address of the page to invalidate
547	* @thread EMT(pVCpu)
548	*/
549	VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
550	{
551	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
552	Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));
553	GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);
554	Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));
555	uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr);
556
557	# ifdef IEM_WITH_CODE_TLB
558	if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.CodeTlb.uTlbRevision))
559	{
560	pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
561	if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc))
562	pVCpu->iem.s.cbInstrBufTotal = 0;
563	}
564	# endif
565
566	# ifdef IEM_WITH_DATA_TLB
567	if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.DataTlb.uTlbRevision))
568	pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
569	# endif
570	#else
571	NOREF(pVCpu); NOREF(GCPtr);
572	#endif
573	}
574
575
576	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
577	/**
578	* Invalid both TLBs slow fashion following a rollover.
579	*
580	* Worker for IEMTlbInvalidateAllPhysical,
581	* IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,
582	* iemMemMapJmp and others.
583	*
584	* @thread EMT(pVCpu)
585	*/
586	static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu)
587	{
588	Log10(("IEMTlbInvalidateAllPhysicalSlow\n"));
589	ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
590	ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
591
592	unsigned i;
593	# ifdef IEM_WITH_CODE_TLB
594	i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
595	while (i-- > 0)
596	{
597	pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
598	pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
599	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
600	}
601	# endif
602	# ifdef IEM_WITH_DATA_TLB
603	i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
604	while (i-- > 0)
605	{
606	pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
607	pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
608	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
609	}
610	# endif
611
612	}
613	#endif
614
615
616	/**
617	* Invalidates the host physical aspects of the IEM TLBs.
618	*
619	* This is called internally as well as by PGM when moving GC mappings.
620	*
621	* @param pVCpu The cross context virtual CPU structure of the calling
622	* thread.
623	* @note Currently not used.
624	*/
625	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
626	{
627	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
628	/* Note! This probably won't end up looking exactly like this, but it give an idea... */
629	Log10(("IEMTlbInvalidateAllPhysical\n"));
630
631	# ifdef IEM_WITH_CODE_TLB
632	pVCpu->iem.s.cbInstrBufTotal = 0;
633	# endif
634	uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
635	if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))
636	{
637	pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
638	pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
639	}
640	else
641	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
642	#else
643	NOREF(pVCpu);
644	#endif
645	}
646
647
648	/**
649	* Invalidates the host physical aspects of the IEM TLBs.
650	*
651	* This is called internally as well as by PGM when moving GC mappings.
652	*
653	* @param pVM The cross context VM structure.
654	* @param idCpuCaller The ID of the calling EMT if available to the caller,
655	* otherwise NIL_VMCPUID.
656	*
657	* @remarks Caller holds the PGM lock.
658	*/
659	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller)
660	{
661	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
662	PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);
663	if (pVCpuCaller)
664	VMCPU_ASSERT_EMT(pVCpuCaller);
665	Log10(("IEMTlbInvalidateAllPhysicalAllCpus\n"));
666
667	VMCC_FOR_EACH_VMCPU(pVM)
668	{
669	# ifdef IEM_WITH_CODE_TLB
670	if (pVCpuCaller == pVCpu)
671	pVCpu->iem.s.cbInstrBufTotal = 0;
672	# endif
673
674	uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);
675	uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;
676	if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))
677	{ /* likely */}
678	else if (pVCpuCaller == pVCpu)
679	uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;
680	else
681	{
682	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
683	continue;
684	}
685	ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
686	ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
687	}
688	VMCC_FOR_EACH_VMCPU_END(pVM);
689
690	#else
691	RT_NOREF(pVM, idCpuCaller);
692	#endif
693	}
694
695	#ifdef IEM_WITH_CODE_TLB
696
697	/**
698	* Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
699	* failure and jumps.
700	*
701	* We end up here for a number of reasons:
702	* - pbInstrBuf isn't yet initialized.
703	* - Advancing beyond the buffer boundrary (e.g. cross page).
704	* - Advancing beyond the CS segment limit.
705	* - Fetching from non-mappable page (e.g. MMIO).
706	*
707	* @param pVCpu The cross context virtual CPU structure of the
708	* calling thread.
709	* @param pvDst Where to return the bytes.
710	* @param cbDst Number of bytes to read.
711	*
712	* @todo Make cbDst = 0 a way of initializing pbInstrBuf?
713	*/
714	void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT
715	{
716	#ifdef IN_RING3
717	for (;;)
718	{
719	Assert(cbDst <= 8);
720	uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
721
722	/*
723	* We might have a partial buffer match, deal with that first to make the
724	* rest simpler. This is the first part of the cross page/buffer case.
725	*/
726	if (pVCpu->iem.s.pbInstrBuf != NULL)
727	{
728	if (offBuf < pVCpu->iem.s.cbInstrBuf)
729	{
730	Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
731	uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
732	memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
733
734	cbDst -= cbCopy;
735	pvDst = (uint8_t *)pvDst + cbCopy;
736	offBuf += cbCopy;
737	pVCpu->iem.s.offInstrNextByte += offBuf;
738	}
739	}
740
741	/*
742	* Check segment limit, figuring how much we're allowed to access at this point.
743	*
744	* We will fault immediately if RIP is past the segment limit / in non-canonical
745	* territory. If we do continue, there are one or more bytes to read before we
746	* end up in trouble and we need to do that first before faulting.
747	*/
748	RTGCPTR GCPtrFirst;
749	uint32_t cbMaxRead;
750	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
751	{
752	GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
753	if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
754	{ /* likely */ }
755	else
756	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
757	cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
758	}
759	else
760	{
761	GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
762	Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
763	if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
764	{ /* likely */ }
765	else
766	iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
767	cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
768	if (cbMaxRead != 0)
769	{ /* likely */ }
770	else
771	{
772	/* Overflowed because address is 0 and limit is max. */
773	Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
774	cbMaxRead = X86_PAGE_SIZE;
775	}
776	GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
777	uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
778	if (cbMaxRead2 < cbMaxRead)
779	cbMaxRead = cbMaxRead2;
780	/** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
781	}
782
783	/*
784	* Get the TLB entry for this piece of code.
785	*/
786	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.CodeTlb, GCPtrFirst);
787	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag);
788	if (pTlbe->uTag == uTag)
789	{
790	/* likely when executing lots of code, otherwise unlikely */
791	# ifdef VBOX_WITH_STATISTICS
792	pVCpu->iem.s.CodeTlb.cTlbHits++;
793	# endif
794	}
795	else
796	{
797	pVCpu->iem.s.CodeTlb.cTlbMisses++;
798	PGMPTWALK Walk;
799	int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
800	if (RT_FAILURE(rc))
801	{
802	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
803	/** @todo Nested VMX: Need to handle EPT violation/misconfig here? */
804	Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
805	#endif
806	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
807	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
808	}
809
810	AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
811	Assert(Walk.fSucceeded);
812	pTlbe->uTag = uTag;
813	pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A))
814	\| (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
815	pTlbe->GCPhys = Walk.GCPhys;
816	pTlbe->pbMappingR3 = NULL;
817	}
818
819	/*
820	* Check TLB page table level access flags.
821	*/
822	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_EXEC))
823	{
824	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
825	{
826	Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
827	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
828	}
829	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
830	{
831	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
832	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
833	}
834	}
835
836	/*
837	* Look up the physical page info if necessary.
838	*/
839	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
840	{ /* not necessary */ }
841	else
842	{
843	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
844	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
845	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
846	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
847	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
848	{ /* likely */ }
849	else
850	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
851	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
852	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
853	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
854	&pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
855	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
856	}
857
858	# if defined(IN_RING3) \|\| defined(IN_RING0) /** @todo fixme */
859	/*
860	* Try do a direct read using the pbMappingR3 pointer.
861	*/
862	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ))
863	== pVCpu->iem.s.CodeTlb.uTlbPhysRev)
864	{
865	uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
866	pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
867	if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
868	{
869	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
870	pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
871	}
872	else
873	{
874	uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
875	Assert(cbInstr < cbMaxRead);
876	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
877	pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
878	}
879	if (cbDst <= cbMaxRead)
880	{
881	pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
882	pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
883	pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
884	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
885	return;
886	}
887	pVCpu->iem.s.pbInstrBuf = NULL;
888
889	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
890	pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
891	}
892	else
893	# endif
894	#if 0
895	/*
896	* If there is no special read handling, so we can read a bit more and
897	* put it in the prefetch buffer.
898	*/
899	if ( cbDst < cbMaxRead
900	&& (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
901	{
902	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
903	&pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
904	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
905	{ /* likely */ }
906	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
907	{
908	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
909	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
910	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
911	AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
912	}
913	else
914	{
915	Log((RT_SUCCESS(rcStrict)
916	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
917	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
918	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
919	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
920	}
921	}
922	/*
923	* Special read handling, so only read exactly what's needed.
924	* This is a highly unlikely scenario.
925	*/
926	else
927	#endif
928	{
929	pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
930	uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
931	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
932	pvDst, cbToRead, PGMACCESSORIGIN_IEM);
933	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
934	{ /* likely */ }
935	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
936	{
937	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
938	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
939	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
940	AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
941	}
942	else
943	{
944	Log((RT_SUCCESS(rcStrict)
945	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
946	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
947	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
948	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
949	}
950	pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
951	if (cbToRead == cbDst)
952	return;
953	}
954
955	/*
956	* More to read, loop.
957	*/
958	cbDst -= cbMaxRead;
959	pvDst = (uint8_t *)pvDst + cbMaxRead;
960	}
961	#else
962	RT_NOREF(pvDst, cbDst);
963	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
964	#endif
965	}
966
967	#else
968
969	/**
970	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
971	* exception if it fails.
972	*
973	* @returns Strict VBox status code.
974	* @param pVCpu The cross context virtual CPU structure of the
975	* calling thread.
976	* @param cbMin The minimum number of bytes relative offOpcode
977	* that must be read.
978	*/
979	VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
980	{
981	/*
982	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
983	*
984	* First translate CS:rIP to a physical address.
985	*/
986	uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
987	uint32_t cbToTryRead;
988	RTGCPTR GCPtrNext;
989	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
990	{
991	cbToTryRead = GUEST_PAGE_SIZE;
992	GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
993	if (!IEM_IS_CANONICAL(GCPtrNext))
994	return iemRaiseGeneralProtectionFault0(pVCpu);
995	}
996	else
997	{
998	uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
999	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1000	GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1001	if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1002	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1003	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1004	if (!cbToTryRead) /* overflowed */
1005	{
1006	Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1007	cbToTryRead = UINT32_MAX;
1008	/** @todo check out wrapping around the code segment. */
1009	}
1010	if (cbToTryRead < cbMin - cbLeft)
1011	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1012	GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1013	}
1014
1015	/* Only read up to the end of the page, and make sure we don't read more
1016	than the opcode buffer can hold. */
1017	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1018	if (cbToTryRead > cbLeftOnPage)
1019	cbToTryRead = cbLeftOnPage;
1020	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
1021	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
1022	/** @todo r=bird: Convert assertion into undefined opcode exception? */
1023	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1024
1025	PGMPTWALK Walk;
1026	int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
1027	if (RT_FAILURE(rc))
1028	{
1029	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1030	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1031	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1032	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1033	#endif
1034	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1035	}
1036	if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
1037	{
1038	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1040	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1041	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1042	#endif
1043	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1044	}
1045	if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1046	{
1047	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1048	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1049	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1050	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1051	#endif
1052	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1053	}
1054	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1055	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
1056	/** @todo Check reserved bits and such stuff. PGM is better at doing
1057	* that, so do it when implementing the guest virtual address
1058	* TLB... */
1059
1060	/*
1061	* Read the bytes at this address.
1062	*
1063	* We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1064	* and since PATM should only patch the start of an instruction there
1065	* should be no need to check again here.
1066	*/
1067	if (!pVCpu->iem.s.fBypassHandlers)
1068	{
1069	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
1070	cbToTryRead, PGMACCESSORIGIN_IEM);
1071	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1072	{ /* likely */ }
1073	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1074	{
1075	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1076	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1077	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1078	}
1079	else
1080	{
1081	Log((RT_SUCCESS(rcStrict)
1082	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1083	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1084	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1085	return rcStrict;
1086	}
1087	}
1088	else
1089	{
1090	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
1091	if (RT_SUCCESS(rc))
1092	{ /* likely */ }
1093	else
1094	{
1095	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
1096	return rc;
1097	}
1098	}
1099	pVCpu->iem.s.cbOpcode += cbToTryRead;
1100	Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
1101
1102	return VINF_SUCCESS;
1103	}
1104
1105	#endif /* !IEM_WITH_CODE_TLB */
1106	#ifndef IEM_WITH_SETJMP
1107
1108	/**
1109	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
1110	*
1111	* @returns Strict VBox status code.
1112	* @param pVCpu The cross context virtual CPU structure of the
1113	* calling thread.
1114	* @param pb Where to return the opcode byte.
1115	*/
1116	VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
1117	{
1118	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1119	if (rcStrict == VINF_SUCCESS)
1120	{
1121	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1122	*pb = pVCpu->iem.s.abOpcode[offOpcode];
1123	pVCpu->iem.s.offOpcode = offOpcode + 1;
1124	}
1125	else
1126	*pb = 0;
1127	return rcStrict;
1128	}
1129
1130	#else /* IEM_WITH_SETJMP */
1131
1132	/**
1133	* Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
1134	*
1135	* @returns The opcode byte.
1136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1137	*/
1138	uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1139	{
1140	# ifdef IEM_WITH_CODE_TLB
1141	uint8_t u8;
1142	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
1143	return u8;
1144	# else
1145	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1146	if (rcStrict == VINF_SUCCESS)
1147	return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
1148	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1149	# endif
1150	}
1151
1152	#endif /* IEM_WITH_SETJMP */
1153
1154	#ifndef IEM_WITH_SETJMP
1155
1156	/**
1157	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
1158	*
1159	* @returns Strict VBox status code.
1160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1161	* @param pu16 Where to return the opcode dword.
1162	*/
1163	VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1164	{
1165	uint8_t u8;
1166	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1167	if (rcStrict == VINF_SUCCESS)
1168	*pu16 = (int8_t)u8;
1169	return rcStrict;
1170	}
1171
1172
1173	/**
1174	* Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
1175	*
1176	* @returns Strict VBox status code.
1177	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1178	* @param pu32 Where to return the opcode dword.
1179	*/
1180	VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1181	{
1182	uint8_t u8;
1183	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1184	if (rcStrict == VINF_SUCCESS)
1185	*pu32 = (int8_t)u8;
1186	return rcStrict;
1187	}
1188
1189
1190	/**
1191	* Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
1192	*
1193	* @returns Strict VBox status code.
1194	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1195	* @param pu64 Where to return the opcode qword.
1196	*/
1197	VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1198	{
1199	uint8_t u8;
1200	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1201	if (rcStrict == VINF_SUCCESS)
1202	*pu64 = (int8_t)u8;
1203	return rcStrict;
1204	}
1205
1206	#endif /* !IEM_WITH_SETJMP */
1207
1208
1209	#ifndef IEM_WITH_SETJMP
1210
1211	/**
1212	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
1213	*
1214	* @returns Strict VBox status code.
1215	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1216	* @param pu16 Where to return the opcode word.
1217	*/
1218	VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1219	{
1220	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1221	if (rcStrict == VINF_SUCCESS)
1222	{
1223	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1224	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1225	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1226	# else
1227	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1228	# endif
1229	pVCpu->iem.s.offOpcode = offOpcode + 2;
1230	}
1231	else
1232	*pu16 = 0;
1233	return rcStrict;
1234	}
1235
1236	#else /* IEM_WITH_SETJMP */
1237
1238	/**
1239	* Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
1240	*
1241	* @returns The opcode word.
1242	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1243	*/
1244	uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1245	{
1246	# ifdef IEM_WITH_CODE_TLB
1247	uint16_t u16;
1248	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
1249	return u16;
1250	# else
1251	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1252	if (rcStrict == VINF_SUCCESS)
1253	{
1254	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1255	pVCpu->iem.s.offOpcode += 2;
1256	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1257	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1258	# else
1259	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1260	# endif
1261	}
1262	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1263	# endif
1264	}
1265
1266	#endif /* IEM_WITH_SETJMP */
1267
1268	#ifndef IEM_WITH_SETJMP
1269
1270	/**
1271	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1272	*
1273	* @returns Strict VBox status code.
1274	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1275	* @param pu32 Where to return the opcode double word.
1276	*/
1277	VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1278	{
1279	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1280	if (rcStrict == VINF_SUCCESS)
1281	{
1282	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1283	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1284	pVCpu->iem.s.offOpcode = offOpcode + 2;
1285	}
1286	else
1287	*pu32 = 0;
1288	return rcStrict;
1289	}
1290
1291
1292	/**
1293	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1294	*
1295	* @returns Strict VBox status code.
1296	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1297	* @param pu64 Where to return the opcode quad word.
1298	*/
1299	VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1300	{
1301	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1302	if (rcStrict == VINF_SUCCESS)
1303	{
1304	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1305	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1306	pVCpu->iem.s.offOpcode = offOpcode + 2;
1307	}
1308	else
1309	*pu64 = 0;
1310	return rcStrict;
1311	}
1312
1313	#endif /* !IEM_WITH_SETJMP */
1314
1315	#ifndef IEM_WITH_SETJMP
1316
1317	/**
1318	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1319	*
1320	* @returns Strict VBox status code.
1321	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1322	* @param pu32 Where to return the opcode dword.
1323	*/
1324	VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1325	{
1326	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1327	if (rcStrict == VINF_SUCCESS)
1328	{
1329	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1330	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1331	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1332	# else
1333	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1334	pVCpu->iem.s.abOpcode[offOpcode + 1],
1335	pVCpu->iem.s.abOpcode[offOpcode + 2],
1336	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1337	# endif
1338	pVCpu->iem.s.offOpcode = offOpcode + 4;
1339	}
1340	else
1341	*pu32 = 0;
1342	return rcStrict;
1343	}
1344
1345	#else /* IEM_WITH_SETJMP */
1346
1347	/**
1348	* Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
1349	*
1350	* @returns The opcode dword.
1351	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1352	*/
1353	uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1354	{
1355	# ifdef IEM_WITH_CODE_TLB
1356	uint32_t u32;
1357	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
1358	return u32;
1359	# else
1360	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1361	if (rcStrict == VINF_SUCCESS)
1362	{
1363	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1364	pVCpu->iem.s.offOpcode = offOpcode + 4;
1365	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1366	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1367	# else
1368	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1369	pVCpu->iem.s.abOpcode[offOpcode + 1],
1370	pVCpu->iem.s.abOpcode[offOpcode + 2],
1371	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1372	# endif
1373	}
1374	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1375	# endif
1376	}
1377
1378	#endif /* IEM_WITH_SETJMP */
1379
1380	#ifndef IEM_WITH_SETJMP
1381
1382	/**
1383	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1384	*
1385	* @returns Strict VBox status code.
1386	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1387	* @param pu64 Where to return the opcode dword.
1388	*/
1389	VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1390	{
1391	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1392	if (rcStrict == VINF_SUCCESS)
1393	{
1394	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1395	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1396	pVCpu->iem.s.abOpcode[offOpcode + 1],
1397	pVCpu->iem.s.abOpcode[offOpcode + 2],
1398	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1399	pVCpu->iem.s.offOpcode = offOpcode + 4;
1400	}
1401	else
1402	*pu64 = 0;
1403	return rcStrict;
1404	}
1405
1406
1407	/**
1408	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1409	*
1410	* @returns Strict VBox status code.
1411	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1412	* @param pu64 Where to return the opcode qword.
1413	*/
1414	VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1415	{
1416	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1417	if (rcStrict == VINF_SUCCESS)
1418	{
1419	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1420	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1421	pVCpu->iem.s.abOpcode[offOpcode + 1],
1422	pVCpu->iem.s.abOpcode[offOpcode + 2],
1423	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1424	pVCpu->iem.s.offOpcode = offOpcode + 4;
1425	}
1426	else
1427	*pu64 = 0;
1428	return rcStrict;
1429	}
1430
1431	#endif /* !IEM_WITH_SETJMP */
1432
1433	#ifndef IEM_WITH_SETJMP
1434
1435	/**
1436	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1437	*
1438	* @returns Strict VBox status code.
1439	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1440	* @param pu64 Where to return the opcode qword.
1441	*/
1442	VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1443	{
1444	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1445	if (rcStrict == VINF_SUCCESS)
1446	{
1447	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1448	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1449	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1450	# else
1451	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1452	pVCpu->iem.s.abOpcode[offOpcode + 1],
1453	pVCpu->iem.s.abOpcode[offOpcode + 2],
1454	pVCpu->iem.s.abOpcode[offOpcode + 3],
1455	pVCpu->iem.s.abOpcode[offOpcode + 4],
1456	pVCpu->iem.s.abOpcode[offOpcode + 5],
1457	pVCpu->iem.s.abOpcode[offOpcode + 6],
1458	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1459	# endif
1460	pVCpu->iem.s.offOpcode = offOpcode + 8;
1461	}
1462	else
1463	*pu64 = 0;
1464	return rcStrict;
1465	}
1466
1467	#else /* IEM_WITH_SETJMP */
1468
1469	/**
1470	* Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
1471	*
1472	* @returns The opcode qword.
1473	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1474	*/
1475	uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1476	{
1477	# ifdef IEM_WITH_CODE_TLB
1478	uint64_t u64;
1479	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
1480	return u64;
1481	# else
1482	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1483	if (rcStrict == VINF_SUCCESS)
1484	{
1485	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1486	pVCpu->iem.s.offOpcode = offOpcode + 8;
1487	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1488	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1489	# else
1490	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1491	pVCpu->iem.s.abOpcode[offOpcode + 1],
1492	pVCpu->iem.s.abOpcode[offOpcode + 2],
1493	pVCpu->iem.s.abOpcode[offOpcode + 3],
1494	pVCpu->iem.s.abOpcode[offOpcode + 4],
1495	pVCpu->iem.s.abOpcode[offOpcode + 5],
1496	pVCpu->iem.s.abOpcode[offOpcode + 6],
1497	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1498	# endif
1499	}
1500	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1501	# endif
1502	}
1503
1504	#endif /* IEM_WITH_SETJMP */
1505
1506
1507
1508	/** @name Misc Worker Functions.
1509	* @{
1510	*/
1511
1512	/**
1513	* Gets the exception class for the specified exception vector.
1514	*
1515	* @returns The class of the specified exception.
1516	* @param uVector The exception vector.
1517	*/
1518	static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT
1519	{
1520	Assert(uVector <= X86_XCPT_LAST);
1521	switch (uVector)
1522	{
1523	case X86_XCPT_DE:
1524	case X86_XCPT_TS:
1525	case X86_XCPT_NP:
1526	case X86_XCPT_SS:
1527	case X86_XCPT_GP:
1528	case X86_XCPT_SX: /* AMD only */
1529	return IEMXCPTCLASS_CONTRIBUTORY;
1530
1531	case X86_XCPT_PF:
1532	case X86_XCPT_VE: /* Intel only */
1533	return IEMXCPTCLASS_PAGE_FAULT;
1534
1535	case X86_XCPT_DF:
1536	return IEMXCPTCLASS_DOUBLE_FAULT;
1537	}
1538	return IEMXCPTCLASS_BENIGN;
1539	}
1540
1541
1542	/**
1543	* Evaluates how to handle an exception caused during delivery of another event
1544	* (exception / interrupt).
1545	*
1546	* @returns How to handle the recursive exception.
1547	* @param pVCpu The cross context virtual CPU structure of the
1548	* calling thread.
1549	* @param fPrevFlags The flags of the previous event.
1550	* @param uPrevVector The vector of the previous event.
1551	* @param fCurFlags The flags of the current exception.
1552	* @param uCurVector The vector of the current exception.
1553	* @param pfXcptRaiseInfo Where to store additional information about the
1554	* exception condition. Optional.
1555	*/
1556	VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
1557	uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
1558	{
1559	/*
1560	* Only CPU exceptions can be raised while delivering other events, software interrupt
1561	* (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
1562	*/
1563	AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
1564	Assert(pVCpu); RT_NOREF(pVCpu);
1565	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
1566
1567	IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
1568	IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
1569	if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1570	{
1571	IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
1572	if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
1573	{
1574	IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
1575	if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
1576	&& ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
1577	\|\| enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
1578	{
1579	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1580	fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
1581	: IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
1582	Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
1583	uCurVector, pVCpu->cpum.GstCtx.cr2));
1584	}
1585	else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1586	&& enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
1587	{
1588	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1589	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
1590	}
1591	else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
1592	&& ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1593	\|\| enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
1594	{
1595	enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
1596	Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
1597	}
1598	}
1599	else
1600	{
1601	if (uPrevVector == X86_XCPT_NMI)
1602	{
1603	fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
1604	if (uCurVector == X86_XCPT_PF)
1605	{
1606	fRaiseInfo \|= IEMXCPTRAISEINFO_NMI_PF;
1607	Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
1608	}
1609	}
1610	else if ( uPrevVector == X86_XCPT_AC
1611	&& uCurVector == X86_XCPT_AC)
1612	{
1613	enmRaise = IEMXCPTRAISE_CPU_HANG;
1614	fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
1615	Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
1616	}
1617	}
1618	}
1619	else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
1620	{
1621	fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
1622	if (uCurVector == X86_XCPT_PF)
1623	fRaiseInfo \|= IEMXCPTRAISEINFO_EXT_INT_PF;
1624	}
1625	else
1626	{
1627	Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
1628	fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
1629	}
1630
1631	if (pfXcptRaiseInfo)
1632	*pfXcptRaiseInfo = fRaiseInfo;
1633	return enmRaise;
1634	}
1635
1636
1637	/**
1638	* Enters the CPU shutdown state initiated by a triple fault or other
1639	* unrecoverable conditions.
1640	*
1641	* @returns Strict VBox status code.
1642	* @param pVCpu The cross context virtual CPU structure of the
1643	* calling thread.
1644	*/
1645	static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT
1646	{
1647	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
1648	IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
1649
1650	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
1651	{
1652	Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
1653	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
1654	}
1655
1656	RT_NOREF(pVCpu);
1657	return VINF_EM_TRIPLE_FAULT;
1658	}
1659
1660
1661	/**
1662	* Validates a new SS segment.
1663	*
1664	* @returns VBox strict status code.
1665	* @param pVCpu The cross context virtual CPU structure of the
1666	* calling thread.
1667	* @param NewSS The new SS selctor.
1668	* @param uCpl The CPL to load the stack for.
1669	* @param pDesc Where to return the descriptor.
1670	*/
1671	static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT
1672	{
1673	/* Null selectors are not allowed (we're not called for dispatching
1674	interrupts with SS=0 in long mode). */
1675	if (!(NewSS & X86_SEL_MASK_OFF_RPL))
1676	{
1677	Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
1678	return iemRaiseTaskSwitchFault0(pVCpu);
1679	}
1680
1681	/** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
1682	if ((NewSS & X86_SEL_RPL) != uCpl)
1683	{
1684	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
1685	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1686	}
1687
1688	/*
1689	* Read the descriptor.
1690	*/
1691	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
1692	if (rcStrict != VINF_SUCCESS)
1693	return rcStrict;
1694
1695	/*
1696	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1697	*/
1698	if (!pDesc->Legacy.Gen.u1DescType)
1699	{
1700	Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1701	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1702	}
1703
1704	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1705	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1706	{
1707	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1708	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1709	}
1710	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1711	{
1712	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1713	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1714	}
1715
1716	/* Is it there? */
1717	/** @todo testcase: Is this checked before the canonical / limit check below? */
1718	if (!pDesc->Legacy.Gen.u1Present)
1719	{
1720	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1721	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
1722	}
1723
1724	return VINF_SUCCESS;
1725	}
1726
1727	/** @} */
1728
1729
1730	/** @name Raising Exceptions.
1731	*
1732	* @{
1733	*/
1734
1735
1736	/**
1737	* Loads the specified stack far pointer from the TSS.
1738	*
1739	* @returns VBox strict status code.
1740	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1741	* @param uCpl The CPL to load the stack for.
1742	* @param pSelSS Where to return the new stack segment.
1743	* @param puEsp Where to return the new stack pointer.
1744	*/
1745	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT
1746	{
1747	VBOXSTRICTRC rcStrict;
1748	Assert(uCpl < 4);
1749
1750	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1751	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
1752	{
1753	/*
1754	* 16-bit TSS (X86TSS16).
1755	*/
1756	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1757	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1758	{
1759	uint32_t off = uCpl * 4 + 2;
1760	if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1761	{
1762	/** @todo check actual access pattern here. */
1763	uint32_t u32Tmp = 0; /* gcc maybe... */
1764	rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1765	if (rcStrict == VINF_SUCCESS)
1766	{
1767	*puEsp = RT_LOWORD(u32Tmp);
1768	*pSelSS = RT_HIWORD(u32Tmp);
1769	return VINF_SUCCESS;
1770	}
1771	}
1772	else
1773	{
1774	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1775	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1776	}
1777	break;
1778	}
1779
1780	/*
1781	* 32-bit TSS (X86TSS32).
1782	*/
1783	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1784	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1785	{
1786	uint32_t off = uCpl * 8 + 4;
1787	if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1788	{
1789	/** @todo check actual access pattern here. */
1790	uint64_t u64Tmp;
1791	rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1792	if (rcStrict == VINF_SUCCESS)
1793	{
1794	*puEsp = u64Tmp & UINT32_MAX;
1795	*pSelSS = (RTSEL)(u64Tmp >> 32);
1796	return VINF_SUCCESS;
1797	}
1798	}
1799	else
1800	{
1801	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1802	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1803	}
1804	break;
1805	}
1806
1807	default:
1808	AssertFailed();
1809	rcStrict = VERR_IEM_IPE_4;
1810	break;
1811	}
1812
1813	puEsp = 0; / make gcc happy */
1814	pSelSS = 0; / make gcc happy */
1815	return rcStrict;
1816	}
1817
1818
1819	/**
1820	* Loads the specified stack pointer from the 64-bit TSS.
1821	*
1822	* @returns VBox strict status code.
1823	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1824	* @param uCpl The CPL to load the stack for.
1825	* @param uIst The interrupt stack table index, 0 if to use uCpl.
1826	* @param puRsp Where to return the new stack pointer.
1827	*/
1828	static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT
1829	{
1830	Assert(uCpl < 4);
1831	Assert(uIst < 8);
1832	puRsp = 0; / make gcc happy */
1833
1834	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1835	AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
1836
1837	uint32_t off;
1838	if (uIst)
1839	off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
1840	else
1841	off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
1842	if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
1843	{
1844	Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
1845	return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1846	}
1847
1848	return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1849	}
1850
1851
1852	/**
1853	* Adjust the CPU state according to the exception being raised.
1854	*
1855	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1856	* @param u8Vector The exception that has been raised.
1857	*/
1858	DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
1859	{
1860	switch (u8Vector)
1861	{
1862	case X86_XCPT_DB:
1863	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
1864	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
1865	break;
1866	/** @todo Read the AMD and Intel exception reference... */
1867	}
1868	}
1869
1870
1871	/**
1872	* Implements exceptions and interrupts for real mode.
1873	*
1874	* @returns VBox strict status code.
1875	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1876	* @param cbInstr The number of bytes to offset rIP by in the return
1877	* address.
1878	* @param u8Vector The interrupt / exception vector number.
1879	* @param fFlags The flags.
1880	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1881	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1882	*/
1883	static VBOXSTRICTRC
1884	iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
1885	uint8_t cbInstr,
1886	uint8_t u8Vector,
1887	uint32_t fFlags,
1888	uint16_t uErr,
1889	uint64_t uCr2) RT_NOEXCEPT
1890	{
1891	NOREF(uErr); NOREF(uCr2);
1892	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1893
1894	/*
1895	* Read the IDT entry.
1896	*/
1897	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1898	{
1899	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
1900	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1901	}
1902	RTFAR16 Idte;
1903	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t )&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) u8Vector);
1904	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1905	{
1906	Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
1907	return rcStrict;
1908	}
1909
1910	/*
1911	* Push the stack frame.
1912	*/
1913	uint16_t *pu16Frame;
1914	uint64_t uNewRsp;
1915	rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &uNewRsp);
1916	if (rcStrict != VINF_SUCCESS)
1917	return rcStrict;
1918
1919	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
1920	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
1921	AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
1922	if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
1923	fEfl \|= UINT16_C(0xf000);
1924	#endif
1925	pu16Frame[2] = (uint16_t)fEfl;
1926	pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
1927	pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
1928	rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
1929	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1930	return rcStrict;
1931
1932	/*
1933	* Load the vector address into cs:ip and make exception specific state
1934	* adjustments.
1935	*/
1936	pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
1937	pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
1938	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1939	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
1940	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1941	pVCpu->cpum.GstCtx.rip = Idte.off;
1942	fEfl &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_AC);
1943	IEMMISC_SET_EFL(pVCpu, fEfl);
1944
1945	/** @todo do we actually do this in real mode? */
1946	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1947	iemRaiseXcptAdjustState(pVCpu, u8Vector);
1948
1949	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1950	}
1951
1952
1953	/**
1954	* Loads a NULL data selector into when coming from V8086 mode.
1955	*
1956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1957	* @param pSReg Pointer to the segment register.
1958	*/
1959	DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1960	{
1961	pSReg->Sel = 0;
1962	pSReg->ValidSel = 0;
1963	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1964	{
1965	/* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
1966	pSReg->Attr.u &= X86DESCATTR_DT \| X86DESCATTR_TYPE \| X86DESCATTR_DPL \| X86DESCATTR_G \| X86DESCATTR_D;
1967	pSReg->Attr.u \|= X86DESCATTR_UNUSABLE;
1968	}
1969	else
1970	{
1971	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1972	/** @todo check this on AMD-V */
1973	pSReg->u64Base = 0;
1974	pSReg->u32Limit = 0;
1975	}
1976	}
1977
1978
1979	/**
1980	* Loads a segment selector during a task switch in V8086 mode.
1981	*
1982	* @param pSReg Pointer to the segment register.
1983	* @param uSel The selector value to load.
1984	*/
1985	DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
1986	{
1987	/* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
1988	pSReg->Sel = uSel;
1989	pSReg->ValidSel = uSel;
1990	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1991	pSReg->u64Base = uSel << 4;
1992	pSReg->u32Limit = 0xffff;
1993	pSReg->Attr.u = 0xf3;
1994	}
1995
1996
1997	/**
1998	* Loads a segment selector during a task switch in protected mode.
1999	*
2000	* In this task switch scenario, we would throw \#TS exceptions rather than
2001	* \#GPs.
2002	*
2003	* @returns VBox strict status code.
2004	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2005	* @param pSReg Pointer to the segment register.
2006	* @param uSel The new selector value.
2007	*
2008	* @remarks This does _not_ handle CS or SS.
2009	* @remarks This expects pVCpu->iem.s.uCpl to be up to date.
2010	*/
2011	static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT
2012	{
2013	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2014
2015	/* Null data selector. */
2016	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2017	{
2018	iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
2019	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2020	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2021	return VINF_SUCCESS;
2022	}
2023
2024	/* Fetch the descriptor. */
2025	IEMSELDESC Desc;
2026	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
2027	if (rcStrict != VINF_SUCCESS)
2028	{
2029	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
2030	VBOXSTRICTRC_VAL(rcStrict)));
2031	return rcStrict;
2032	}
2033
2034	/* Must be a data segment or readable code segment. */
2035	if ( !Desc.Legacy.Gen.u1DescType
2036	\|\| (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2037	{
2038	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
2039	Desc.Legacy.Gen.u4Type));
2040	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2041	}
2042
2043	/* Check privileges for data segments and non-conforming code segments. */
2044	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2045	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2046	{
2047	/* The RPL and the new CPL must be less than or equal to the DPL. */
2048	if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
2049	\|\| (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
2050	{
2051	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
2052	uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
2053	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2054	}
2055	}
2056
2057	/* Is it there? */
2058	if (!Desc.Legacy.Gen.u1Present)
2059	{
2060	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
2061	return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2062	}
2063
2064	/* The base and limit. */
2065	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2066	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
2067
2068	/*
2069	* Ok, everything checked out fine. Now set the accessed bit before
2070	* committing the result into the registers.
2071	*/
2072	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2073	{
2074	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2075	if (rcStrict != VINF_SUCCESS)
2076	return rcStrict;
2077	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2078	}
2079
2080	/* Commit */
2081	pSReg->Sel = uSel;
2082	pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2083	pSReg->u32Limit = cbLimit;
2084	pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
2085	pSReg->ValidSel = uSel;
2086	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2087	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2088	pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
2089
2090	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2091	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2092	return VINF_SUCCESS;
2093	}
2094
2095
2096	/**
2097	* Performs a task switch.
2098	*
2099	* If the task switch is the result of a JMP, CALL or IRET instruction, the
2100	* caller is responsible for performing the necessary checks (like DPL, TSS
2101	* present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
2102	* reference for JMP, CALL, IRET.
2103	*
2104	* If the task switch is the due to a software interrupt or hardware exception,
2105	* the caller is responsible for validating the TSS selector and descriptor. See
2106	* Intel Instruction reference for INT n.
2107	*
2108	* @returns VBox strict status code.
2109	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2110	* @param enmTaskSwitch The cause of the task switch.
2111	* @param uNextEip The EIP effective after the task switch.
2112	* @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
2113	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2114	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2115	* @param SelTSS The TSS selector of the new task.
2116	* @param pNewDescTSS Pointer to the new TSS descriptor.
2117	*/
2118	VBOXSTRICTRC
2119	iemTaskSwitch(PVMCPUCC pVCpu,
2120	IEMTASKSWITCH enmTaskSwitch,
2121	uint32_t uNextEip,
2122	uint32_t fFlags,
2123	uint16_t uErr,
2124	uint64_t uCr2,
2125	RTSEL SelTSS,
2126	PIEMSELDESC pNewDescTSS) RT_NOEXCEPT
2127	{
2128	Assert(!IEM_IS_REAL_MODE(pVCpu));
2129	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2130	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2131
2132	uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
2133	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2134	\|\| uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2135	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2136	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2137
2138	bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2139	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2140
2141	Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
2142	fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
2143
2144	/* Update CR2 in case it's a page-fault. */
2145	/** @todo This should probably be done much earlier in IEM/PGM. See
2146	* @bugref{5653#c49}. */
2147	if (fFlags & IEM_XCPT_FLAGS_CR2)
2148	pVCpu->cpum.GstCtx.cr2 = uCr2;
2149
2150	/*
2151	* Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
2152	* subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
2153	*/
2154	uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow \| (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
2155	uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
2156	if (uNewTSSLimit < uNewTSSLimitMin)
2157	{
2158	Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
2159	enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
2160	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2161	}
2162
2163	/*
2164	* Task switches in VMX non-root mode always cause task switches.
2165	* The new TSS must have been read and validated (DPL, limits etc.) before a
2166	* task-switch VM-exit commences.
2167	*
2168	* See Intel spec. 25.4.2 "Treatment of Task Switches".
2169	*/
2170	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
2171	{
2172	Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
2173	IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
2174	}
2175
2176	/*
2177	* The SVM nested-guest intercept for task-switch takes priority over all exceptions
2178	* after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
2179	*/
2180	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
2181	{
2182	uint32_t const uExitInfo1 = SelTSS;
2183	uint32_t uExitInfo2 = uErr;
2184	switch (enmTaskSwitch)
2185	{
2186	case IEMTASKSWITCH_JUMP: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_JUMP; break;
2187	case IEMTASKSWITCH_IRET: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_IRET; break;
2188	default: break;
2189	}
2190	if (fFlags & IEM_XCPT_FLAGS_ERR)
2191	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
2192	if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
2193	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
2194
2195	Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
2196	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
2197	RT_NOREF2(uExitInfo1, uExitInfo2);
2198	}
2199
2200	/*
2201	* Check the current TSS limit. The last written byte to the current TSS during the
2202	* task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
2203	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2204	*
2205	* The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
2206	* end up with smaller than "legal" TSS limits.
2207	*/
2208	uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
2209	uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
2210	if (uCurTSSLimit < uCurTSSLimitMin)
2211	{
2212	Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
2213	enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
2214	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2215	}
2216
2217	/*
2218	* Verify that the new TSS can be accessed and map it. Map only the required contents
2219	* and not the entire TSS.
2220	*/
2221	void *pvNewTSS;
2222	uint32_t const cbNewTSS = uNewTSSLimitMin + 1;
2223	RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
2224	AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
2225	/** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
2226	* not perform correct translation if this happens. See Intel spec. 7.2.1
2227	* "Task-State Segment". */
2228	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW, 0);
2229	if (rcStrict != VINF_SUCCESS)
2230	{
2231	Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
2232	cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
2233	return rcStrict;
2234	}
2235
2236	/*
2237	* Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
2238	*/
2239	uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
2240	if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
2241	\|\| enmTaskSwitch == IEMTASKSWITCH_IRET)
2242	{
2243	PX86DESC pDescCurTSS;
2244	rcStrict = iemMemMap(pVCpu, (void *)&pDescCurTSS, sizeof(pDescCurTSS), UINT8_MAX,
2245	pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2246	if (rcStrict != VINF_SUCCESS)
2247	{
2248	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2249	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2250	return rcStrict;
2251	}
2252
2253	pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2254	rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
2255	if (rcStrict != VINF_SUCCESS)
2256	{
2257	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2258	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2259	return rcStrict;
2260	}
2261
2262	/* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
2263	if (enmTaskSwitch == IEMTASKSWITCH_IRET)
2264	{
2265	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2266	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2267	u32EFlags &= ~X86_EFL_NT;
2268	}
2269	}
2270
2271	/*
2272	* Save the CPU state into the current TSS.
2273	*/
2274	RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
2275	if (GCPtrNewTSS == GCPtrCurTSS)
2276	{
2277	Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur\|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
2278	Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
2279	pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
2280	pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
2281	pVCpu->cpum.GstCtx.ldtr.Sel));
2282	}
2283	if (fIsNewTSS386)
2284	{
2285	/*
2286	* Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
2287	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2288	*/
2289	void *pvCurTSS32;
2290	uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
2291	uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
2292	AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
2293	rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2294	if (rcStrict != VINF_SUCCESS)
2295	{
2296	Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2297	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2298	return rcStrict;
2299	}
2300
2301	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2302	PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
2303	pCurTSS32->eip = uNextEip;
2304	pCurTSS32->eflags = u32EFlags;
2305	pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
2306	pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
2307	pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
2308	pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
2309	pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
2310	pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
2311	pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
2312	pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
2313	pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
2314	pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
2315	pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
2316	pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
2317	pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
2318	pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
2319
2320	rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
2321	if (rcStrict != VINF_SUCCESS)
2322	{
2323	Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2324	VBOXSTRICTRC_VAL(rcStrict)));
2325	return rcStrict;
2326	}
2327	}
2328	else
2329	{
2330	/*
2331	* Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
2332	*/
2333	void *pvCurTSS16;
2334	uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
2335	uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
2336	AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
2337	rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2338	if (rcStrict != VINF_SUCCESS)
2339	{
2340	Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2341	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2342	return rcStrict;
2343	}
2344
2345	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2346	PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
2347	pCurTSS16->ip = uNextEip;
2348	pCurTSS16->flags = u32EFlags;
2349	pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
2350	pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
2351	pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
2352	pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
2353	pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
2354	pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
2355	pCurTSS16->si = pVCpu->cpum.GstCtx.si;
2356	pCurTSS16->di = pVCpu->cpum.GstCtx.di;
2357	pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
2358	pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
2359	pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
2360	pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
2361
2362	rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
2363	if (rcStrict != VINF_SUCCESS)
2364	{
2365	Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2366	VBOXSTRICTRC_VAL(rcStrict)));
2367	return rcStrict;
2368	}
2369	}
2370
2371	/*
2372	* Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
2373	*/
2374	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2375	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2376	{
2377	/* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
2378	PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
2379	pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
2380	}
2381
2382	/*
2383	* Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
2384	* it's done further below with error handling (e.g. CR3 changes will go through PGM).
2385	*/
2386	uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
2387	uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
2388	bool fNewDebugTrap;
2389	if (fIsNewTSS386)
2390	{
2391	PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
2392	uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
2393	uNewEip = pNewTSS32->eip;
2394	uNewEflags = pNewTSS32->eflags;
2395	uNewEax = pNewTSS32->eax;
2396	uNewEcx = pNewTSS32->ecx;
2397	uNewEdx = pNewTSS32->edx;
2398	uNewEbx = pNewTSS32->ebx;
2399	uNewEsp = pNewTSS32->esp;
2400	uNewEbp = pNewTSS32->ebp;
2401	uNewEsi = pNewTSS32->esi;
2402	uNewEdi = pNewTSS32->edi;
2403	uNewES = pNewTSS32->es;
2404	uNewCS = pNewTSS32->cs;
2405	uNewSS = pNewTSS32->ss;
2406	uNewDS = pNewTSS32->ds;
2407	uNewFS = pNewTSS32->fs;
2408	uNewGS = pNewTSS32->gs;
2409	uNewLdt = pNewTSS32->selLdt;
2410	fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
2411	}
2412	else
2413	{
2414	PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
2415	uNewCr3 = 0;
2416	uNewEip = pNewTSS16->ip;
2417	uNewEflags = pNewTSS16->flags;
2418	uNewEax = UINT32_C(0xffff0000) \| pNewTSS16->ax;
2419	uNewEcx = UINT32_C(0xffff0000) \| pNewTSS16->cx;
2420	uNewEdx = UINT32_C(0xffff0000) \| pNewTSS16->dx;
2421	uNewEbx = UINT32_C(0xffff0000) \| pNewTSS16->bx;
2422	uNewEsp = UINT32_C(0xffff0000) \| pNewTSS16->sp;
2423	uNewEbp = UINT32_C(0xffff0000) \| pNewTSS16->bp;
2424	uNewEsi = UINT32_C(0xffff0000) \| pNewTSS16->si;
2425	uNewEdi = UINT32_C(0xffff0000) \| pNewTSS16->di;
2426	uNewES = pNewTSS16->es;
2427	uNewCS = pNewTSS16->cs;
2428	uNewSS = pNewTSS16->ss;
2429	uNewDS = pNewTSS16->ds;
2430	uNewFS = 0;
2431	uNewGS = 0;
2432	uNewLdt = pNewTSS16->selLdt;
2433	fNewDebugTrap = false;
2434	}
2435
2436	if (GCPtrNewTSS == GCPtrCurTSS)
2437	Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
2438	uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
2439
2440	/*
2441	* We're done accessing the new TSS.
2442	*/
2443	rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
2444	if (rcStrict != VINF_SUCCESS)
2445	{
2446	Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
2447	return rcStrict;
2448	}
2449
2450	/*
2451	* Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
2452	*/
2453	if (enmTaskSwitch != IEMTASKSWITCH_IRET)
2454	{
2455	rcStrict = iemMemMap(pVCpu, (void *)&pNewDescTSS, sizeof(pNewDescTSS), UINT8_MAX,
2456	pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2457	if (rcStrict != VINF_SUCCESS)
2458	{
2459	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2460	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2461	return rcStrict;
2462	}
2463
2464	/* Check that the descriptor indicates the new TSS is available (not busy). */
2465	AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2466	\|\| pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
2467	("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
2468
2469	pNewDescTSS->Legacy.Gate.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2470	rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
2471	if (rcStrict != VINF_SUCCESS)
2472	{
2473	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2474	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2475	return rcStrict;
2476	}
2477	}
2478
2479	/*
2480	* From this point on, we're technically in the new task. We will defer exceptions
2481	* until the completion of the task switch but before executing any instructions in the new task.
2482	*/
2483	pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
2484	pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
2485	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
2486	pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
2487	pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
2488	pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
2489	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
2490
2491	/* Set the busy bit in TR. */
2492	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2493
2494	/* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
2495	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2496	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2497	{
2498	uNewEflags \|= X86_EFL_NT;
2499	}
2500
2501	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
2502	pVCpu->cpum.GstCtx.cr0 \|= X86_CR0_TS;
2503	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
2504
2505	pVCpu->cpum.GstCtx.eip = uNewEip;
2506	pVCpu->cpum.GstCtx.eax = uNewEax;
2507	pVCpu->cpum.GstCtx.ecx = uNewEcx;
2508	pVCpu->cpum.GstCtx.edx = uNewEdx;
2509	pVCpu->cpum.GstCtx.ebx = uNewEbx;
2510	pVCpu->cpum.GstCtx.esp = uNewEsp;
2511	pVCpu->cpum.GstCtx.ebp = uNewEbp;
2512	pVCpu->cpum.GstCtx.esi = uNewEsi;
2513	pVCpu->cpum.GstCtx.edi = uNewEdi;
2514
2515	uNewEflags &= X86_EFL_LIVE_MASK;
2516	uNewEflags \|= X86_EFL_RA1_MASK;
2517	IEMMISC_SET_EFL(pVCpu, uNewEflags);
2518
2519	/*
2520	* Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
2521	* will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
2522	* due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
2523	*/
2524	pVCpu->cpum.GstCtx.es.Sel = uNewES;
2525	pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
2526
2527	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2528	pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
2529
2530	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2531	pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
2532
2533	pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
2534	pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
2535
2536	pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
2537	pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
2538
2539	pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
2540	pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
2541	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2542
2543	pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
2544	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
2545	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
2546	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
2547
2548	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2549	{
2550	pVCpu->cpum.GstCtx.es.Attr.u \|= X86DESCATTR_UNUSABLE;
2551	pVCpu->cpum.GstCtx.cs.Attr.u \|= X86DESCATTR_UNUSABLE;
2552	pVCpu->cpum.GstCtx.ss.Attr.u \|= X86DESCATTR_UNUSABLE;
2553	pVCpu->cpum.GstCtx.ds.Attr.u \|= X86DESCATTR_UNUSABLE;
2554	pVCpu->cpum.GstCtx.fs.Attr.u \|= X86DESCATTR_UNUSABLE;
2555	pVCpu->cpum.GstCtx.gs.Attr.u \|= X86DESCATTR_UNUSABLE;
2556	pVCpu->cpum.GstCtx.ldtr.Attr.u \|= X86DESCATTR_UNUSABLE;
2557	}
2558
2559	/*
2560	* Switch CR3 for the new task.
2561	*/
2562	if ( fIsNewTSS386
2563	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
2564	{
2565	/** @todo Should we update and flush TLBs only if CR3 value actually changes? */
2566	int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
2567	AssertRCSuccessReturn(rc, rc);
2568
2569	/* Inform PGM. */
2570	/** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
2571	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
2572	AssertRCReturn(rc, rc);
2573	/* ignore informational status codes */
2574
2575	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
2576	}
2577
2578	/*
2579	* Switch LDTR for the new task.
2580	*/
2581	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
2582	iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
2583	else
2584	{
2585	Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
2586
2587	IEMSELDESC DescNewLdt;
2588	rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
2589	if (rcStrict != VINF_SUCCESS)
2590	{
2591	Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
2592	uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
2593	return rcStrict;
2594	}
2595	if ( !DescNewLdt.Legacy.Gen.u1Present
2596	\|\| DescNewLdt.Legacy.Gen.u1DescType
2597	\|\| DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
2598	{
2599	Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
2600	uNewLdt, DescNewLdt.Legacy.u));
2601	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
2602	}
2603
2604	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
2605	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
2606	pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
2607	pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
2608	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
2609	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2610	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
2611	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
2612	}
2613
2614	IEMSELDESC DescSS;
2615	if (IEM_IS_V86_MODE(pVCpu))
2616	{
2617	pVCpu->iem.s.uCpl = 3;
2618	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
2619	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
2620	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
2621	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
2622	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
2623	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
2624
2625	/* Quick fix: fake DescSS. / /* @todo fix the code further down? */
2626	DescSS.Legacy.u = 0;
2627	DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
2628	DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
2629	DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
2630	DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
2631	DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
2632	DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2633	DescSS.Legacy.Gen.u2Dpl = 3;
2634	}
2635	else
2636	{
2637	uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
2638
2639	/*
2640	* Load the stack segment for the new task.
2641	*/
2642	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
2643	{
2644	Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
2645	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2646	}
2647
2648	/* Fetch the descriptor. */
2649	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
2650	if (rcStrict != VINF_SUCCESS)
2651	{
2652	Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
2653	VBOXSTRICTRC_VAL(rcStrict)));
2654	return rcStrict;
2655	}
2656
2657	/* SS must be a data segment and writable. */
2658	if ( !DescSS.Legacy.Gen.u1DescType
2659	\|\| (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2660	\|\| !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
2661	{
2662	Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
2663	uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
2664	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2665	}
2666
2667	/* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
2668	if ( (uNewSS & X86_SEL_RPL) != uNewCpl
2669	\|\| DescSS.Legacy.Gen.u2Dpl != uNewCpl)
2670	{
2671	Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
2672	uNewCpl));
2673	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2674	}
2675
2676	/* Is it there? */
2677	if (!DescSS.Legacy.Gen.u1Present)
2678	{
2679	Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
2680	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2681	}
2682
2683	uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
2684	uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
2685
2686	/* Set the accessed bit before committing the result into SS. */
2687	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2688	{
2689	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
2690	if (rcStrict != VINF_SUCCESS)
2691	return rcStrict;
2692	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2693	}
2694
2695	/* Commit SS. */
2696	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2697	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
2698	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
2699	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
2700	pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
2701	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2702	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
2703
2704	/* CPL has changed, update IEM before loading rest of segments. */
2705	pVCpu->iem.s.uCpl = uNewCpl;
2706
2707	/*
2708	* Load the data segments for the new task.
2709	*/
2710	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
2711	if (rcStrict != VINF_SUCCESS)
2712	return rcStrict;
2713	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
2714	if (rcStrict != VINF_SUCCESS)
2715	return rcStrict;
2716	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
2717	if (rcStrict != VINF_SUCCESS)
2718	return rcStrict;
2719	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
2720	if (rcStrict != VINF_SUCCESS)
2721	return rcStrict;
2722
2723	/*
2724	* Load the code segment for the new task.
2725	*/
2726	if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
2727	{
2728	Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
2729	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2730	}
2731
2732	/* Fetch the descriptor. */
2733	IEMSELDESC DescCS;
2734	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
2735	if (rcStrict != VINF_SUCCESS)
2736	{
2737	Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
2738	return rcStrict;
2739	}
2740
2741	/* CS must be a code segment. */
2742	if ( !DescCS.Legacy.Gen.u1DescType
2743	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2744	{
2745	Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
2746	DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
2747	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2748	}
2749
2750	/* For conforming CS, DPL must be less than or equal to the RPL. */
2751	if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2752	&& DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
2753	{
2754	Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
2755	DescCS.Legacy.Gen.u2Dpl));
2756	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2757	}
2758
2759	/* For non-conforming CS, DPL must match RPL. */
2760	if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2761	&& DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
2762	{
2763	Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
2764	DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
2765	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2766	}
2767
2768	/* Is it there? */
2769	if (!DescCS.Legacy.Gen.u1Present)
2770	{
2771	Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
2772	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2773	}
2774
2775	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
2776	u64Base = X86DESC_BASE(&DescCS.Legacy);
2777
2778	/* Set the accessed bit before committing the result into CS. */
2779	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2780	{
2781	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
2782	if (rcStrict != VINF_SUCCESS)
2783	return rcStrict;
2784	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2785	}
2786
2787	/* Commit CS. */
2788	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2789	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
2790	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
2791	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
2792	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2793	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2794	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
2795	}
2796
2797	/** @todo Debug trap. */
2798	if (fIsNewTSS386 && fNewDebugTrap)
2799	Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
2800
2801	/*
2802	* Construct the error code masks based on what caused this task switch.
2803	* See Intel Instruction reference for INT.
2804	*/
2805	uint16_t uExt;
2806	if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
2807	&& ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2808	\|\| (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
2809	{
2810	uExt = 1;
2811	}
2812	else
2813	uExt = 0;
2814
2815	/*
2816	* Push any error code on to the new stack.
2817	*/
2818	if (fFlags & IEM_XCPT_FLAGS_ERR)
2819	{
2820	Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
2821	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
2822	uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
2823
2824	/* Check that there is sufficient space on the stack. */
2825	/** @todo Factor out segment limit checking for normal/expand down segments
2826	* into a separate function. */
2827	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
2828	{
2829	if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
2830	\|\| pVCpu->cpum.GstCtx.esp < cbStackFrame)
2831	{
2832	/** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2833	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
2834	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2835	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2836	}
2837	}
2838	else
2839	{
2840	if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2841	\|\| pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
2842	{
2843	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
2844	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2845	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2846	}
2847	}
2848
2849
2850	if (fIsNewTSS386)
2851	rcStrict = iemMemStackPushU32(pVCpu, uErr);
2852	else
2853	rcStrict = iemMemStackPushU16(pVCpu, uErr);
2854	if (rcStrict != VINF_SUCCESS)
2855	{
2856	Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
2857	fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
2858	return rcStrict;
2859	}
2860	}
2861
2862	/* Check the new EIP against the new CS limit. */
2863	if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
2864	{
2865	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
2866	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
2867	/** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2868	return iemRaiseGeneralProtectionFault(pVCpu, uExt);
2869	}
2870
2871	Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
2872	pVCpu->cpum.GstCtx.ss.Sel));
2873	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2874	}
2875
2876
2877	/**
2878	* Implements exceptions and interrupts for protected mode.
2879	*
2880	* @returns VBox strict status code.
2881	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2882	* @param cbInstr The number of bytes to offset rIP by in the return
2883	* address.
2884	* @param u8Vector The interrupt / exception vector number.
2885	* @param fFlags The flags.
2886	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2887	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2888	*/
2889	static VBOXSTRICTRC
2890	iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
2891	uint8_t cbInstr,
2892	uint8_t u8Vector,
2893	uint32_t fFlags,
2894	uint16_t uErr,
2895	uint64_t uCr2) RT_NOEXCEPT
2896	{
2897	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2898
2899	/*
2900	* Read the IDT entry.
2901	*/
2902	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
2903	{
2904	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
2905	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2906	}
2907	X86DESC Idte;
2908	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
2909	pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
2910	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2911	{
2912	Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
2913	return rcStrict;
2914	}
2915	Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
2916	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
2917	Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
2918
2919	/*
2920	* Check the descriptor type, DPL and such.
2921	* ASSUMES this is done in the same order as described for call-gate calls.
2922	*/
2923	if (Idte.Gate.u1DescType)
2924	{
2925	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2926	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2927	}
2928	bool fTaskGate = false;
2929	uint8_t f32BitGate = true;
2930	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
2931	switch (Idte.Gate.u4Type)
2932	{
2933	case X86_SEL_TYPE_SYS_UNDEFINED:
2934	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
2935	case X86_SEL_TYPE_SYS_LDT:
2936	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
2937	case X86_SEL_TYPE_SYS_286_CALL_GATE:
2938	case X86_SEL_TYPE_SYS_UNDEFINED2:
2939	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
2940	case X86_SEL_TYPE_SYS_UNDEFINED3:
2941	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
2942	case X86_SEL_TYPE_SYS_386_CALL_GATE:
2943	case X86_SEL_TYPE_SYS_UNDEFINED4:
2944	{
2945	/** @todo check what actually happens when the type is wrong...
2946	* esp. call gates. */
2947	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2948	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2949	}
2950
2951	case X86_SEL_TYPE_SYS_286_INT_GATE:
2952	f32BitGate = false;
2953	RT_FALL_THRU();
2954	case X86_SEL_TYPE_SYS_386_INT_GATE:
2955	fEflToClear \|= X86_EFL_IF;
2956	break;
2957
2958	case X86_SEL_TYPE_SYS_TASK_GATE:
2959	fTaskGate = true;
2960	#ifndef IEM_IMPLEMENTS_TASKSWITCH
2961	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
2962	#endif
2963	break;
2964
2965	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
2966	f32BitGate = false;
2967	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
2968	break;
2969
2970	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2971	}
2972
2973	/* Check DPL against CPL if applicable. */
2974	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
2975	{
2976	if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
2977	{
2978	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
2979	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2980	}
2981	}
2982
2983	/* Is it there? */
2984	if (!Idte.Gate.u1Present)
2985	{
2986	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
2987	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2988	}
2989
2990	/* Is it a task-gate? */
2991	if (fTaskGate)
2992	{
2993	/*
2994	* Construct the error code masks based on what caused this task switch.
2995	* See Intel Instruction reference for INT.
2996	*/
2997	uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2998	&& !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
2999	uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
3000	RTSEL SelTSS = Idte.Gate.u16Sel;
3001
3002	/*
3003	* Fetch the TSS descriptor in the GDT.
3004	*/
3005	IEMSELDESC DescTSS;
3006	rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) \| uExt);
3007	if (rcStrict != VINF_SUCCESS)
3008	{
3009	Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
3010	VBOXSTRICTRC_VAL(rcStrict)));
3011	return rcStrict;
3012	}
3013
3014	/* The TSS descriptor must be a system segment and be available (not busy). */
3015	if ( DescTSS.Legacy.Gen.u1DescType
3016	\|\| ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
3017	&& DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
3018	{
3019	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
3020	u8Vector, SelTSS, DescTSS.Legacy.au64));
3021	return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) \| uExt);
3022	}
3023
3024	/* The TSS must be present. */
3025	if (!DescTSS.Legacy.Gen.u1Present)
3026	{
3027	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
3028	return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) \| uExt);
3029	}
3030
3031	/* Do the actual task switch. */
3032	return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
3033	(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
3034	fFlags, uErr, uCr2, SelTSS, &DescTSS);
3035	}
3036
3037	/* A null CS is bad. */
3038	RTSEL NewCS = Idte.Gate.u16Sel;
3039	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3040	{
3041	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3042	return iemRaiseGeneralProtectionFault0(pVCpu);
3043	}
3044
3045	/* Fetch the descriptor for the new CS. */
3046	IEMSELDESC DescCS;
3047	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
3048	if (rcStrict != VINF_SUCCESS)
3049	{
3050	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3051	return rcStrict;
3052	}
3053
3054	/* Must be a code segment. */
3055	if (!DescCS.Legacy.Gen.u1DescType)
3056	{
3057	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3058	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3059	}
3060	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3061	{
3062	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3063	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3064	}
3065
3066	/* Don't allow lowering the privilege level. */
3067	/** @todo Does the lowering of privileges apply to software interrupts
3068	* only? This has bearings on the more-privileged or
3069	* same-privilege stack behavior further down. A testcase would
3070	* be nice. */
3071	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3072	{
3073	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3074	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3075	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3076	}
3077
3078	/* Make sure the selector is present. */
3079	if (!DescCS.Legacy.Gen.u1Present)
3080	{
3081	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3082	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3083	}
3084
3085	/* Check the new EIP against the new CS limit. */
3086	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
3087	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
3088	? Idte.Gate.u16OffsetLow
3089	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
3090	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3091	if (uNewEip > cbLimitCS)
3092	{
3093	Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
3094	u8Vector, uNewEip, cbLimitCS, NewCS));
3095	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3096	}
3097	Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
3098
3099	/* Calc the flag image to push. */
3100	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3101	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3102	fEfl &= ~X86_EFL_RF;
3103	else
3104	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3105
3106	/* From V8086 mode only go to CPL 0. */
3107	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3108	? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3109	if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
3110	{
3111	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
3112	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3113	}
3114
3115	/*
3116	* If the privilege level changes, we need to get a new stack from the TSS.
3117	* This in turns means validating the new SS and ESP...
3118	*/
3119	if (uNewCpl != pVCpu->iem.s.uCpl)
3120	{
3121	RTSEL NewSS;
3122	uint32_t uNewEsp;
3123	rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
3124	if (rcStrict != VINF_SUCCESS)
3125	return rcStrict;
3126
3127	IEMSELDESC DescSS;
3128	rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
3129	if (rcStrict != VINF_SUCCESS)
3130	return rcStrict;
3131	/* If the new SS is 16-bit, we are only going to use SP, not ESP. */
3132	if (!DescSS.Legacy.Gen.u1DefBig)
3133	{
3134	Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
3135	uNewEsp = (uint16_t)uNewEsp;
3136	}
3137
3138	Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3139
3140	/* Check that there is sufficient space for the stack frame. */
3141	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3142	uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
3143	? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
3144	: (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
3145
3146	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
3147	{
3148	if ( uNewEsp - 1 > cbLimitSS
3149	\|\| uNewEsp < cbStackFrame)
3150	{
3151	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
3152	u8Vector, NewSS, uNewEsp, cbStackFrame));
3153	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3154	}
3155	}
3156	else
3157	{
3158	if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
3159	\|\| uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
3160	{
3161	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
3162	u8Vector, NewSS, uNewEsp, cbStackFrame));
3163	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3164	}
3165	}
3166
3167	/*
3168	* Start making changes.
3169	*/
3170
3171	/* Set the new CPL so that stack accesses use it. */
3172	uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3173	pVCpu->iem.s.uCpl = uNewCpl;
3174
3175	/* Create the stack frame. */
3176	RTPTRUNION uStackFrame;
3177	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3178	uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy),
3179	IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3180	if (rcStrict != VINF_SUCCESS)
3181	return rcStrict;
3182	void * const pvStackFrame = uStackFrame.pv;
3183	if (f32BitGate)
3184	{
3185	if (fFlags & IEM_XCPT_FLAGS_ERR)
3186	*uStackFrame.pu32++ = uErr;
3187	uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3188	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3189	uStackFrame.pu32[2] = fEfl;
3190	uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
3191	uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
3192	Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3193	if (fEfl & X86_EFL_VM)
3194	{
3195	uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
3196	uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
3197	uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
3198	uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
3199	uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
3200	}
3201	}
3202	else
3203	{
3204	if (fFlags & IEM_XCPT_FLAGS_ERR)
3205	*uStackFrame.pu16++ = uErr;
3206	uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3207	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3208	uStackFrame.pu16[2] = fEfl;
3209	uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
3210	uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
3211	Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
3212	if (fEfl & X86_EFL_VM)
3213	{
3214	uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
3215	uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
3216	uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
3217	uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
3218	uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
3219	}
3220	}
3221	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3222	if (rcStrict != VINF_SUCCESS)
3223	return rcStrict;
3224
3225	/* Mark the selectors 'accessed' (hope this is the correct time). */
3226	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3227	* after pushing the stack frame? (Write protect the gdt + stack to
3228	* find out.) */
3229	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3230	{
3231	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3232	if (rcStrict != VINF_SUCCESS)
3233	return rcStrict;
3234	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3235	}
3236
3237	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3238	{
3239	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
3240	if (rcStrict != VINF_SUCCESS)
3241	return rcStrict;
3242	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3243	}
3244
3245	/*
3246	* Start comitting the register changes (joins with the DPL=CPL branch).
3247	*/
3248	pVCpu->cpum.GstCtx.ss.Sel = NewSS;
3249	pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
3250	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3251	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
3252	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3253	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3254	/** @todo When coming from 32-bit code and operating with a 16-bit TSS and
3255	* 16-bit handler, the high word of ESP remains unchanged (i.e. only
3256	* SP is loaded).
3257	* Need to check the other combinations too:
3258	* - 16-bit TSS, 32-bit handler
3259	* - 32-bit TSS, 16-bit handler */
3260	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3261	pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
3262	else
3263	pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
3264
3265	if (fEfl & X86_EFL_VM)
3266	{
3267	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
3268	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
3269	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
3270	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
3271	}
3272	}
3273	/*
3274	* Same privilege, no stack change and smaller stack frame.
3275	*/
3276	else
3277	{
3278	uint64_t uNewRsp;
3279	RTPTRUNION uStackFrame;
3280	uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
3281	rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1, &uStackFrame.pv, &uNewRsp);
3282	if (rcStrict != VINF_SUCCESS)
3283	return rcStrict;
3284	void * const pvStackFrame = uStackFrame.pv;
3285
3286	if (f32BitGate)
3287	{
3288	if (fFlags & IEM_XCPT_FLAGS_ERR)
3289	*uStackFrame.pu32++ = uErr;
3290	uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3291	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| pVCpu->iem.s.uCpl;
3292	uStackFrame.pu32[2] = fEfl;
3293	}
3294	else
3295	{
3296	if (fFlags & IEM_XCPT_FLAGS_ERR)
3297	*uStackFrame.pu16++ = uErr;
3298	uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3299	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| pVCpu->iem.s.uCpl;
3300	uStackFrame.pu16[2] = fEfl;
3301	}
3302	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
3303	if (rcStrict != VINF_SUCCESS)
3304	return rcStrict;
3305
3306	/* Mark the CS selector as 'accessed'. */
3307	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3308	{
3309	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3310	if (rcStrict != VINF_SUCCESS)
3311	return rcStrict;
3312	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3313	}
3314
3315	/*
3316	* Start committing the register changes (joins with the other branch).
3317	*/
3318	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3319	}
3320
3321	/* ... register committing continues. */
3322	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3323	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3324	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3325	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3326	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3327	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3328
3329	pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
3330	fEfl &= ~fEflToClear;
3331	IEMMISC_SET_EFL(pVCpu, fEfl);
3332
3333	if (fFlags & IEM_XCPT_FLAGS_CR2)
3334	pVCpu->cpum.GstCtx.cr2 = uCr2;
3335
3336	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3337	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3338
3339	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3340	}
3341
3342
3343	/**
3344	* Implements exceptions and interrupts for long mode.
3345	*
3346	* @returns VBox strict status code.
3347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3348	* @param cbInstr The number of bytes to offset rIP by in the return
3349	* address.
3350	* @param u8Vector The interrupt / exception vector number.
3351	* @param fFlags The flags.
3352	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3353	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3354	*/
3355	static VBOXSTRICTRC
3356	iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
3357	uint8_t cbInstr,
3358	uint8_t u8Vector,
3359	uint32_t fFlags,
3360	uint16_t uErr,
3361	uint64_t uCr2) RT_NOEXCEPT
3362	{
3363	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3364
3365	/*
3366	* Read the IDT entry.
3367	*/
3368	uint16_t offIdt = (uint16_t)u8Vector << 4;
3369	if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
3370	{
3371	Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3372	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3373	}
3374	X86DESC64 Idte;
3375	#ifdef _MSC_VER /* Shut up silly compiler warning. */
3376	Idte.au64[0] = 0;
3377	Idte.au64[1] = 0;
3378	#endif
3379	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
3380	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3381	rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
3382	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3383	{
3384	Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3385	return rcStrict;
3386	}
3387	Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
3388	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3389	Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3390
3391	/*
3392	* Check the descriptor type, DPL and such.
3393	* ASSUMES this is done in the same order as described for call-gate calls.
3394	*/
3395	if (Idte.Gate.u1DescType)
3396	{
3397	Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3398	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3399	}
3400	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3401	switch (Idte.Gate.u4Type)
3402	{
3403	case AMD64_SEL_TYPE_SYS_INT_GATE:
3404	fEflToClear \|= X86_EFL_IF;
3405	break;
3406	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
3407	break;
3408
3409	default:
3410	Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3411	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3412	}
3413
3414	/* Check DPL against CPL if applicable. */
3415	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
3416	{
3417	if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
3418	{
3419	Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
3420	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3421	}
3422	}
3423
3424	/* Is it there? */
3425	if (!Idte.Gate.u1Present)
3426	{
3427	Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
3428	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3429	}
3430
3431	/* A null CS is bad. */
3432	RTSEL NewCS = Idte.Gate.u16Sel;
3433	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3434	{
3435	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3436	return iemRaiseGeneralProtectionFault0(pVCpu);
3437	}
3438
3439	/* Fetch the descriptor for the new CS. */
3440	IEMSELDESC DescCS;
3441	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
3442	if (rcStrict != VINF_SUCCESS)
3443	{
3444	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3445	return rcStrict;
3446	}
3447
3448	/* Must be a 64-bit code segment. */
3449	if (!DescCS.Long.Gen.u1DescType)
3450	{
3451	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3452	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3453	}
3454	if ( !DescCS.Long.Gen.u1Long
3455	\|\| DescCS.Long.Gen.u1DefBig
3456	\|\| !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
3457	{
3458	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
3459	u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
3460	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3461	}
3462
3463	/* Don't allow lowering the privilege level. For non-conforming CS
3464	selectors, the CS.DPL sets the privilege level the trap/interrupt
3465	handler runs at. For conforming CS selectors, the CPL remains
3466	unchanged, but the CS.DPL must be <= CPL. */
3467	/** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
3468	* when CPU in Ring-0. Result \#GP? */
3469	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3470	{
3471	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3472	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3473	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3474	}
3475
3476
3477	/* Make sure the selector is present. */
3478	if (!DescCS.Legacy.Gen.u1Present)
3479	{
3480	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3481	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3482	}
3483
3484	/* Check that the new RIP is canonical. */
3485	uint64_t const uNewRip = Idte.Gate.u16OffsetLow
3486	\| ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
3487	\| ((uint64_t)Idte.Gate.u32OffsetTop << 32);
3488	if (!IEM_IS_CANONICAL(uNewRip))
3489	{
3490	Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
3491	return iemRaiseGeneralProtectionFault0(pVCpu);
3492	}
3493
3494	/*
3495	* If the privilege level changes or if the IST isn't zero, we need to get
3496	* a new stack from the TSS.
3497	*/
3498	uint64_t uNewRsp;
3499	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3500	? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3501	if ( uNewCpl != pVCpu->iem.s.uCpl
3502	\|\| Idte.Gate.u3IST != 0)
3503	{
3504	rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
3505	if (rcStrict != VINF_SUCCESS)
3506	return rcStrict;
3507	}
3508	else
3509	uNewRsp = pVCpu->cpum.GstCtx.rsp;
3510	uNewRsp &= ~(uint64_t)0xf;
3511
3512	/*
3513	* Calc the flag image to push.
3514	*/
3515	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3516	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3517	fEfl &= ~X86_EFL_RF;
3518	else
3519	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3520
3521	/*
3522	* Start making changes.
3523	*/
3524	/* Set the new CPL so that stack accesses use it. */
3525	uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3526	pVCpu->iem.s.uCpl = uNewCpl;
3527
3528	/* Create the stack frame. */
3529	uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
3530	RTPTRUNION uStackFrame;
3531	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3532	uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3533	if (rcStrict != VINF_SUCCESS)
3534	return rcStrict;
3535	void * const pvStackFrame = uStackFrame.pv;
3536
3537	if (fFlags & IEM_XCPT_FLAGS_ERR)
3538	*uStackFrame.pu64++ = uErr;
3539	uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
3540	uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl; /* CPL paranoia */
3541	uStackFrame.pu64[2] = fEfl;
3542	uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
3543	uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
3544	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3545	if (rcStrict != VINF_SUCCESS)
3546	return rcStrict;
3547
3548	/* Mark the CS selectors 'accessed' (hope this is the correct time). */
3549	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3550	* after pushing the stack frame? (Write protect the gdt + stack to
3551	* find out.) */
3552	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3553	{
3554	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3555	if (rcStrict != VINF_SUCCESS)
3556	return rcStrict;
3557	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3558	}
3559
3560	/*
3561	* Start comitting the register changes.
3562	*/
3563	/** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
3564	* hidden registers when interrupting 32-bit or 16-bit code! */
3565	if (uNewCpl != uOldCpl)
3566	{
3567	pVCpu->cpum.GstCtx.ss.Sel = 0 \| uNewCpl;
3568	pVCpu->cpum.GstCtx.ss.ValidSel = 0 \| uNewCpl;
3569	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3570	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3571	pVCpu->cpum.GstCtx.ss.u64Base = 0;
3572	pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) \| X86DESCATTR_UNUSABLE;
3573	}
3574	pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
3575	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3576	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3577	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3578	pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
3579	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3580	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3581	pVCpu->cpum.GstCtx.rip = uNewRip;
3582
3583	fEfl &= ~fEflToClear;
3584	IEMMISC_SET_EFL(pVCpu, fEfl);
3585
3586	if (fFlags & IEM_XCPT_FLAGS_CR2)
3587	pVCpu->cpum.GstCtx.cr2 = uCr2;
3588
3589	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3590	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3591
3592	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3593	}
3594
3595
3596	/**
3597	* Implements exceptions and interrupts.
3598	*
3599	* All exceptions and interrupts goes thru this function!
3600	*
3601	* @returns VBox strict status code.
3602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3603	* @param cbInstr The number of bytes to offset rIP by in the return
3604	* address.
3605	* @param u8Vector The interrupt / exception vector number.
3606	* @param fFlags The flags.
3607	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3608	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3609	*/
3610	VBOXSTRICTRC
3611	iemRaiseXcptOrInt(PVMCPUCC pVCpu,
3612	uint8_t cbInstr,
3613	uint8_t u8Vector,
3614	uint32_t fFlags,
3615	uint16_t uErr,
3616	uint64_t uCr2) RT_NOEXCEPT
3617	{
3618	/*
3619	* Get all the state that we might need here.
3620	*/
3621	IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3622	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3623
3624	#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
3625	/*
3626	* Flush prefetch buffer
3627	*/
3628	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3629	#endif
3630
3631	/*
3632	* Perform the V8086 IOPL check and upgrade the fault without nesting.
3633	*/
3634	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
3635	&& pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
3636	&& (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
3637	\| IEM_XCPT_FLAGS_BP_INSTR
3638	\| IEM_XCPT_FLAGS_ICEBP_INSTR
3639	\| IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
3640	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
3641	{
3642	Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
3643	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3644	u8Vector = X86_XCPT_GP;
3645	uErr = 0;
3646	}
3647	#ifdef DBGFTRACE_ENABLED
3648	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
3649	pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
3650	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
3651	#endif
3652
3653	/*
3654	* Evaluate whether NMI blocking should be in effect.
3655	* Normally, NMI blocking is in effect whenever we inject an NMI.
3656	*/
3657	bool fBlockNmi;
3658	if ( u8Vector == X86_XCPT_NMI
3659	&& (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3660	fBlockNmi = true;
3661	else
3662	fBlockNmi = false;
3663
3664	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3665	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3666	{
3667	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
3668	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3669	return rcStrict0;
3670
3671	/* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
3672	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
3673	{
3674	Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
3675	fBlockNmi = false;
3676	}
3677	}
3678	#endif
3679
3680	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3681	if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
3682	{
3683	/*
3684	* If the event is being injected as part of VMRUN, it isn't subject to event
3685	* intercepts in the nested-guest. However, secondary exceptions that occur
3686	* during injection of any event -are- subject to exception intercepts.
3687	*
3688	* See AMD spec. 15.20 "Event Injection".
3689	*/
3690	if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
3691	pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
3692	else
3693	{
3694	/*
3695	* Check and handle if the event being raised is intercepted.
3696	*/
3697	VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
3698	if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
3699	return rcStrict0;
3700	}
3701	}
3702	#endif
3703
3704	/*
3705	* Set NMI blocking if necessary.
3706	*/
3707	if ( fBlockNmi
3708	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3709	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3710
3711	/*
3712	* Do recursion accounting.
3713	*/
3714	uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
3715	uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
3716	if (pVCpu->iem.s.cXcptRecursions == 0)
3717	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
3718	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
3719	else
3720	{
3721	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
3722	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
3723	pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
3724
3725	if (pVCpu->iem.s.cXcptRecursions >= 4)
3726	{
3727	#ifdef DEBUG_bird
3728	AssertFailed();
3729	#endif
3730	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
3731	}
3732
3733	/*
3734	* Evaluate the sequence of recurring events.
3735	*/
3736	IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
3737	NULL /* pXcptRaiseInfo */);
3738	if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
3739	{ /* likely */ }
3740	else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
3741	{
3742	Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
3743	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3744	u8Vector = X86_XCPT_DF;
3745	uErr = 0;
3746	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3747	/* VMX nested-guest #DF intercept needs to be checked here. */
3748	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3749	{
3750	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
3751	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3752	return rcStrict0;
3753	}
3754	#endif
3755	/* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
3756	if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
3757	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
3758	}
3759	else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
3760	{
3761	Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
3762	return iemInitiateCpuShutdown(pVCpu);
3763	}
3764	else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
3765	{
3766	/* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
3767	Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
3768	if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
3769	&& !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
3770	return VERR_EM_GUEST_CPU_HANG;
3771	}
3772	else
3773	{
3774	AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
3775	enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
3776	return VERR_IEM_IPE_9;
3777	}
3778
3779	/*
3780	* The 'EXT' bit is set when an exception occurs during deliver of an external
3781	* event (such as an interrupt or earlier exception)[1]. Privileged software
3782	* exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
3783	* interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
3784	*
3785	* [1] - Intel spec. 6.13 "Error Code"
3786	* [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
3787	* [3] - Intel Instruction reference for INT n.
3788	*/
3789	if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_T_EXT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR))
3790	&& (fFlags & IEM_XCPT_FLAGS_ERR)
3791	&& u8Vector != X86_XCPT_PF
3792	&& u8Vector != X86_XCPT_DF)
3793	{
3794	uErr \|= X86_TRAP_ERR_EXTERNAL;
3795	}
3796	}
3797
3798	pVCpu->iem.s.cXcptRecursions++;
3799	pVCpu->iem.s.uCurXcpt = u8Vector;
3800	pVCpu->iem.s.fCurXcpt = fFlags;
3801	pVCpu->iem.s.uCurXcptErr = uErr;
3802	pVCpu->iem.s.uCurXcptCr2 = uCr2;
3803
3804	/*
3805	* Extensive logging.
3806	*/
3807	#if defined(LOG_ENABLED) && defined(IN_RING3)
3808	if (LogIs3Enabled())
3809	{
3810	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
3811	PVM pVM = pVCpu->CTX_SUFF(pVM);
3812	char szRegs[4096];
3813	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
3814	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
3815	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
3816	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
3817	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
3818	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
3819	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
3820	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
3821	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
3822	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
3823	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
3824	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
3825	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
3826	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
3827	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
3828	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
3829	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
3830	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
3831	" efer=%016VR{efer}\n"
3832	" pat=%016VR{pat}\n"
3833	" sf_mask=%016VR{sf_mask}\n"
3834	"krnl_gs_base=%016VR{krnl_gs_base}\n"
3835	" lstar=%016VR{lstar}\n"
3836	" star=%016VR{star} cstar=%016VR{cstar}\n"
3837	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
3838	);
3839
3840	char szInstr[256];
3841	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
3842	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
3843	szInstr, sizeof(szInstr), NULL);
3844	Log3(("%s%s\n", szRegs, szInstr));
3845	}
3846	#endif /* LOG_ENABLED */
3847
3848	/*
3849	* Stats.
3850	*/
3851	if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3852	STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; });
3853	else if (u8Vector <= X86_XCPT_LAST)
3854	{
3855	STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]);
3856	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector),
3857	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, ASMReadTSC());
3858	}
3859
3860	/*
3861	* #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3)
3862	* to ensure that a stale TLB or paging cache entry will only cause one
3863	* spurious #PF.
3864	*/
3865	if ( u8Vector == X86_XCPT_PF
3866	&& (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2))
3867	IEMTlbInvalidatePage(pVCpu, uCr2);
3868
3869	/*
3870	* Call the mode specific worker function.
3871	*/
3872	VBOXSTRICTRC rcStrict;
3873	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
3874	rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3875	else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
3876	rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3877	else
3878	rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3879
3880	/* Flush the prefetch buffer. */
3881	#ifdef IEM_WITH_CODE_TLB
3882	pVCpu->iem.s.pbInstrBuf = NULL;
3883	#else
3884	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
3885	#endif
3886
3887	/*
3888	* Unwind.
3889	*/
3890	pVCpu->iem.s.cXcptRecursions--;
3891	pVCpu->iem.s.uCurXcpt = uPrevXcpt;
3892	pVCpu->iem.s.fCurXcpt = fPrevXcpt;
3893	Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
3894	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
3895	pVCpu->iem.s.cXcptRecursions + 1));
3896	return rcStrict;
3897	}
3898
3899	#ifdef IEM_WITH_SETJMP
3900	/**
3901	* See iemRaiseXcptOrInt. Will not return.
3902	*/
3903	DECL_NO_RETURN(void)
3904	iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
3905	uint8_t cbInstr,
3906	uint8_t u8Vector,
3907	uint32_t fFlags,
3908	uint16_t uErr,
3909	uint64_t uCr2) RT_NOEXCEPT
3910	{
3911	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3912	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3913	}
3914	#endif
3915
3916
3917	/** \#DE - 00. */
3918	VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT
3919	{
3920	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3921	}
3922
3923
3924	/** \#DB - 01.
3925	* @note This automatically clear DR7.GD. */
3926	VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT
3927	{
3928	/** @todo set/clear RF. */
3929	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3930	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3931	}
3932
3933
3934	/** \#BR - 05. */
3935	VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT
3936	{
3937	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3938	}
3939
3940
3941	/** \#UD - 06. */
3942	VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT
3943	{
3944	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3945	}
3946
3947
3948	/** \#NM - 07. */
3949	VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT
3950	{
3951	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3952	}
3953
3954
3955	/** \#TS(err) - 0a. */
3956	VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3957	{
3958	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
3959	}
3960
3961
3962	/** \#TS(tr) - 0a. */
3963	VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT
3964	{
3965	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3966	pVCpu->cpum.GstCtx.tr.Sel, 0);
3967	}
3968
3969
3970	/** \#TS(0) - 0a. */
3971	VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
3972	{
3973	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3974	0, 0);
3975	}
3976
3977
3978	/** \#TS(err) - 0a. */
3979	VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3980	{
3981	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3982	uSel & X86_SEL_MASK_OFF_RPL, 0);
3983	}
3984
3985
3986	/** \#NP(err) - 0b. */
3987	VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3988	{
3989	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
3990	}
3991
3992
3993	/** \#NP(sel) - 0b. */
3994	VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3995	{
3996	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3997	uSel & ~X86_SEL_RPL, 0);
3998	}
3999
4000
4001	/** \#SS(seg) - 0c. */
4002	VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4003	{
4004	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4005	uSel & ~X86_SEL_RPL, 0);
4006	}
4007
4008
4009	/** \#SS(err) - 0c. */
4010	VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4011	{
4012	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4013	}
4014
4015
4016	/** \#GP(n) - 0d. */
4017	VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4018	{
4019	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4020	}
4021
4022
4023	/** \#GP(0) - 0d. */
4024	VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
4025	{
4026	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4027	}
4028
4029	#ifdef IEM_WITH_SETJMP
4030	/** \#GP(0) - 0d. */
4031	DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4032	{
4033	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4034	}
4035	#endif
4036
4037
4038	/** \#GP(sel) - 0d. */
4039	VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4040	{
4041	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4042	Sel & ~X86_SEL_RPL, 0);
4043	}
4044
4045
4046	/** \#GP(0) - 0d. */
4047	VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT
4048	{
4049	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4050	}
4051
4052
4053	/** \#GP(sel) - 0d. */
4054	VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4055	{
4056	NOREF(iSegReg); NOREF(fAccess);
4057	return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4058	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4059	}
4060
4061	#ifdef IEM_WITH_SETJMP
4062	/** \#GP(sel) - 0d, longjmp. */
4063	DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4064	{
4065	NOREF(iSegReg); NOREF(fAccess);
4066	iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4067	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4068	}
4069	#endif
4070
4071	/** \#GP(sel) - 0d. */
4072	VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4073	{
4074	NOREF(Sel);
4075	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4076	}
4077
4078	#ifdef IEM_WITH_SETJMP
4079	/** \#GP(sel) - 0d, longjmp. */
4080	DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4081	{
4082	NOREF(Sel);
4083	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4084	}
4085	#endif
4086
4087
4088	/** \#GP(sel) - 0d. */
4089	VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4090	{
4091	NOREF(iSegReg); NOREF(fAccess);
4092	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4093	}
4094
4095	#ifdef IEM_WITH_SETJMP
4096	/** \#GP(sel) - 0d, longjmp. */
4097	DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4098	{
4099	NOREF(iSegReg); NOREF(fAccess);
4100	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4101	}
4102	#endif
4103
4104
4105	/** \#PF(n) - 0e. */
4106	VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4107	{
4108	uint16_t uErr;
4109	switch (rc)
4110	{
4111	case VERR_PAGE_NOT_PRESENT:
4112	case VERR_PAGE_TABLE_NOT_PRESENT:
4113	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
4114	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
4115	uErr = 0;
4116	break;
4117
4118	default:
4119	AssertMsgFailed(("%Rrc\n", rc));
4120	RT_FALL_THRU();
4121	case VERR_ACCESS_DENIED:
4122	uErr = X86_TRAP_PF_P;
4123	break;
4124
4125	/** @todo reserved */
4126	}
4127
4128	if (pVCpu->iem.s.uCpl == 3)
4129	uErr \|= X86_TRAP_PF_US;
4130
4131	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
4132	&& ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
4133	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
4134	uErr \|= X86_TRAP_PF_ID;
4135
4136	#if 0 /* This is so much non-sense, really. Why was it done like that? */
4137	/* Note! RW access callers reporting a WRITE protection fault, will clear
4138	the READ flag before calling. So, read-modify-write accesses (RW)
4139	can safely be reported as READ faults. */
4140	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
4141	uErr \|= X86_TRAP_PF_RW;
4142	#else
4143	if (fAccess & IEM_ACCESS_TYPE_WRITE)
4144	{
4145	/// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
4146	/// (regardless of outcome of the comparison in the latter case).
4147	//if (!(fAccess & IEM_ACCESS_TYPE_READ))
4148	uErr \|= X86_TRAP_PF_RW;
4149	}
4150	#endif
4151
4152	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
4153	uErr, GCPtrWhere);
4154	}
4155
4156	#ifdef IEM_WITH_SETJMP
4157	/** \#PF(n) - 0e, longjmp. */
4158	DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4159	{
4160	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
4161	}
4162	#endif
4163
4164
4165	/** \#MF(0) - 10. */
4166	VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu)
4167	{
4168	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4169	}
4170
4171
4172	/** \#AC(0) - 11. */
4173	VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
4174	{
4175	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4176	}
4177
4178	#ifdef IEM_WITH_SETJMP
4179	/** \#AC(0) - 11, longjmp. */
4180	DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4181	{
4182	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu)));
4183	}
4184	#endif
4185
4186
4187	/** \#XF(0)/\#XM(0) - 19. */
4188	VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT
4189	{
4190	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4191	}
4192
4193
4194	/** Accessed via IEMOP_RAISE_DIVIDE_ERROR. */
4195	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
4196	{
4197	NOREF(cbInstr);
4198	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4199	}
4200
4201
4202	/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */
4203	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
4204	{
4205	NOREF(cbInstr);
4206	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4207	}
4208
4209
4210	/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */
4211	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
4212	{
4213	NOREF(cbInstr);
4214	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4215	}
4216
4217
4218	/** @} */
4219
4220	/** @name Common opcode decoders.
4221	* @{
4222	*/
4223	//#include <iprt/mem.h>
4224
4225	/**
4226	* Used to add extra details about a stub case.
4227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4228	*/
4229	void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT
4230	{
4231	#if defined(LOG_ENABLED) && defined(IN_RING3)
4232	PVM pVM = pVCpu->CTX_SUFF(pVM);
4233	char szRegs[4096];
4234	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4235	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4236	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4237	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4238	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4239	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4240	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4241	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4242	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4243	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4244	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4245	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4246	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4247	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4248	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4249	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4250	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4251	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4252	" efer=%016VR{efer}\n"
4253	" pat=%016VR{pat}\n"
4254	" sf_mask=%016VR{sf_mask}\n"
4255	"krnl_gs_base=%016VR{krnl_gs_base}\n"
4256	" lstar=%016VR{lstar}\n"
4257	" star=%016VR{star} cstar=%016VR{cstar}\n"
4258	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4259	);
4260
4261	char szInstr[256];
4262	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4263	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
4264	szInstr, sizeof(szInstr), NULL);
4265
4266	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
4267	#else
4268	RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
4269	#endif
4270	}
4271
4272	/** @} */
4273
4274
4275
4276	/** @name Register Access.
4277	* @{
4278	*/
4279
4280	/**
4281	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
4282	*
4283	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4284	* segment limit.
4285	*
4286	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4287	* @param offNextInstr The offset of the next instruction.
4288	*/
4289	VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT
4290	{
4291	switch (pVCpu->iem.s.enmEffOpSize)
4292	{
4293	case IEMMODE_16BIT:
4294	{
4295	uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4296	if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4297	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4298	return iemRaiseGeneralProtectionFault0(pVCpu);
4299	pVCpu->cpum.GstCtx.rip = uNewIp;
4300	break;
4301	}
4302
4303	case IEMMODE_32BIT:
4304	{
4305	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4306	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4307
4308	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4309	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4310	return iemRaiseGeneralProtectionFault0(pVCpu);
4311	pVCpu->cpum.GstCtx.rip = uNewEip;
4312	break;
4313	}
4314
4315	case IEMMODE_64BIT:
4316	{
4317	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4318
4319	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4320	if (!IEM_IS_CANONICAL(uNewRip))
4321	return iemRaiseGeneralProtectionFault0(pVCpu);
4322	pVCpu->cpum.GstCtx.rip = uNewRip;
4323	break;
4324	}
4325
4326	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4327	}
4328
4329	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4330
4331	#ifndef IEM_WITH_CODE_TLB
4332	/* Flush the prefetch buffer. */
4333	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4334	#endif
4335
4336	return VINF_SUCCESS;
4337	}
4338
4339
4340	/**
4341	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
4342	*
4343	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4344	* segment limit.
4345	*
4346	* @returns Strict VBox status code.
4347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4348	* @param offNextInstr The offset of the next instruction.
4349	*/
4350	VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT
4351	{
4352	Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
4353
4354	uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4355	if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4356	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4357	return iemRaiseGeneralProtectionFault0(pVCpu);
4358	/** @todo Test 16-bit jump in 64-bit mode. possible? */
4359	pVCpu->cpum.GstCtx.rip = uNewIp;
4360	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4361
4362	#ifndef IEM_WITH_CODE_TLB
4363	/* Flush the prefetch buffer. */
4364	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4365	#endif
4366
4367	return VINF_SUCCESS;
4368	}
4369
4370
4371	/**
4372	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
4373	*
4374	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4375	* segment limit.
4376	*
4377	* @returns Strict VBox status code.
4378	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4379	* @param offNextInstr The offset of the next instruction.
4380	*/
4381	VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT
4382	{
4383	Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
4384
4385	if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
4386	{
4387	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4388
4389	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4390	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4391	return iemRaiseGeneralProtectionFault0(pVCpu);
4392	pVCpu->cpum.GstCtx.rip = uNewEip;
4393	}
4394	else
4395	{
4396	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4397
4398	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4399	if (!IEM_IS_CANONICAL(uNewRip))
4400	return iemRaiseGeneralProtectionFault0(pVCpu);
4401	pVCpu->cpum.GstCtx.rip = uNewRip;
4402	}
4403	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4404
4405	#ifndef IEM_WITH_CODE_TLB
4406	/* Flush the prefetch buffer. */
4407	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4408	#endif
4409
4410	return VINF_SUCCESS;
4411	}
4412
4413
4414	/**
4415	* Performs a near jump to the specified address.
4416	*
4417	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4418	* segment limit.
4419	*
4420	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4421	* @param uNewRip The new RIP value.
4422	*/
4423	VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
4424	{
4425	switch (pVCpu->iem.s.enmEffOpSize)
4426	{
4427	case IEMMODE_16BIT:
4428	{
4429	Assert(uNewRip <= UINT16_MAX);
4430	if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
4431	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4432	return iemRaiseGeneralProtectionFault0(pVCpu);
4433	/** @todo Test 16-bit jump in 64-bit mode. */
4434	pVCpu->cpum.GstCtx.rip = uNewRip;
4435	break;
4436	}
4437
4438	case IEMMODE_32BIT:
4439	{
4440	Assert(uNewRip <= UINT32_MAX);
4441	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4442	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4443
4444	if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
4445	return iemRaiseGeneralProtectionFault0(pVCpu);
4446	pVCpu->cpum.GstCtx.rip = uNewRip;
4447	break;
4448	}
4449
4450	case IEMMODE_64BIT:
4451	{
4452	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4453
4454	if (!IEM_IS_CANONICAL(uNewRip))
4455	return iemRaiseGeneralProtectionFault0(pVCpu);
4456	pVCpu->cpum.GstCtx.rip = uNewRip;
4457	break;
4458	}
4459
4460	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4461	}
4462
4463	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4464
4465	#ifndef IEM_WITH_CODE_TLB
4466	/* Flush the prefetch buffer. */
4467	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4468	#endif
4469
4470	return VINF_SUCCESS;
4471	}
4472
4473	/** @} */
4474
4475
4476	/** @name FPU access and helpers.
4477	*
4478	* @{
4479	*/
4480
4481	/**
4482	* Updates the x87.DS and FPUDP registers.
4483	*
4484	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4485	* @param pFpuCtx The FPU context.
4486	* @param iEffSeg The effective segment register.
4487	* @param GCPtrEff The effective address relative to @a iEffSeg.
4488	*/
4489	DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
4490	{
4491	RTSEL sel;
4492	switch (iEffSeg)
4493	{
4494	case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
4495	case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
4496	case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
4497	case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
4498	case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
4499	case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
4500	default:
4501	AssertMsgFailed(("%d\n", iEffSeg));
4502	sel = pVCpu->cpum.GstCtx.ds.Sel;
4503	}
4504	/** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
4505	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4506	{
4507	pFpuCtx->DS = 0;
4508	pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
4509	}
4510	else if (!IEM_IS_LONG_MODE(pVCpu))
4511	{
4512	pFpuCtx->DS = sel;
4513	pFpuCtx->FPUDP = GCPtrEff;
4514	}
4515	else
4516	(uint64_t )&pFpuCtx->FPUDP = GCPtrEff;
4517	}
4518
4519
4520	/**
4521	* Rotates the stack registers in the push direction.
4522	*
4523	* @param pFpuCtx The FPU context.
4524	* @remarks This is a complete waste of time, but fxsave stores the registers in
4525	* stack order.
4526	*/
4527	DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
4528	{
4529	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
4530	pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
4531	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
4532	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
4533	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
4534	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
4535	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
4536	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
4537	pFpuCtx->aRegs[0].r80 = r80Tmp;
4538	}
4539
4540
4541	/**
4542	* Rotates the stack registers in the pop direction.
4543	*
4544	* @param pFpuCtx The FPU context.
4545	* @remarks This is a complete waste of time, but fxsave stores the registers in
4546	* stack order.
4547	*/
4548	DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
4549	{
4550	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
4551	pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
4552	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
4553	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
4554	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
4555	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
4556	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
4557	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
4558	pFpuCtx->aRegs[7].r80 = r80Tmp;
4559	}
4560
4561
4562	/**
4563	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
4564	* exception prevents it.
4565	*
4566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4567	* @param pResult The FPU operation result to push.
4568	* @param pFpuCtx The FPU context.
4569	*/
4570	static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4571	{
4572	/* Update FSW and bail if there are pending exceptions afterwards. */
4573	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4574	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4575	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4576	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4577	{
4578	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES))
4579	Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n",
4580	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4581	pFpuCtx->FSW = fFsw;
4582	return;
4583	}
4584
4585	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4586	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4587	{
4588	/* All is fine, push the actual value. */
4589	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4590	pFpuCtx->aRegs[7].r80 = pResult->r80Result;
4591	}
4592	else if (pFpuCtx->FCW & X86_FCW_IM)
4593	{
4594	/* Masked stack overflow, push QNaN. */
4595	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4596	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4597	}
4598	else
4599	{
4600	/* Raise stack overflow, don't push anything. */
4601	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4602	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4603	Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4604	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4605	return;
4606	}
4607
4608	fFsw &= ~X86_FSW_TOP_MASK;
4609	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4610	pFpuCtx->FSW = fFsw;
4611
4612	iemFpuRotateStackPush(pFpuCtx);
4613	RT_NOREF(pVCpu);
4614	}
4615
4616
4617	/**
4618	* Stores a result in a FPU register and updates the FSW and FTW.
4619	*
4620	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4621	* @param pFpuCtx The FPU context.
4622	* @param pResult The result to store.
4623	* @param iStReg Which FPU register to store it in.
4624	*/
4625	static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4626	{
4627	Assert(iStReg < 8);
4628	uint16_t fNewFsw = pFpuCtx->FSW;
4629	uint16_t const iReg = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK;
4630	fNewFsw &= ~X86_FSW_C_MASK;
4631	fNewFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4632	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4633	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4634	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4635	pFpuCtx->FSW = fNewFsw;
4636	pFpuCtx->FTW \|= RT_BIT(iReg);
4637	pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
4638	RT_NOREF(pVCpu);
4639	}
4640
4641
4642	/**
4643	* Only updates the FPU status word (FSW) with the result of the current
4644	* instruction.
4645	*
4646	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4647	* @param pFpuCtx The FPU context.
4648	* @param u16FSW The FSW output of the current instruction.
4649	*/
4650	static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT
4651	{
4652	uint16_t fNewFsw = pFpuCtx->FSW;
4653	fNewFsw &= ~X86_FSW_C_MASK;
4654	fNewFsw \|= u16FSW & ~X86_FSW_TOP_MASK;
4655	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4656	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4657	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4658	pFpuCtx->FSW = fNewFsw;
4659	RT_NOREF(pVCpu);
4660	}
4661
4662
4663	/**
4664	* Pops one item off the FPU stack if no pending exception prevents it.
4665	*
4666	* @param pFpuCtx The FPU context.
4667	*/
4668	static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4669	{
4670	/* Check pending exceptions. */
4671	uint16_t uFSW = pFpuCtx->FSW;
4672	if ( (pFpuCtx->FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4673	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4674	return;
4675
4676	/* TOP--. */
4677	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
4678	uFSW &= ~X86_FSW_TOP_MASK;
4679	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
4680	pFpuCtx->FSW = uFSW;
4681
4682	/* Mark the previous ST0 as empty. */
4683	iOldTop >>= X86_FSW_TOP_SHIFT;
4684	pFpuCtx->FTW &= ~RT_BIT(iOldTop);
4685
4686	/* Rotate the registers. */
4687	iemFpuRotateStackPop(pFpuCtx);
4688	}
4689
4690
4691	/**
4692	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
4693	*
4694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4695	* @param pResult The FPU operation result to push.
4696	*/
4697	void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT
4698	{
4699	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4700	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4701	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4702	}
4703
4704
4705	/**
4706	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
4707	* and sets FPUDP and FPUDS.
4708	*
4709	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4710	* @param pResult The FPU operation result to push.
4711	* @param iEffSeg The effective segment register.
4712	* @param GCPtrEff The effective address relative to @a iEffSeg.
4713	*/
4714	void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4715	{
4716	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4717	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4718	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4719	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4720	}
4721
4722
4723	/**
4724	* Replace ST0 with the first value and push the second onto the FPU stack,
4725	* unless a pending exception prevents it.
4726	*
4727	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4728	* @param pResult The FPU operation result to store and push.
4729	*/
4730	void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT
4731	{
4732	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4733	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4734
4735	/* Update FSW and bail if there are pending exceptions afterwards. */
4736	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4737	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4738	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4739	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4740	{
4741	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4742	Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n",
4743	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4744	pFpuCtx->FSW = fFsw;
4745	return;
4746	}
4747
4748	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4749	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4750	{
4751	/* All is fine, push the actual value. */
4752	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4753	pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
4754	pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
4755	}
4756	else if (pFpuCtx->FCW & X86_FCW_IM)
4757	{
4758	/* Masked stack overflow, push QNaN. */
4759	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4760	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
4761	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4762	}
4763	else
4764	{
4765	/* Raise stack overflow, don't push anything. */
4766	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4767	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4768	Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4769	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4770	return;
4771	}
4772
4773	fFsw &= ~X86_FSW_TOP_MASK;
4774	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4775	pFpuCtx->FSW = fFsw;
4776
4777	iemFpuRotateStackPush(pFpuCtx);
4778	}
4779
4780
4781	/**
4782	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4783	* FOP.
4784	*
4785	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4786	* @param pResult The result to store.
4787	* @param iStReg Which FPU register to store it in.
4788	*/
4789	void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4790	{
4791	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4792	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4793	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4794	}
4795
4796
4797	/**
4798	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4799	* FOP, and then pops the stack.
4800	*
4801	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4802	* @param pResult The result to store.
4803	* @param iStReg Which FPU register to store it in.
4804	*/
4805	void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4806	{
4807	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4808	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4809	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4810	iemFpuMaybePopOne(pFpuCtx);
4811	}
4812
4813
4814	/**
4815	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4816	* FPUDP, and FPUDS.
4817	*
4818	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4819	* @param pResult The result to store.
4820	* @param iStReg Which FPU register to store it in.
4821	* @param iEffSeg The effective memory operand selector register.
4822	* @param GCPtrEff The effective memory operand offset.
4823	*/
4824	void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
4825	uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4826	{
4827	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4828	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4829	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4830	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4831	}
4832
4833
4834	/**
4835	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4836	* FPUDP, and FPUDS, and then pops the stack.
4837	*
4838	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4839	* @param pResult The result to store.
4840	* @param iStReg Which FPU register to store it in.
4841	* @param iEffSeg The effective memory operand selector register.
4842	* @param GCPtrEff The effective memory operand offset.
4843	*/
4844	void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
4845	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4846	{
4847	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4848	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4849	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4850	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4851	iemFpuMaybePopOne(pFpuCtx);
4852	}
4853
4854
4855	/**
4856	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
4857	*
4858	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4859	*/
4860	void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT
4861	{
4862	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4863	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4864	}
4865
4866
4867	/**
4868	* Updates the FSW, FOP, FPUIP, and FPUCS.
4869	*
4870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4871	* @param u16FSW The FSW from the current instruction.
4872	*/
4873	void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4874	{
4875	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4876	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4877	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4878	}
4879
4880
4881	/**
4882	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
4883	*
4884	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4885	* @param u16FSW The FSW from the current instruction.
4886	*/
4887	void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4888	{
4889	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4890	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4891	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4892	iemFpuMaybePopOne(pFpuCtx);
4893	}
4894
4895
4896	/**
4897	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
4898	*
4899	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4900	* @param u16FSW The FSW from the current instruction.
4901	* @param iEffSeg The effective memory operand selector register.
4902	* @param GCPtrEff The effective memory operand offset.
4903	*/
4904	void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4905	{
4906	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4907	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4908	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4909	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4910	}
4911
4912
4913	/**
4914	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
4915	*
4916	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4917	* @param u16FSW The FSW from the current instruction.
4918	*/
4919	void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4920	{
4921	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4922	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4923	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4924	iemFpuMaybePopOne(pFpuCtx);
4925	iemFpuMaybePopOne(pFpuCtx);
4926	}
4927
4928
4929	/**
4930	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
4931	*
4932	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4933	* @param u16FSW The FSW from the current instruction.
4934	* @param iEffSeg The effective memory operand selector register.
4935	* @param GCPtrEff The effective memory operand offset.
4936	*/
4937	void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4938	{
4939	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4940	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4941	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4942	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4943	iemFpuMaybePopOne(pFpuCtx);
4944	}
4945
4946
4947	/**
4948	* Worker routine for raising an FPU stack underflow exception.
4949	*
4950	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4951	* @param pFpuCtx The FPU context.
4952	* @param iStReg The stack register being accessed.
4953	*/
4954	static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg)
4955	{
4956	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
4957	if (pFpuCtx->FCW & X86_FCW_IM)
4958	{
4959	/* Masked underflow. */
4960	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4961	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
4962	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
4963	if (iStReg != UINT8_MAX)
4964	{
4965	pFpuCtx->FTW \|= RT_BIT(iReg);
4966	iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
4967	}
4968	}
4969	else
4970	{
4971	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4972	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
4973	Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n",
4974	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4975	}
4976	RT_NOREF(pVCpu);
4977	}
4978
4979
4980	/**
4981	* Raises a FPU stack underflow exception.
4982	*
4983	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4984	* @param iStReg The destination register that should be loaded
4985	* with QNaN if \#IS is not masked. Specify
4986	* UINT8_MAX if none (like for fcom).
4987	*/
4988	void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
4989	{
4990	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4991	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4992	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
4993	}
4994
4995
4996	void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4997	{
4998	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4999	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5000	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5001	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5002	}
5003
5004
5005	void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
5006	{
5007	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5008	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5009	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5010	iemFpuMaybePopOne(pFpuCtx);
5011	}
5012
5013
5014	void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5015	{
5016	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5017	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5018	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5019	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5020	iemFpuMaybePopOne(pFpuCtx);
5021	}
5022
5023
5024	void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT
5025	{
5026	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5027	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5028	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX);
5029	iemFpuMaybePopOne(pFpuCtx);
5030	iemFpuMaybePopOne(pFpuCtx);
5031	}
5032
5033
5034	void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5035	{
5036	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5037	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5038
5039	if (pFpuCtx->FCW & X86_FCW_IM)
5040	{
5041	/* Masked overflow - Push QNaN. */
5042	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5043	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5044	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5045	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5046	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5047	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5048	iemFpuRotateStackPush(pFpuCtx);
5049	}
5050	else
5051	{
5052	/* Exception pending - don't change TOP or the register stack. */
5053	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5054	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5055	Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n",
5056	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5057	}
5058	}
5059
5060
5061	void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT
5062	{
5063	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5064	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5065
5066	if (pFpuCtx->FCW & X86_FCW_IM)
5067	{
5068	/* Masked overflow - Push QNaN. */
5069	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5070	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5071	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5072	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5073	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5074	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
5075	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5076	iemFpuRotateStackPush(pFpuCtx);
5077	}
5078	else
5079	{
5080	/* Exception pending - don't change TOP or the register stack. */
5081	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5082	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5083	Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n",
5084	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5085	}
5086	}
5087
5088
5089	/**
5090	* Worker routine for raising an FPU stack overflow exception on a push.
5091	*
5092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5093	* @param pFpuCtx The FPU context.
5094	*/
5095	static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
5096	{
5097	if (pFpuCtx->FCW & X86_FCW_IM)
5098	{
5099	/* Masked overflow. */
5100	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5101	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5102	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
5103	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5104	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5105	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5106	iemFpuRotateStackPush(pFpuCtx);
5107	}
5108	else
5109	{
5110	/* Exception pending - don't change TOP or the register stack. */
5111	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5112	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5113	Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n",
5114	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5115	}
5116	RT_NOREF(pVCpu);
5117	}
5118
5119
5120	/**
5121	* Raises a FPU stack overflow exception on a push.
5122	*
5123	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5124	*/
5125	void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5126	{
5127	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5128	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5129	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5130	}
5131
5132
5133	/**
5134	* Raises a FPU stack overflow exception on a push with a memory operand.
5135	*
5136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5137	* @param iEffSeg The effective memory operand selector register.
5138	* @param GCPtrEff The effective memory operand offset.
5139	*/
5140	void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5141	{
5142	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5143	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5144	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5145	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5146	}
5147
5148	/** @} */
5149
5150
5151	/** @name SSE+AVX SIMD access and helpers.
5152	*
5153	* @{
5154	*/
5155	/**
5156	* Stores a result in a SIMD XMM register, updates the MXCSR.
5157	*
5158	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5159	* @param pResult The result to store.
5160	* @param iXmmReg Which SIMD XMM register to store the result in.
5161	*/
5162	void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT
5163	{
5164	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5165	pFpuCtx->MXCSR \|= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS;
5166	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult;
5167	}
5168
5169	/** @} */
5170
5171
5172	/** @name Memory access.
5173	*
5174	* @{
5175	*/
5176
5177
5178	/**
5179	* Updates the IEMCPU::cbWritten counter if applicable.
5180	*
5181	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5182	* @param fAccess The access being accounted for.
5183	* @param cbMem The access size.
5184	*/
5185	DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
5186	{
5187	if ( (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK \| IEM_ACCESS_TYPE_WRITE)
5188	\|\| (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_TYPE_WRITE) )
5189	pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
5190	}
5191
5192
5193	/**
5194	* Applies the segment limit, base and attributes.
5195	*
5196	* This may raise a \#GP or \#SS.
5197	*
5198	* @returns VBox strict status code.
5199	*
5200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5201	* @param fAccess The kind of access which is being performed.
5202	* @param iSegReg The index of the segment register to apply.
5203	* This is UINT8_MAX if none (for IDT, GDT, LDT,
5204	* TSS, ++).
5205	* @param cbMem The access size.
5206	* @param pGCPtrMem Pointer to the guest memory address to apply
5207	* segmentation to. Input and output parameter.
5208	*/
5209	VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT
5210	{
5211	if (iSegReg == UINT8_MAX)
5212	return VINF_SUCCESS;
5213
5214	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
5215	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
5216	switch (pVCpu->iem.s.enmCpuMode)
5217	{
5218	case IEMMODE_16BIT:
5219	case IEMMODE_32BIT:
5220	{
5221	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
5222	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
5223
5224	if ( pSel->Attr.n.u1Present
5225	&& !pSel->Attr.n.u1Unusable)
5226	{
5227	Assert(pSel->Attr.n.u1DescType);
5228	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5229	{
5230	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5231	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
5232	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5233
5234	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5235	{
5236	/** @todo CPL check. */
5237	}
5238
5239	/*
5240	* There are two kinds of data selectors, normal and expand down.
5241	*/
5242	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5243	{
5244	if ( GCPtrFirst32 > pSel->u32Limit
5245	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5246	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5247	}
5248	else
5249	{
5250	/*
5251	* The upper boundary is defined by the B bit, not the G bit!
5252	*/
5253	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5254	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5255	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5256	}
5257	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5258	}
5259	else
5260	{
5261	/*
5262	* Code selector and usually be used to read thru, writing is
5263	* only permitted in real and V8086 mode.
5264	*/
5265	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5266	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
5267	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
5268	&& !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
5269	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5270
5271	if ( GCPtrFirst32 > pSel->u32Limit
5272	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5273	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5274
5275	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5276	{
5277	/** @todo CPL check. */
5278	}
5279
5280	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5281	}
5282	}
5283	else
5284	return iemRaiseGeneralProtectionFault0(pVCpu);
5285	return VINF_SUCCESS;
5286	}
5287
5288	case IEMMODE_64BIT:
5289	{
5290	RTGCPTR GCPtrMem = *pGCPtrMem;
5291	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
5292	*pGCPtrMem = GCPtrMem + pSel->u64Base;
5293
5294	Assert(cbMem >= 1);
5295	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
5296	return VINF_SUCCESS;
5297	/** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
5298	* 4.12.2 "Data Limit Checks in 64-bit Mode". */
5299	return iemRaiseGeneralProtectionFault0(pVCpu);
5300	}
5301
5302	default:
5303	AssertFailedReturn(VERR_IEM_IPE_7);
5304	}
5305	}
5306
5307
5308	/**
5309	* Translates a virtual address to a physical physical address and checks if we
5310	* can access the page as specified.
5311	*
5312	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5313	* @param GCPtrMem The virtual address.
5314	* @param fAccess The intended access.
5315	* @param pGCPhysMem Where to return the physical address.
5316	*/
5317	VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT
5318	{
5319	/** @todo Need a different PGM interface here. We're currently using
5320	* generic / REM interfaces. this won't cut it for R0. */
5321	/** @todo If/when PGM handles paged real-mode, we can remove the hack in
5322	* iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
5323	* here. */
5324	PGMPTWALK Walk;
5325	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5326	if (RT_FAILURE(rc))
5327	{
5328	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
5329	/** @todo Check unassigned memory in unpaged mode. */
5330	/** @todo Reserved bits in page tables. Requires new PGM interface. */
5331	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5332	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5333	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
5334	#endif
5335	*pGCPhysMem = NIL_RTGCPHYS;
5336	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
5337	}
5338
5339	/* If the page is writable and does not have the no-exec bit set, all
5340	access is allowed. Otherwise we'll have to check more carefully... */
5341	if ((Walk.fEffective & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
5342	{
5343	/* Write to read only memory? */
5344	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5345	&& !(Walk.fEffective & X86_PTE_RW)
5346	&& ( ( pVCpu->iem.s.uCpl == 3
5347	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5348	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
5349	{
5350	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
5351	*pGCPhysMem = NIL_RTGCPHYS;
5352	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5353	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5354	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5355	#endif
5356	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
5357	}
5358
5359	/* Kernel memory accessed by userland? */
5360	if ( !(Walk.fEffective & X86_PTE_US)
5361	&& pVCpu->iem.s.uCpl == 3
5362	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5363	{
5364	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
5365	*pGCPhysMem = NIL_RTGCPHYS;
5366	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5367	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5368	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5369	#endif
5370	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
5371	}
5372
5373	/* Executing non-executable memory? */
5374	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
5375	&& (Walk.fEffective & X86_PTE_PAE_NX)
5376	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
5377	{
5378	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
5379	*pGCPhysMem = NIL_RTGCPHYS;
5380	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5381	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5382	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5383	#endif
5384	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
5385	VERR_ACCESS_DENIED);
5386	}
5387	}
5388
5389	/*
5390	* Set the dirty / access flags.
5391	* ASSUMES this is set when the address is translated rather than on committ...
5392	*/
5393	/** @todo testcase: check when A and D bits are actually set by the CPU. */
5394	uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
5395	if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
5396	{
5397	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
5398	AssertRC(rc2);
5399	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
5400	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
5401	}
5402
5403	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
5404	*pGCPhysMem = GCPhys;
5405	return VINF_SUCCESS;
5406	}
5407
5408
5409	/**
5410	* Looks up a memory mapping entry.
5411	*
5412	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
5413	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5414	* @param pvMem The memory address.
5415	* @param fAccess The access to.
5416	*/
5417	DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
5418	{
5419	Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5420	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
5421	if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
5422	&& (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5423	return 0;
5424	if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
5425	&& (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5426	return 1;
5427	if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
5428	&& (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5429	return 2;
5430	return VERR_NOT_FOUND;
5431	}
5432
5433
5434	/**
5435	* Finds a free memmap entry when using iNextMapping doesn't work.
5436	*
5437	* @returns Memory mapping index, 1024 on failure.
5438	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5439	*/
5440	static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
5441	{
5442	/*
5443	* The easy case.
5444	*/
5445	if (pVCpu->iem.s.cActiveMappings == 0)
5446	{
5447	pVCpu->iem.s.iNextMapping = 1;
5448	return 0;
5449	}
5450
5451	/* There should be enough mappings for all instructions. */
5452	AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
5453
5454	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
5455	if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
5456	return i;
5457
5458	AssertFailedReturn(1024);
5459	}
5460
5461
5462	/**
5463	* Commits a bounce buffer that needs writing back and unmaps it.
5464	*
5465	* @returns Strict VBox status code.
5466	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5467	* @param iMemMap The index of the buffer to commit.
5468	* @param fPostponeFail Whether we can postpone writer failures to ring-3.
5469	* Always false in ring-3, obviously.
5470	*/
5471	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
5472	{
5473	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
5474	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
5475	#ifdef IN_RING3
5476	Assert(!fPostponeFail);
5477	RT_NOREF_PV(fPostponeFail);
5478	#endif
5479
5480	/*
5481	* Do the writing.
5482	*/
5483	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5484	if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
5485	{
5486	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
5487	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5488	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5489	if (!pVCpu->iem.s.fBypassHandlers)
5490	{
5491	/*
5492	* Carefully and efficiently dealing with access handler return
5493	* codes make this a little bloated.
5494	*/
5495	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
5496	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5497	pbBuf,
5498	cbFirst,
5499	PGMACCESSORIGIN_IEM);
5500	if (rcStrict == VINF_SUCCESS)
5501	{
5502	if (cbSecond)
5503	{
5504	rcStrict = PGMPhysWrite(pVM,
5505	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5506	pbBuf + cbFirst,
5507	cbSecond,
5508	PGMACCESSORIGIN_IEM);
5509	if (rcStrict == VINF_SUCCESS)
5510	{ /* nothing */ }
5511	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5512	{
5513	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
5514	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5515	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5516	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5517	}
5518	#ifndef IN_RING3
5519	else if (fPostponeFail)
5520	{
5521	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5522	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5523	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5524	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5525	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5526	return iemSetPassUpStatus(pVCpu, rcStrict);
5527	}
5528	#endif
5529	else
5530	{
5531	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5532	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5533	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5534	return rcStrict;
5535	}
5536	}
5537	}
5538	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5539	{
5540	if (!cbSecond)
5541	{
5542	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
5543	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5544	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5545	}
5546	else
5547	{
5548	VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
5549	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5550	pbBuf + cbFirst,
5551	cbSecond,
5552	PGMACCESSORIGIN_IEM);
5553	if (rcStrict2 == VINF_SUCCESS)
5554	{
5555	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
5556	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5557	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5558	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5559	}
5560	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5561	{
5562	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
5563	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5564	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5565	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5566	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5567	}
5568	#ifndef IN_RING3
5569	else if (fPostponeFail)
5570	{
5571	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5572	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5573	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5574	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5575	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5576	return iemSetPassUpStatus(pVCpu, rcStrict);
5577	}
5578	#endif
5579	else
5580	{
5581	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5582	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5583	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5584	return rcStrict2;
5585	}
5586	}
5587	}
5588	#ifndef IN_RING3
5589	else if (fPostponeFail)
5590	{
5591	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5592	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5593	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5594	if (!cbSecond)
5595	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST;
5596	else
5597	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND;
5598	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5599	return iemSetPassUpStatus(pVCpu, rcStrict);
5600	}
5601	#endif
5602	else
5603	{
5604	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5605	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5606	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5607	return rcStrict;
5608	}
5609	}
5610	else
5611	{
5612	/*
5613	* No access handlers, much simpler.
5614	*/
5615	int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
5616	if (RT_SUCCESS(rc))
5617	{
5618	if (cbSecond)
5619	{
5620	rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
5621	if (RT_SUCCESS(rc))
5622	{ /* likely */ }
5623	else
5624	{
5625	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5626	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5627	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
5628	return rc;
5629	}
5630	}
5631	}
5632	else
5633	{
5634	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5635	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
5636	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5637	return rc;
5638	}
5639	}
5640	}
5641
5642	#if defined(IEM_LOG_MEMORY_WRITES)
5643	Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5644	RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
5645	if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
5646	Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5647	RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
5648	&pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
5649
5650	size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5651	g_cbIemWrote = cbWrote;
5652	memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
5653	#endif
5654
5655	/*
5656	* Free the mapping entry.
5657	*/
5658	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
5659	Assert(pVCpu->iem.s.cActiveMappings != 0);
5660	pVCpu->iem.s.cActiveMappings--;
5661	return VINF_SUCCESS;
5662	}
5663
5664
5665	/**
5666	* iemMemMap worker that deals with a request crossing pages.
5667	*/
5668	static VBOXSTRICTRC
5669	iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
5670	{
5671	/*
5672	* Do the address translations.
5673	*/
5674	RTGCPHYS GCPhysFirst;
5675	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
5676	if (rcStrict != VINF_SUCCESS)
5677	return rcStrict;
5678
5679	RTGCPHYS GCPhysSecond;
5680	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
5681	fAccess, &GCPhysSecond);
5682	if (rcStrict != VINF_SUCCESS)
5683	return rcStrict;
5684	GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
5685
5686	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5687
5688	/*
5689	* Read in the current memory content if it's a read, execute or partial
5690	* write access.
5691	*/
5692	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5693	uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (GCPhysFirst & GUEST_PAGE_OFFSET_MASK);
5694	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
5695
5696	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
5697	{
5698	if (!pVCpu->iem.s.fBypassHandlers)
5699	{
5700	/*
5701	* Must carefully deal with access handler status codes here,
5702	* makes the code a bit bloated.
5703	*/
5704	rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
5705	if (rcStrict == VINF_SUCCESS)
5706	{
5707	rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5708	if (rcStrict == VINF_SUCCESS)
5709	{ /likely / }
5710	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5711	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5712	else
5713	{
5714	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
5715	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5716	return rcStrict;
5717	}
5718	}
5719	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5720	{
5721	VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5722	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5723	{
5724	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5725	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5726	}
5727	else
5728	{
5729	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
5730	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
5731	return rcStrict2;
5732	}
5733	}
5734	else
5735	{
5736	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5737	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5738	return rcStrict;
5739	}
5740	}
5741	else
5742	{
5743	/*
5744	* No informational status codes here, much more straight forward.
5745	*/
5746	int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
5747	if (RT_SUCCESS(rc))
5748	{
5749	Assert(rc == VINF_SUCCESS);
5750	rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
5751	if (RT_SUCCESS(rc))
5752	Assert(rc == VINF_SUCCESS);
5753	else
5754	{
5755	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
5756	return rc;
5757	}
5758	}
5759	else
5760	{
5761	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
5762	return rc;
5763	}
5764	}
5765	}
5766	#ifdef VBOX_STRICT
5767	else
5768	memset(pbBuf, 0xcc, cbMem);
5769	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5770	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5771	#endif
5772
5773	/*
5774	* Commit the bounce buffer entry.
5775	*/
5776	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5777	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
5778	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
5779	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
5780	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
5781	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5782	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
5783	pVCpu->iem.s.iNextMapping = iMemMap + 1;
5784	pVCpu->iem.s.cActiveMappings++;
5785
5786	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5787	*ppvMem = pbBuf;
5788	return VINF_SUCCESS;
5789	}
5790
5791
5792	/**
5793	* iemMemMap woker that deals with iemMemPageMap failures.
5794	*/
5795	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
5796	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
5797	{
5798	/*
5799	* Filter out conditions we can handle and the ones which shouldn't happen.
5800	*/
5801	if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
5802	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
5803	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
5804	{
5805	AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
5806	return rcMap;
5807	}
5808	pVCpu->iem.s.cPotentialExits++;
5809
5810	/*
5811	* Read in the current memory content if it's a read, execute or partial
5812	* write access.
5813	*/
5814	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5815	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
5816	{
5817	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
5818	memset(pbBuf, 0xff, cbMem);
5819	else
5820	{
5821	int rc;
5822	if (!pVCpu->iem.s.fBypassHandlers)
5823	{
5824	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
5825	if (rcStrict == VINF_SUCCESS)
5826	{ /* nothing */ }
5827	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5828	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5829	else
5830	{
5831	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5832	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5833	return rcStrict;
5834	}
5835	}
5836	else
5837	{
5838	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
5839	if (RT_SUCCESS(rc))
5840	{ /* likely */ }
5841	else
5842	{
5843	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5844	GCPhysFirst, rc));
5845	return rc;
5846	}
5847	}
5848	}
5849	}
5850	#ifdef VBOX_STRICT
5851	else
5852	memset(pbBuf, 0xcc, cbMem);
5853	#endif
5854	#ifdef VBOX_STRICT
5855	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5856	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5857	#endif
5858
5859	/*
5860	* Commit the bounce buffer entry.
5861	*/
5862	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5863	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
5864	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
5865	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
5866	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
5867	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5868	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
5869	pVCpu->iem.s.iNextMapping = iMemMap + 1;
5870	pVCpu->iem.s.cActiveMappings++;
5871
5872	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5873	*ppvMem = pbBuf;
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878
5879	/**
5880	* Maps the specified guest memory for the given kind of access.
5881	*
5882	* This may be using bounce buffering of the memory if it's crossing a page
5883	* boundary or if there is an access handler installed for any of it. Because
5884	* of lock prefix guarantees, we're in for some extra clutter when this
5885	* happens.
5886	*
5887	* This may raise a \#GP, \#SS, \#PF or \#AC.
5888	*
5889	* @returns VBox strict status code.
5890	*
5891	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5892	* @param ppvMem Where to return the pointer to the mapped memory.
5893	* @param cbMem The number of bytes to map. This is usually 1, 2, 4, 6,
5894	* 8, 12, 16, 32 or 512. When used by string operations
5895	* it can be up to a page.
5896	* @param iSegReg The index of the segment register to use for this
5897	* access. The base and limits are checked. Use UINT8_MAX
5898	* to indicate that no segmentation is required (for IDT,
5899	* GDT and LDT accesses).
5900	* @param GCPtrMem The address of the guest memory.
5901	* @param fAccess How the memory is being accessed. The
5902	* IEM_ACCESS_TYPE_XXX bit is used to figure out how to map
5903	* the memory, while the IEM_ACCESS_WHAT_XXX bit is used
5904	* when raising exceptions.
5905	* @param uAlignCtl Alignment control:
5906	* - Bits 15:0 is the alignment mask.
5907	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
5908	* IEM_MEMMAP_F_ALIGN_SSE, and
5909	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
5910	* Pass zero to skip alignment.
5911	*/
5912	VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
5913	uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT
5914	{
5915	/*
5916	* Check the input and figure out which mapping entry to use.
5917	*/
5918	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 256 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 102 \|\| cbMem == 94); /* 512 is the max! */
5919	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
5920	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5921
5922	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
5923	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
5924	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
5925	{
5926	iMemMap = iemMemMapFindFree(pVCpu);
5927	AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
5928	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
5929	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
5930	pVCpu->iem.s.aMemMappings[2].fAccess),
5931	VERR_IEM_IPE_9);
5932	}
5933
5934	/*
5935	* Map the memory, checking that we can actually access it. If something
5936	* slightly complicated happens, fall back on bounce buffering.
5937	*/
5938	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
5939	if (rcStrict == VINF_SUCCESS)
5940	{ /* likely */ }
5941	else
5942	return rcStrict;
5943
5944	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */
5945	{ /* likely */ }
5946	else
5947	return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
5948
5949	/*
5950	* Alignment check.
5951	*/
5952	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
5953	{ /* likelyish */ }
5954	else
5955	{
5956	/* Misaligned access. */
5957	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
5958	{
5959	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
5960	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
5961	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
5962	{
5963	AssertCompile(X86_CR0_AM == X86_EFL_AC);
5964
5965	if (iemMemAreAlignmentChecksEnabled(pVCpu))
5966	return iemRaiseAlignmentCheckException(pVCpu);
5967	}
5968	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
5969	&& iemMemAreAlignmentChecksEnabled(pVCpu)
5970	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
5971	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
5972	)
5973	return iemRaiseAlignmentCheckException(pVCpu);
5974	else
5975	return iemRaiseGeneralProtectionFault0(pVCpu);
5976	}
5977	}
5978
5979	#ifdef IEM_WITH_DATA_TLB
5980	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
5981
5982	/*
5983	* Get the TLB entry for this page.
5984	*/
5985	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
5986	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
5987	if (pTlbe->uTag == uTag)
5988	{
5989	# ifdef VBOX_WITH_STATISTICS
5990	pVCpu->iem.s.DataTlb.cTlbHits++;
5991	# endif
5992	}
5993	else
5994	{
5995	pVCpu->iem.s.DataTlb.cTlbMisses++;
5996	PGMPTWALK Walk;
5997	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5998	if (RT_FAILURE(rc))
5999	{
6000	Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6001	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6002	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6003	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6004	# endif
6005	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6006	}
6007
6008	Assert(Walk.fSucceeded);
6009	pTlbe->uTag = uTag;
6010	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6011	pTlbe->GCPhys = Walk.GCPhys;
6012	pTlbe->pbMappingR3 = NULL;
6013	}
6014
6015	/*
6016	* Check TLB page table level access flags.
6017	*/
6018	/* If the page is either supervisor only or non-writable, we need to do
6019	more careful access checks. */
6020	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_WRITE))
6021	{
6022	/* Write to read only memory? */
6023	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
6024	&& (fAccess & IEM_ACCESS_TYPE_WRITE)
6025	&& ( ( pVCpu->iem.s.uCpl == 3
6026	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6027	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
6028	{
6029	Log(("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6030	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6031	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6032	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6033	# endif
6034	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6035	}
6036
6037	/* Kernel memory accessed by userland? */
6038	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
6039	&& pVCpu->iem.s.uCpl == 3
6040	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6041	{
6042	Log(("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6043	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6044	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6045	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6046	# endif
6047	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6048	}
6049	}
6050
6051	/*
6052	* Set the dirty / access flags.
6053	* ASSUMES this is set when the address is translated rather than on commit...
6054	*/
6055	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6056	uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) \| IEMTLBE_F_PT_NO_ACCESSED;
6057	if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty)
6058	{
6059	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6060	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6061	AssertRC(rc2);
6062	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6063	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6064	pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty;
6065	}
6066
6067	/*
6068	* Look up the physical page info if necessary.
6069	*/
6070	uint8_t *pbMem = NULL;
6071	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6072	# ifdef IN_RING3
6073	pbMem = pTlbe->pbMappingR3;
6074	# else
6075	pbMem = NULL;
6076	# endif
6077	else
6078	{
6079	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6080	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6081	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6082	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6083	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
6084	{ /* likely */ }
6085	else
6086	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
6087	pTlbe->pbMappingR3 = NULL;
6088	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6089	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
6090	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6091	&pbMem, &pTlbe->fFlagsAndPhysRev);
6092	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6093	# ifdef IN_RING3
6094	pTlbe->pbMappingR3 = pbMem;
6095	# endif
6096	}
6097
6098	/*
6099	* Check the physical page level access and mapping.
6100	*/
6101	if ( !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))
6102	\|\| !(pTlbe->fFlagsAndPhysRev & ( (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0)
6103	\| (fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0))) )
6104	{ /* probably likely */ }
6105	else
6106	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem,
6107	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6108	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6109	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6110	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6111	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6112
6113	if (pbMem)
6114	{
6115	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6116	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6117	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6118	}
6119	else
6120	{
6121	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6122	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6123	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6124	if (rcStrict != VINF_SUCCESS)
6125	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6126	}
6127
6128	void * const pvMem = pbMem;
6129
6130	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6131	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6132	if (fAccess & IEM_ACCESS_TYPE_READ)
6133	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6134
6135	#else /* !IEM_WITH_DATA_TLB */
6136
6137	RTGCPHYS GCPhysFirst;
6138	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6139	if (rcStrict != VINF_SUCCESS)
6140	return rcStrict;
6141
6142	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6143	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6144	if (fAccess & IEM_ACCESS_TYPE_READ)
6145	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6146
6147	void *pvMem;
6148	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6149	if (rcStrict != VINF_SUCCESS)
6150	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6151
6152	#endif /* !IEM_WITH_DATA_TLB */
6153
6154	/*
6155	* Fill in the mapping table entry.
6156	*/
6157	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6158	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6159	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6160	pVCpu->iem.s.cActiveMappings += 1;
6161
6162	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6163	*ppvMem = pvMem;
6164
6165	return VINF_SUCCESS;
6166	}
6167
6168
6169	/**
6170	* Commits the guest memory if bounce buffered and unmaps it.
6171	*
6172	* @returns Strict VBox status code.
6173	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6174	* @param pvMem The mapping.
6175	* @param fAccess The kind of access.
6176	*/
6177	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6178	{
6179	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6180	AssertReturn(iMemMap >= 0, iMemMap);
6181
6182	/* If it's bounce buffered, we may need to write back the buffer. */
6183	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6184	{
6185	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6186	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6187	}
6188	/* Otherwise unlock it. */
6189	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6190	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6191
6192	/* Free the entry. */
6193	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6194	Assert(pVCpu->iem.s.cActiveMappings != 0);
6195	pVCpu->iem.s.cActiveMappings--;
6196	return VINF_SUCCESS;
6197	}
6198
6199	#ifdef IEM_WITH_SETJMP
6200
6201	/**
6202	* Maps the specified guest memory for the given kind of access, longjmp on
6203	* error.
6204	*
6205	* This may be using bounce buffering of the memory if it's crossing a page
6206	* boundary or if there is an access handler installed for any of it. Because
6207	* of lock prefix guarantees, we're in for some extra clutter when this
6208	* happens.
6209	*
6210	* This may raise a \#GP, \#SS, \#PF or \#AC.
6211	*
6212	* @returns Pointer to the mapped memory.
6213	*
6214	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6215	* @param cbMem The number of bytes to map. This is usually 1,
6216	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
6217	* string operations it can be up to a page.
6218	* @param iSegReg The index of the segment register to use for
6219	* this access. The base and limits are checked.
6220	* Use UINT8_MAX to indicate that no segmentation
6221	* is required (for IDT, GDT and LDT accesses).
6222	* @param GCPtrMem The address of the guest memory.
6223	* @param fAccess How the memory is being accessed. The
6224	* IEM_ACCESS_TYPE_XXX bit is used to figure out
6225	* how to map the memory, while the
6226	* IEM_ACCESS_WHAT_XXX bit is used when raising
6227	* exceptions.
6228	* @param uAlignCtl Alignment control:
6229	* - Bits 15:0 is the alignment mask.
6230	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
6231	* IEM_MEMMAP_F_ALIGN_SSE, and
6232	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
6233	* Pass zero to skip alignment.
6234	*/
6235	void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess,
6236	uint32_t uAlignCtl) RT_NOEXCEPT
6237	{
6238	/*
6239	* Check the input, check segment access and adjust address
6240	* with segment base.
6241	*/
6242	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 94); /* 512 is the max! */
6243	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
6244	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
6245
6246	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6247	if (rcStrict == VINF_SUCCESS) { /likely/ }
6248	else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6249
6250	/*
6251	* Alignment check.
6252	*/
6253	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
6254	{ /* likelyish */ }
6255	else
6256	{
6257	/* Misaligned access. */
6258	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6259	{
6260	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
6261	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
6262	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
6263	{
6264	AssertCompile(X86_CR0_AM == X86_EFL_AC);
6265
6266	if (iemMemAreAlignmentChecksEnabled(pVCpu))
6267	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6268	}
6269	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
6270	&& iemMemAreAlignmentChecksEnabled(pVCpu)
6271	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
6272	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
6273	)
6274	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6275	else
6276	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
6277	}
6278	}
6279
6280	/*
6281	* Figure out which mapping entry to use.
6282	*/
6283	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
6284	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6285	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6286	{
6287	iMemMap = iemMemMapFindFree(pVCpu);
6288	AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
6289	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
6290	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
6291	pVCpu->iem.s.aMemMappings[2].fAccess),
6292	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
6293	}
6294
6295	/*
6296	* Crossing a page boundary?
6297	*/
6298	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
6299	{ /* No (likely). */ }
6300	else
6301	{
6302	void *pvMem;
6303	rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
6304	if (rcStrict == VINF_SUCCESS)
6305	return pvMem;
6306	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6307	}
6308
6309	#ifdef IEM_WITH_DATA_TLB
6310	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
6311
6312	/*
6313	* Get the TLB entry for this page.
6314	*/
6315	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6316	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6317	if (pTlbe->uTag == uTag)
6318	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6319	else
6320	{
6321	pVCpu->iem.s.DataTlb.cTlbMisses++;
6322	PGMPTWALK Walk;
6323	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6324	if (RT_FAILURE(rc))
6325	{
6326	Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6327	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6328	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6329	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6330	# endif
6331	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6332	}
6333
6334	Assert(Walk.fSucceeded);
6335	pTlbe->uTag = uTag;
6336	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6337	pTlbe->GCPhys = Walk.GCPhys;
6338	pTlbe->pbMappingR3 = NULL;
6339	}
6340
6341	/*
6342	* Check the flags and physical revision.
6343	*/
6344	/** @todo make the caller pass these in with fAccess. */
6345	uint64_t const fNoUser = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && pVCpu->iem.s.uCpl == 3
6346	? IEMTLBE_F_PT_NO_USER : 0;
6347	uint64_t const fNoWriteNoDirty = fAccess & IEM_ACCESS_TYPE_WRITE
6348	? IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY
6349	\| ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)
6350	\|\| (pVCpu->iem.s.uCpl == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6351	? IEMTLBE_F_PT_NO_WRITE : 0)
6352	: 0;
6353	uint64_t const fNoRead = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0;
6354	uint8_t *pbMem = NULL;
6355	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PT_NO_ACCESSED \| fNoRead \| fNoWriteNoDirty \| fNoUser))
6356	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6357	# ifdef IN_RING3
6358	pbMem = pTlbe->pbMappingR3;
6359	# else
6360	pbMem = NULL;
6361	# endif
6362	else
6363	{
6364	/*
6365	* Okay, something isn't quite right or needs refreshing.
6366	*/
6367	/* Write to read only memory? */
6368	if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
6369	{
6370	Log(("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6371	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6372	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6373	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6374	# endif
6375	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6376	}
6377
6378	/* Kernel memory accessed by userland? */
6379	if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER)
6380	{
6381	Log(("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6382	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6383	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6384	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6385	# endif
6386	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6387	}
6388
6389	/* Set the dirty / access flags.
6390	ASSUMES this is set when the address is translated rather than on commit... */
6391	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6392	if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED))
6393	{
6394	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6395	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6396	AssertRC(rc2);
6397	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6398	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6399	pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED);
6400	}
6401
6402	/*
6403	* Check if the physical page info needs updating.
6404	*/
6405	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6406	# ifdef IN_RING3
6407	pbMem = pTlbe->pbMappingR3;
6408	# else
6409	pbMem = NULL;
6410	# endif
6411	else
6412	{
6413	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6414	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6415	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6416	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6417	pTlbe->pbMappingR3 = NULL;
6418	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6419	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
6420	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6421	&pbMem, &pTlbe->fFlagsAndPhysRev);
6422	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6423	# ifdef IN_RING3
6424	pTlbe->pbMappingR3 = pbMem;
6425	# endif
6426	}
6427
6428	/*
6429	* Check the physical page level access and mapping.
6430	*/
6431	if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty \| fNoRead) & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))))
6432	{ /* probably likely */ }
6433	else
6434	{
6435	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, cbMem,
6436	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6437	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6438	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6439	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6440	if (rcStrict == VINF_SUCCESS)
6441	return pbMem;
6442	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6443	}
6444	}
6445	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6446
6447	if (pbMem)
6448	{
6449	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6450	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6451	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6452	}
6453	else
6454	{
6455	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6456	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6457	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6458	if (rcStrict == VINF_SUCCESS)
6459	return pbMem;
6460	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6461	}
6462
6463	void * const pvMem = pbMem;
6464
6465	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6466	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6467	if (fAccess & IEM_ACCESS_TYPE_READ)
6468	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6469
6470	#else /* !IEM_WITH_DATA_TLB */
6471
6472
6473	RTGCPHYS GCPhysFirst;
6474	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6475	if (rcStrict == VINF_SUCCESS) { /likely/ }
6476	else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6477
6478	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6479	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6480	if (fAccess & IEM_ACCESS_TYPE_READ)
6481	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6482
6483	void *pvMem;
6484	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6485	if (rcStrict == VINF_SUCCESS)
6486	{ /* likely */ }
6487	else
6488	{
6489	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6490	if (rcStrict == VINF_SUCCESS)
6491	return pvMem;
6492	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6493	}
6494
6495	#endif /* !IEM_WITH_DATA_TLB */
6496
6497	/*
6498	* Fill in the mapping table entry.
6499	*/
6500	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6501	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6502	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6503	pVCpu->iem.s.cActiveMappings++;
6504
6505	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6506	return pvMem;
6507	}
6508
6509
6510	/**
6511	* Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
6512	*
6513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6514	* @param pvMem The mapping.
6515	* @param fAccess The kind of access.
6516	*/
6517	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6518	{
6519	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6520	AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
6521
6522	/* If it's bounce buffered, we may need to write back the buffer. */
6523	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6524	{
6525	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6526	{
6527	VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6528	if (rcStrict == VINF_SUCCESS)
6529	return;
6530	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6531	}
6532	}
6533	/* Otherwise unlock it. */
6534	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6535	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6536
6537	/* Free the entry. */
6538	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6539	Assert(pVCpu->iem.s.cActiveMappings != 0);
6540	pVCpu->iem.s.cActiveMappings--;
6541	}
6542
6543	#endif /* IEM_WITH_SETJMP */
6544
6545	#ifndef IN_RING3
6546	/**
6547	* Commits the guest memory if bounce buffered and unmaps it, if any bounce
6548	* buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
6549	*
6550	* Allows the instruction to be completed and retired, while the IEM user will
6551	* return to ring-3 immediately afterwards and do the postponed writes there.
6552	*
6553	* @returns VBox status code (no strict statuses). Caller must check
6554	* VMCPU_FF_IEM before repeating string instructions and similar stuff.
6555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6556	* @param pvMem The mapping.
6557	* @param fAccess The kind of access.
6558	*/
6559	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6560	{
6561	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6562	AssertReturn(iMemMap >= 0, iMemMap);
6563
6564	/* If it's bounce buffered, we may need to write back the buffer. */
6565	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6566	{
6567	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6568	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /fPostponeFail/);
6569	}
6570	/* Otherwise unlock it. */
6571	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6572	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6573
6574	/* Free the entry. */
6575	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6576	Assert(pVCpu->iem.s.cActiveMappings != 0);
6577	pVCpu->iem.s.cActiveMappings--;
6578	return VINF_SUCCESS;
6579	}
6580	#endif
6581
6582
6583	/**
6584	* Rollbacks mappings, releasing page locks and such.
6585	*
6586	* The caller shall only call this after checking cActiveMappings.
6587	*
6588	* @returns Strict VBox status code to pass up.
6589	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6590	*/
6591	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT
6592	{
6593	Assert(pVCpu->iem.s.cActiveMappings > 0);
6594
6595	uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
6596	while (iMemMap-- > 0)
6597	{
6598	uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
6599	if (fAccess != IEM_ACCESS_INVALID)
6600	{
6601	AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
6602	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6603	if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_NOT_LOCKED)))
6604	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6605	AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
6606	("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
6607	iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
6608	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
6609	pVCpu->iem.s.cActiveMappings--;
6610	}
6611	}
6612	}
6613
6614
6615	/**
6616	* Fetches a data byte.
6617	*
6618	* @returns Strict VBox status code.
6619	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6620	* @param pu8Dst Where to return the byte.
6621	* @param iSegReg The index of the segment register to use for
6622	* this access. The base and limits are checked.
6623	* @param GCPtrMem The address of the guest memory.
6624	*/
6625	VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6626	{
6627	/* The lazy approach for now... */
6628	uint8_t const *pu8Src;
6629	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6630	if (rc == VINF_SUCCESS)
6631	{
6632	pu8Dst = pu8Src;
6633	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6634	}
6635	return rc;
6636	}
6637
6638
6639	#ifdef IEM_WITH_SETJMP
6640	/**
6641	* Fetches a data byte, longjmp on error.
6642	*
6643	* @returns The byte.
6644	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6645	* @param iSegReg The index of the segment register to use for
6646	* this access. The base and limits are checked.
6647	* @param GCPtrMem The address of the guest memory.
6648	*/
6649	uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6650	{
6651	/* The lazy approach for now... */
6652	uint8_t const pu8Src = (uint8_t const )iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6653	uint8_t const bRet = *pu8Src;
6654	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6655	return bRet;
6656	}
6657	#endif /* IEM_WITH_SETJMP */
6658
6659
6660	/**
6661	* Fetches a data word.
6662	*
6663	* @returns Strict VBox status code.
6664	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6665	* @param pu16Dst Where to return the word.
6666	* @param iSegReg The index of the segment register to use for
6667	* this access. The base and limits are checked.
6668	* @param GCPtrMem The address of the guest memory.
6669	*/
6670	VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6671	{
6672	/* The lazy approach for now... */
6673	uint16_t const *pu16Src;
6674	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem,
6675	IEM_ACCESS_DATA_R, sizeof(*pu16Src) - 1);
6676	if (rc == VINF_SUCCESS)
6677	{
6678	pu16Dst = pu16Src;
6679	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6680	}
6681	return rc;
6682	}
6683
6684
6685	#ifdef IEM_WITH_SETJMP
6686	/**
6687	* Fetches a data word, longjmp on error.
6688	*
6689	* @returns The word
6690	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6691	* @param iSegReg The index of the segment register to use for
6692	* this access. The base and limits are checked.
6693	* @param GCPtrMem The address of the guest memory.
6694	*/
6695	uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6696	{
6697	/* The lazy approach for now... */
6698	uint16_t const pu16Src = (uint16_t const )iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6699	sizeof(*pu16Src) - 1);
6700	uint16_t const u16Ret = *pu16Src;
6701	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6702	return u16Ret;
6703	}
6704	#endif
6705
6706
6707	/**
6708	* Fetches a data dword.
6709	*
6710	* @returns Strict VBox status code.
6711	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6712	* @param pu32Dst Where to return the dword.
6713	* @param iSegReg The index of the segment register to use for
6714	* this access. The base and limits are checked.
6715	* @param GCPtrMem The address of the guest memory.
6716	*/
6717	VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6718	{
6719	/* The lazy approach for now... */
6720	uint32_t const *pu32Src;
6721	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem,
6722	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6723	if (rc == VINF_SUCCESS)
6724	{
6725	pu32Dst = pu32Src;
6726	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6727	}
6728	return rc;
6729	}
6730
6731
6732	/**
6733	* Fetches a data dword and zero extends it to a qword.
6734	*
6735	* @returns Strict VBox status code.
6736	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6737	* @param pu64Dst Where to return the qword.
6738	* @param iSegReg The index of the segment register to use for
6739	* this access. The base and limits are checked.
6740	* @param GCPtrMem The address of the guest memory.
6741	*/
6742	VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6743	{
6744	/* The lazy approach for now... */
6745	uint32_t const *pu32Src;
6746	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem,
6747	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6748	if (rc == VINF_SUCCESS)
6749	{
6750	pu64Dst = pu32Src;
6751	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6752	}
6753	return rc;
6754	}
6755
6756
6757	#ifdef IEM_WITH_SETJMP
6758
6759	/**
6760	* Fetches a data dword, longjmp on error, fallback/safe version.
6761	*
6762	* @returns The dword
6763	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6764	* @param iSegReg The index of the segment register to use for
6765	* this access. The base and limits are checked.
6766	* @param GCPtrMem The address of the guest memory.
6767	*/
6768	uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6769	{
6770	uint32_t const pu32Src = (uint32_t const )iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6771	sizeof(*pu32Src) - 1);
6772	uint32_t const u32Ret = *pu32Src;
6773	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6774	return u32Ret;
6775	}
6776
6777
6778	/**
6779	* Fetches a data dword, longjmp on error.
6780	*
6781	* @returns The dword
6782	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6783	* @param iSegReg The index of the segment register to use for
6784	* this access. The base and limits are checked.
6785	* @param GCPtrMem The address of the guest memory.
6786	*/
6787	uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6788	{
6789	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
6790	/*
6791	* Convert from segmented to flat address and check that it doesn't cross a page boundrary.
6792	*/
6793	RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
6794	if (RT_LIKELY((GCPtrEff & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(uint32_t)))
6795	{
6796	/*
6797	* TLB lookup.
6798	*/
6799	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrEff);
6800	PIEMTLBENTRY pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6801	if (pTlbe->uTag == uTag)
6802	{
6803	/*
6804	* Check TLB page table level access flags.
6805	*/
6806	uint64_t const fNoUser = pVCpu->iem.s.uCpl == 3 ? IEMTLBE_F_PT_NO_USER : 0;
6807	if ( (pTlbe->fFlagsAndPhysRev & ( IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PG_NO_READ
6808	\| IEMTLBE_F_PT_NO_ACCESSED \| IEMTLBE_F_NO_MAPPINGR3 \| fNoUser))
6809	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6810	{
6811	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6812
6813	/*
6814	* Alignment check:
6815	*/
6816	/** @todo check priority \#AC vs \#PF */
6817	if ( !(GCPtrEff & (sizeof(uint32_t) - 1))
6818	\|\| !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
6819	\|\| !pVCpu->cpum.GstCtx.eflags.Bits.u1AC
6820	\|\| pVCpu->iem.s.uCpl != 3)
6821	{
6822	/*
6823	* Fetch and return the dword
6824	*/
6825	Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
6826	Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
6827	return (uint32_t const )&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
6828	}
6829	Log10(("iemMemFetchDataU32Jmp: Raising #AC for %RGv\n", GCPtrEff));
6830	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6831	}
6832	}
6833	}
6834
6835	/* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
6836	outdated page pointer, or other troubles. */
6837	Log10(("iemMemFetchDataU32Jmp: %u:%RGv fallback\n", iSegReg, GCPtrMem));
6838	return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
6839
6840	# else
6841	uint32_t const pu32Src = (uint32_t const )iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem,
6842	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6843	uint32_t const u32Ret = *pu32Src;
6844	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6845	return u32Ret;
6846	# endif
6847	}
6848	#endif
6849
6850
6851	#ifdef SOME_UNUSED_FUNCTION
6852	/**
6853	* Fetches a data dword and sign extends it to a qword.
6854	*
6855	* @returns Strict VBox status code.
6856	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6857	* @param pu64Dst Where to return the sign extended value.
6858	* @param iSegReg The index of the segment register to use for
6859	* this access. The base and limits are checked.
6860	* @param GCPtrMem The address of the guest memory.
6861	*/
6862	VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6863	{
6864	/* The lazy approach for now... */
6865	int32_t const *pi32Src;
6866	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem,
6867	IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1);
6868	if (rc == VINF_SUCCESS)
6869	{
6870	pu64Dst = pi32Src;
6871	rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
6872	}
6873	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
6874	else
6875	*pu64Dst = 0;
6876	#endif
6877	return rc;
6878	}
6879	#endif
6880
6881
6882	/**
6883	* Fetches a data qword.
6884	*
6885	* @returns Strict VBox status code.
6886	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6887	* @param pu64Dst Where to return the qword.
6888	* @param iSegReg The index of the segment register to use for
6889	* this access. The base and limits are checked.
6890	* @param GCPtrMem The address of the guest memory.
6891	*/
6892	VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6893	{
6894	/* The lazy approach for now... */
6895	uint64_t const *pu64Src;
6896	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem,
6897	IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6898	if (rc == VINF_SUCCESS)
6899	{
6900	pu64Dst = pu64Src;
6901	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6902	}
6903	return rc;
6904	}
6905
6906
6907	#ifdef IEM_WITH_SETJMP
6908	/**
6909	* Fetches a data qword, longjmp on error.
6910	*
6911	* @returns The qword.
6912	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6913	* @param iSegReg The index of the segment register to use for
6914	* this access. The base and limits are checked.
6915	* @param GCPtrMem The address of the guest memory.
6916	*/
6917	uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6918	{
6919	/* The lazy approach for now... */
6920	uint64_t const pu64Src = (uint64_t const )iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem,
6921	IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6922	uint64_t const u64Ret = *pu64Src;
6923	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6924	return u64Ret;
6925	}
6926	#endif
6927
6928
6929	/**
6930	* Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
6931	*
6932	* @returns Strict VBox status code.
6933	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6934	* @param pu64Dst Where to return the qword.
6935	* @param iSegReg The index of the segment register to use for
6936	* this access. The base and limits are checked.
6937	* @param GCPtrMem The address of the guest memory.
6938	*/
6939	VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6940	{
6941	/* The lazy approach for now... */
6942	uint64_t const *pu64Src;
6943	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem,
6944	IEM_ACCESS_DATA_R, 15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
6945	if (rc == VINF_SUCCESS)
6946	{
6947	pu64Dst = pu64Src;
6948	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6949	}
6950	return rc;
6951	}
6952
6953
6954	#ifdef IEM_WITH_SETJMP
6955	/**
6956	* Fetches a data qword, longjmp on error.
6957	*
6958	* @returns The qword.
6959	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6960	* @param iSegReg The index of the segment register to use for
6961	* this access. The base and limits are checked.
6962	* @param GCPtrMem The address of the guest memory.
6963	*/
6964	uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6965	{
6966	/* The lazy approach for now... */
6967	uint64_t const pu64Src = (uint64_t const )iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6968	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
6969	uint64_t const u64Ret = *pu64Src;
6970	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6971	return u64Ret;
6972	}
6973	#endif
6974
6975
6976	/**
6977	* Fetches a data tword.
6978	*
6979	* @returns Strict VBox status code.
6980	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6981	* @param pr80Dst Where to return the tword.
6982	* @param iSegReg The index of the segment register to use for
6983	* this access. The base and limits are checked.
6984	* @param GCPtrMem The address of the guest memory.
6985	*/
6986	VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6987	{
6988	/* The lazy approach for now... */
6989	PCRTFLOAT80U pr80Src;
6990	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pr80Src, sizeof(pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
6991	if (rc == VINF_SUCCESS)
6992	{
6993	pr80Dst = pr80Src;
6994	rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
6995	}
6996	return rc;
6997	}
6998
6999
7000	#ifdef IEM_WITH_SETJMP
7001	/**
7002	* Fetches a data tword, longjmp on error.
7003	*
7004	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7005	* @param pr80Dst Where to return the tword.
7006	* @param iSegReg The index of the segment register to use for
7007	* this access. The base and limits are checked.
7008	* @param GCPtrMem The address of the guest memory.
7009	*/
7010	void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7011	{
7012	/* The lazy approach for now... */
7013	PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
7014	pr80Dst = pr80Src;
7015	iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7016	}
7017	#endif
7018
7019
7020	/**
7021	* Fetches a data decimal tword.
7022	*
7023	* @returns Strict VBox status code.
7024	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7025	* @param pd80Dst Where to return the tword.
7026	* @param iSegReg The index of the segment register to use for
7027	* this access. The base and limits are checked.
7028	* @param GCPtrMem The address of the guest memory.
7029	*/
7030	VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7031	{
7032	/* The lazy approach for now... */
7033	PCRTPBCD80U pd80Src;
7034	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pd80Src, sizeof(pd80Src), iSegReg, GCPtrMem,
7035	IEM_ACCESS_DATA_R, 7 /** @todo FBLD alignment check */);
7036	if (rc == VINF_SUCCESS)
7037	{
7038	pd80Dst = pd80Src;
7039	rc = iemMemCommitAndUnmap(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7040	}
7041	return rc;
7042	}
7043
7044
7045	#ifdef IEM_WITH_SETJMP
7046	/**
7047	* Fetches a data decimal tword, longjmp on error.
7048	*
7049	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7050	* @param pd80Dst Where to return the tword.
7051	* @param iSegReg The index of the segment register to use for
7052	* this access. The base and limits are checked.
7053	* @param GCPtrMem The address of the guest memory.
7054	*/
7055	void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7056	{
7057	/* The lazy approach for now... */
7058	PCRTPBCD80U pd80Src = (PCRTPBCD80U)iemMemMapJmp(pVCpu, sizeof(*pd80Src), iSegReg, GCPtrMem,
7059	IEM_ACCESS_DATA_R, 7 /** @todo FBSTP alignment check */);
7060	pd80Dst = pd80Src;
7061	iemMemCommitAndUnmapJmp(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7062	}
7063	#endif
7064
7065
7066	/**
7067	* Fetches a data dqword (double qword), generally SSE related.
7068	*
7069	* @returns Strict VBox status code.
7070	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7071	* @param pu128Dst Where to return the qword.
7072	* @param iSegReg The index of the segment register to use for
7073	* this access. The base and limits are checked.
7074	* @param GCPtrMem The address of the guest memory.
7075	*/
7076	VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7077	{
7078	/* The lazy approach for now... */
7079	PCRTUINT128U pu128Src;
7080	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem,
7081	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7082	if (rc == VINF_SUCCESS)
7083	{
7084	pu128Dst->au64[0] = pu128Src->au64[0];
7085	pu128Dst->au64[1] = pu128Src->au64[1];
7086	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7087	}
7088	return rc;
7089	}
7090
7091
7092	#ifdef IEM_WITH_SETJMP
7093	/**
7094	* Fetches a data dqword (double qword), generally SSE related.
7095	*
7096	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7097	* @param pu128Dst Where to return the qword.
7098	* @param iSegReg The index of the segment register to use for
7099	* this access. The base and limits are checked.
7100	* @param GCPtrMem The address of the guest memory.
7101	*/
7102	void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7103	{
7104	/* The lazy approach for now... */
7105	PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem,
7106	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7107	pu128Dst->au64[0] = pu128Src->au64[0];
7108	pu128Dst->au64[1] = pu128Src->au64[1];
7109	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7110	}
7111	#endif
7112
7113
7114	/**
7115	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7116	* related.
7117	*
7118	* Raises \#GP(0) if not aligned.
7119	*
7120	* @returns Strict VBox status code.
7121	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7122	* @param pu128Dst Where to return the qword.
7123	* @param iSegReg The index of the segment register to use for
7124	* this access. The base and limits are checked.
7125	* @param GCPtrMem The address of the guest memory.
7126	*/
7127	VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7128	{
7129	/* The lazy approach for now... */
7130	PCRTUINT128U pu128Src;
7131	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem,
7132	IEM_ACCESS_DATA_R, (sizeof(*pu128Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7133	if (rc == VINF_SUCCESS)
7134	{
7135	pu128Dst->au64[0] = pu128Src->au64[0];
7136	pu128Dst->au64[1] = pu128Src->au64[1];
7137	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7138	}
7139	return rc;
7140	}
7141
7142
7143	#ifdef IEM_WITH_SETJMP
7144	/**
7145	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7146	* related, longjmp on error.
7147	*
7148	* Raises \#GP(0) if not aligned.
7149	*
7150	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7151	* @param pu128Dst Where to return the qword.
7152	* @param iSegReg The index of the segment register to use for
7153	* this access. The base and limits are checked.
7154	* @param GCPtrMem The address of the guest memory.
7155	*/
7156	void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7157	{
7158	/* The lazy approach for now... */
7159	PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7160	(sizeof(*pu128Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7161	pu128Dst->au64[0] = pu128Src->au64[0];
7162	pu128Dst->au64[1] = pu128Src->au64[1];
7163	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7164	}
7165	#endif
7166
7167
7168	/**
7169	* Fetches a data oword (octo word), generally AVX related.
7170	*
7171	* @returns Strict VBox status code.
7172	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7173	* @param pu256Dst Where to return the qword.
7174	* @param iSegReg The index of the segment register to use for
7175	* this access. The base and limits are checked.
7176	* @param GCPtrMem The address of the guest memory.
7177	*/
7178	VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7179	{
7180	/* The lazy approach for now... */
7181	PCRTUINT256U pu256Src;
7182	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, sizeof(pu256Src), iSegReg, GCPtrMem,
7183	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7184	if (rc == VINF_SUCCESS)
7185	{
7186	pu256Dst->au64[0] = pu256Src->au64[0];
7187	pu256Dst->au64[1] = pu256Src->au64[1];
7188	pu256Dst->au64[2] = pu256Src->au64[2];
7189	pu256Dst->au64[3] = pu256Src->au64[3];
7190	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7191	}
7192	return rc;
7193	}
7194
7195
7196	#ifdef IEM_WITH_SETJMP
7197	/**
7198	* Fetches a data oword (octo word), generally AVX related.
7199	*
7200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7201	* @param pu256Dst Where to return the qword.
7202	* @param iSegReg The index of the segment register to use for
7203	* this access. The base and limits are checked.
7204	* @param GCPtrMem The address of the guest memory.
7205	*/
7206	void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7207	{
7208	/* The lazy approach for now... */
7209	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem,
7210	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7211	pu256Dst->au64[0] = pu256Src->au64[0];
7212	pu256Dst->au64[1] = pu256Src->au64[1];
7213	pu256Dst->au64[2] = pu256Src->au64[2];
7214	pu256Dst->au64[3] = pu256Src->au64[3];
7215	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7216	}
7217	#endif
7218
7219
7220	/**
7221	* Fetches a data oword (octo word) at an aligned address, generally AVX
7222	* related.
7223	*
7224	* Raises \#GP(0) if not aligned.
7225	*
7226	* @returns Strict VBox status code.
7227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7228	* @param pu256Dst Where to return the qword.
7229	* @param iSegReg The index of the segment register to use for
7230	* this access. The base and limits are checked.
7231	* @param GCPtrMem The address of the guest memory.
7232	*/
7233	VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7234	{
7235	/* The lazy approach for now... */
7236	PCRTUINT256U pu256Src;
7237	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, sizeof(pu256Src), iSegReg, GCPtrMem,
7238	IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7239	if (rc == VINF_SUCCESS)
7240	{
7241	pu256Dst->au64[0] = pu256Src->au64[0];
7242	pu256Dst->au64[1] = pu256Src->au64[1];
7243	pu256Dst->au64[2] = pu256Src->au64[2];
7244	pu256Dst->au64[3] = pu256Src->au64[3];
7245	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7246	}
7247	return rc;
7248	}
7249
7250
7251	#ifdef IEM_WITH_SETJMP
7252	/**
7253	* Fetches a data oword (octo word) at an aligned address, generally AVX
7254	* related, longjmp on error.
7255	*
7256	* Raises \#GP(0) if not aligned.
7257	*
7258	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7259	* @param pu256Dst Where to return the qword.
7260	* @param iSegReg The index of the segment register to use for
7261	* this access. The base and limits are checked.
7262	* @param GCPtrMem The address of the guest memory.
7263	*/
7264	void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7265	{
7266	/* The lazy approach for now... */
7267	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7268	(sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7269	pu256Dst->au64[0] = pu256Src->au64[0];
7270	pu256Dst->au64[1] = pu256Src->au64[1];
7271	pu256Dst->au64[2] = pu256Src->au64[2];
7272	pu256Dst->au64[3] = pu256Src->au64[3];
7273	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7274	}
7275	#endif
7276
7277
7278
7279	/**
7280	* Fetches a descriptor register (lgdt, lidt).
7281	*
7282	* @returns Strict VBox status code.
7283	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7284	* @param pcbLimit Where to return the limit.
7285	* @param pGCPtrBase Where to return the base.
7286	* @param iSegReg The index of the segment register to use for
7287	* this access. The base and limits are checked.
7288	* @param GCPtrMem The address of the guest memory.
7289	* @param enmOpSize The effective operand size.
7290	*/
7291	VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
7292	RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT
7293	{
7294	/*
7295	* Just like SIDT and SGDT, the LIDT and LGDT instructions are a
7296	* little special:
7297	* - The two reads are done separately.
7298	* - Operand size override works in 16-bit and 32-bit code, but 64-bit.
7299	* - We suspect the 386 to actually commit the limit before the base in
7300	* some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
7301	* don't try emulate this eccentric behavior, because it's not well
7302	* enough understood and rather hard to trigger.
7303	* - The 486 seems to do a dword limit read when the operand size is 32-bit.
7304	*/
7305	VBOXSTRICTRC rcStrict;
7306	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7307	{
7308	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7309	if (rcStrict == VINF_SUCCESS)
7310	rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
7311	}
7312	else
7313	{
7314	uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
7315	if (enmOpSize == IEMMODE_32BIT)
7316	{
7317	if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
7318	{
7319	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7320	if (rcStrict == VINF_SUCCESS)
7321	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7322	}
7323	else
7324	{
7325	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
7326	if (rcStrict == VINF_SUCCESS)
7327	{
7328	*pcbLimit = (uint16_t)uTmp;
7329	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7330	}
7331	}
7332	if (rcStrict == VINF_SUCCESS)
7333	*pGCPtrBase = uTmp;
7334	}
7335	else
7336	{
7337	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7338	if (rcStrict == VINF_SUCCESS)
7339	{
7340	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7341	if (rcStrict == VINF_SUCCESS)
7342	*pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
7343	}
7344	}
7345	}
7346	return rcStrict;
7347	}
7348
7349
7350
7351	/**
7352	* Stores a data byte.
7353	*
7354	* @returns Strict VBox status code.
7355	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7356	* @param iSegReg The index of the segment register to use for
7357	* this access. The base and limits are checked.
7358	* @param GCPtrMem The address of the guest memory.
7359	* @param u8Value The value to store.
7360	*/
7361	VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7362	{
7363	/* The lazy approach for now... */
7364	uint8_t *pu8Dst;
7365	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7366	if (rc == VINF_SUCCESS)
7367	{
7368	*pu8Dst = u8Value;
7369	rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7370	}
7371	return rc;
7372	}
7373
7374
7375	#ifdef IEM_WITH_SETJMP
7376	/**
7377	* Stores a data byte, longjmp on error.
7378	*
7379	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7380	* @param iSegReg The index of the segment register to use for
7381	* this access. The base and limits are checked.
7382	* @param GCPtrMem The address of the guest memory.
7383	* @param u8Value The value to store.
7384	*/
7385	void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7386	{
7387	/* The lazy approach for now... */
7388	uint8_t pu8Dst = (uint8_t )iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7389	*pu8Dst = u8Value;
7390	iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7391	}
7392	#endif
7393
7394
7395	/**
7396	* Stores a data word.
7397	*
7398	* @returns Strict VBox status code.
7399	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7400	* @param iSegReg The index of the segment register to use for
7401	* this access. The base and limits are checked.
7402	* @param GCPtrMem The address of the guest memory.
7403	* @param u16Value The value to store.
7404	*/
7405	VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7406	{
7407	/* The lazy approach for now... */
7408	uint16_t *pu16Dst;
7409	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem,
7410	IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7411	if (rc == VINF_SUCCESS)
7412	{
7413	*pu16Dst = u16Value;
7414	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7415	}
7416	return rc;
7417	}
7418
7419
7420	#ifdef IEM_WITH_SETJMP
7421	/**
7422	* Stores a data word, longjmp on error.
7423	*
7424	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7425	* @param iSegReg The index of the segment register to use for
7426	* this access. The base and limits are checked.
7427	* @param GCPtrMem The address of the guest memory.
7428	* @param u16Value The value to store.
7429	*/
7430	void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7431	{
7432	/* The lazy approach for now... */
7433	uint16_t pu16Dst = (uint16_t )iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem,
7434	IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7435	*pu16Dst = u16Value;
7436	iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7437	}
7438	#endif
7439
7440
7441	/**
7442	* Stores a data dword.
7443	*
7444	* @returns Strict VBox status code.
7445	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7446	* @param iSegReg The index of the segment register to use for
7447	* this access. The base and limits are checked.
7448	* @param GCPtrMem The address of the guest memory.
7449	* @param u32Value The value to store.
7450	*/
7451	VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7452	{
7453	/* The lazy approach for now... */
7454	uint32_t *pu32Dst;
7455	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem,
7456	IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7457	if (rc == VINF_SUCCESS)
7458	{
7459	*pu32Dst = u32Value;
7460	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7461	}
7462	return rc;
7463	}
7464
7465
7466	#ifdef IEM_WITH_SETJMP
7467	/**
7468	* Stores a data dword.
7469	*
7470	* @returns Strict VBox status code.
7471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7472	* @param iSegReg The index of the segment register to use for
7473	* this access. The base and limits are checked.
7474	* @param GCPtrMem The address of the guest memory.
7475	* @param u32Value The value to store.
7476	*/
7477	void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7478	{
7479	/* The lazy approach for now... */
7480	uint32_t pu32Dst = (uint32_t )iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem,
7481	IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7482	*pu32Dst = u32Value;
7483	iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7484	}
7485	#endif
7486
7487
7488	/**
7489	* Stores a data qword.
7490	*
7491	* @returns Strict VBox status code.
7492	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7493	* @param iSegReg The index of the segment register to use for
7494	* this access. The base and limits are checked.
7495	* @param GCPtrMem The address of the guest memory.
7496	* @param u64Value The value to store.
7497	*/
7498	VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7499	{
7500	/* The lazy approach for now... */
7501	uint64_t *pu64Dst;
7502	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem,
7503	IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7504	if (rc == VINF_SUCCESS)
7505	{
7506	*pu64Dst = u64Value;
7507	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7508	}
7509	return rc;
7510	}
7511
7512
7513	#ifdef IEM_WITH_SETJMP
7514	/**
7515	* Stores a data qword, longjmp on error.
7516	*
7517	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7518	* @param iSegReg The index of the segment register to use for
7519	* this access. The base and limits are checked.
7520	* @param GCPtrMem The address of the guest memory.
7521	* @param u64Value The value to store.
7522	*/
7523	void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7524	{
7525	/* The lazy approach for now... */
7526	uint64_t pu64Dst = (uint64_t )iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem,
7527	IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7528	*pu64Dst = u64Value;
7529	iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7530	}
7531	#endif
7532
7533
7534	/**
7535	* Stores a data dqword.
7536	*
7537	* @returns Strict VBox status code.
7538	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7539	* @param iSegReg The index of the segment register to use for
7540	* this access. The base and limits are checked.
7541	* @param GCPtrMem The address of the guest memory.
7542	* @param u128Value The value to store.
7543	*/
7544	VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7545	{
7546	/* The lazy approach for now... */
7547	PRTUINT128U pu128Dst;
7548	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem,
7549	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7550	if (rc == VINF_SUCCESS)
7551	{
7552	pu128Dst->au64[0] = u128Value.au64[0];
7553	pu128Dst->au64[1] = u128Value.au64[1];
7554	rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7555	}
7556	return rc;
7557	}
7558
7559
7560	#ifdef IEM_WITH_SETJMP
7561	/**
7562	* Stores a data dqword, longjmp on error.
7563	*
7564	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7565	* @param iSegReg The index of the segment register to use for
7566	* this access. The base and limits are checked.
7567	* @param GCPtrMem The address of the guest memory.
7568	* @param u128Value The value to store.
7569	*/
7570	void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7571	{
7572	/* The lazy approach for now... */
7573	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem,
7574	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7575	pu128Dst->au64[0] = u128Value.au64[0];
7576	pu128Dst->au64[1] = u128Value.au64[1];
7577	iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7578	}
7579	#endif
7580
7581
7582	/**
7583	* Stores a data dqword, SSE aligned.
7584	*
7585	* @returns Strict VBox status code.
7586	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7587	* @param iSegReg The index of the segment register to use for
7588	* this access. The base and limits are checked.
7589	* @param GCPtrMem The address of the guest memory.
7590	* @param u128Value The value to store.
7591	*/
7592	VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7593	{
7594	/* The lazy approach for now... */
7595	PRTUINT128U pu128Dst;
7596	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7597	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7598	if (rc == VINF_SUCCESS)
7599	{
7600	pu128Dst->au64[0] = u128Value.au64[0];
7601	pu128Dst->au64[1] = u128Value.au64[1];
7602	rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7603	}
7604	return rc;
7605	}
7606
7607
7608	#ifdef IEM_WITH_SETJMP
7609	/**
7610	* Stores a data dqword, SSE aligned.
7611	*
7612	* @returns Strict VBox status code.
7613	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7614	* @param iSegReg The index of the segment register to use for
7615	* this access. The base and limits are checked.
7616	* @param GCPtrMem The address of the guest memory.
7617	* @param u128Value The value to store.
7618	*/
7619	void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7620	{
7621	/* The lazy approach for now... */
7622	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7623	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7624	pu128Dst->au64[0] = u128Value.au64[0];
7625	pu128Dst->au64[1] = u128Value.au64[1];
7626	iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7627	}
7628	#endif
7629
7630
7631	/**
7632	* Stores a data dqword.
7633	*
7634	* @returns Strict VBox status code.
7635	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7636	* @param iSegReg The index of the segment register to use for
7637	* this access. The base and limits are checked.
7638	* @param GCPtrMem The address of the guest memory.
7639	* @param pu256Value Pointer to the value to store.
7640	*/
7641	VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7642	{
7643	/* The lazy approach for now... */
7644	PRTUINT256U pu256Dst;
7645	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, sizeof(pu256Dst), iSegReg, GCPtrMem,
7646	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7647	if (rc == VINF_SUCCESS)
7648	{
7649	pu256Dst->au64[0] = pu256Value->au64[0];
7650	pu256Dst->au64[1] = pu256Value->au64[1];
7651	pu256Dst->au64[2] = pu256Value->au64[2];
7652	pu256Dst->au64[3] = pu256Value->au64[3];
7653	rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7654	}
7655	return rc;
7656	}
7657
7658
7659	#ifdef IEM_WITH_SETJMP
7660	/**
7661	* Stores a data dqword, longjmp on error.
7662	*
7663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7664	* @param iSegReg The index of the segment register to use for
7665	* this access. The base and limits are checked.
7666	* @param GCPtrMem The address of the guest memory.
7667	* @param pu256Value Pointer to the value to store.
7668	*/
7669	void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7670	{
7671	/* The lazy approach for now... */
7672	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7673	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7674	pu256Dst->au64[0] = pu256Value->au64[0];
7675	pu256Dst->au64[1] = pu256Value->au64[1];
7676	pu256Dst->au64[2] = pu256Value->au64[2];
7677	pu256Dst->au64[3] = pu256Value->au64[3];
7678	iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7679	}
7680	#endif
7681
7682
7683	/**
7684	* Stores a data dqword, AVX \#GP(0) aligned.
7685	*
7686	* @returns Strict VBox status code.
7687	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7688	* @param iSegReg The index of the segment register to use for
7689	* this access. The base and limits are checked.
7690	* @param GCPtrMem The address of the guest memory.
7691	* @param pu256Value Pointer to the value to store.
7692	*/
7693	VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7694	{
7695	/* The lazy approach for now... */
7696	PRTUINT256U pu256Dst;
7697	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, sizeof(pu256Dst), iSegReg, GCPtrMem,
7698	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7699	if (rc == VINF_SUCCESS)
7700	{
7701	pu256Dst->au64[0] = pu256Value->au64[0];
7702	pu256Dst->au64[1] = pu256Value->au64[1];
7703	pu256Dst->au64[2] = pu256Value->au64[2];
7704	pu256Dst->au64[3] = pu256Value->au64[3];
7705	rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7706	}
7707	return rc;
7708	}
7709
7710
7711	#ifdef IEM_WITH_SETJMP
7712	/**
7713	* Stores a data dqword, AVX aligned.
7714	*
7715	* @returns Strict VBox status code.
7716	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7717	* @param iSegReg The index of the segment register to use for
7718	* this access. The base and limits are checked.
7719	* @param GCPtrMem The address of the guest memory.
7720	* @param pu256Value Pointer to the value to store.
7721	*/
7722	void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7723	{
7724	/* The lazy approach for now... */
7725	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7726	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7727	pu256Dst->au64[0] = pu256Value->au64[0];
7728	pu256Dst->au64[1] = pu256Value->au64[1];
7729	pu256Dst->au64[2] = pu256Value->au64[2];
7730	pu256Dst->au64[3] = pu256Value->au64[3];
7731	iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7732	}
7733	#endif
7734
7735
7736	/**
7737	* Stores a descriptor register (sgdt, sidt).
7738	*
7739	* @returns Strict VBox status code.
7740	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7741	* @param cbLimit The limit.
7742	* @param GCPtrBase The base address.
7743	* @param iSegReg The index of the segment register to use for
7744	* this access. The base and limits are checked.
7745	* @param GCPtrMem The address of the guest memory.
7746	*/
7747	VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7748	{
7749	/*
7750	* The SIDT and SGDT instructions actually stores the data using two
7751	* independent writes (see bs3CpuBasic2_sidt_sgdt_One). The instructions
7752	* does not respond to opsize prefixes.
7753	*/
7754	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
7755	if (rcStrict == VINF_SUCCESS)
7756	{
7757	if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
7758	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
7759	IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
7760	? (uint32_t)GCPtrBase \| UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
7761	else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
7762	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
7763	else
7764	rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
7765	}
7766	return rcStrict;
7767	}
7768
7769
7770	/**
7771	* Pushes a word onto the stack.
7772	*
7773	* @returns Strict VBox status code.
7774	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7775	* @param u16Value The value to push.
7776	*/
7777	VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT
7778	{
7779	/* Increment the stack pointer. */
7780	uint64_t uNewRsp;
7781	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
7782
7783	/* Write the word the lazy way. */
7784	uint16_t *pu16Dst;
7785	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop,
7786	IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
7787	if (rc == VINF_SUCCESS)
7788	{
7789	*pu16Dst = u16Value;
7790	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
7791	}
7792
7793	/* Commit the new RSP value unless we an access handler made trouble. */
7794	if (rc == VINF_SUCCESS)
7795	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7796
7797	return rc;
7798	}
7799
7800
7801	/**
7802	* Pushes a dword onto the stack.
7803	*
7804	* @returns Strict VBox status code.
7805	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7806	* @param u32Value The value to push.
7807	*/
7808	VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7809	{
7810	/* Increment the stack pointer. */
7811	uint64_t uNewRsp;
7812	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7813
7814	/* Write the dword the lazy way. */
7815	uint32_t *pu32Dst;
7816	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop,
7817	IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
7818	if (rc == VINF_SUCCESS)
7819	{
7820	*pu32Dst = u32Value;
7821	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
7822	}
7823
7824	/* Commit the new RSP value unless we an access handler made trouble. */
7825	if (rc == VINF_SUCCESS)
7826	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7827
7828	return rc;
7829	}
7830
7831
7832	/**
7833	* Pushes a dword segment register value onto the stack.
7834	*
7835	* @returns Strict VBox status code.
7836	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7837	* @param u32Value The value to push.
7838	*/
7839	VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7840	{
7841	/* Increment the stack pointer. */
7842	uint64_t uNewRsp;
7843	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7844
7845	/* The intel docs talks about zero extending the selector register
7846	value. My actual intel CPU here might be zero extending the value
7847	but it still only writes the lower word... */
7848	/** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
7849	* happens when crossing an electric page boundrary, is the high word checked
7850	* for write accessibility or not? Probably it is. What about segment limits?
7851	* It appears this behavior is also shared with trap error codes.
7852	*
7853	* Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
7854	* ancient hardware when it actually did change. */
7855	uint16_t *pu16Dst;
7856	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop,
7857	IEM_ACCESS_STACK_RW, sizeof(pu16Dst) - 1); /* @todo 2 or 4 alignment check for PUSH SS? */
7858	if (rc == VINF_SUCCESS)
7859	{
7860	*pu16Dst = (uint16_t)u32Value;
7861	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
7862	}
7863
7864	/* Commit the new RSP value unless we an access handler made trouble. */
7865	if (rc == VINF_SUCCESS)
7866	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7867
7868	return rc;
7869	}
7870
7871
7872	/**
7873	* Pushes a qword onto the stack.
7874	*
7875	* @returns Strict VBox status code.
7876	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7877	* @param u64Value The value to push.
7878	*/
7879	VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT
7880	{
7881	/* Increment the stack pointer. */
7882	uint64_t uNewRsp;
7883	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
7884
7885	/* Write the word the lazy way. */
7886	uint64_t *pu64Dst;
7887	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop,
7888	IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
7889	if (rc == VINF_SUCCESS)
7890	{
7891	*pu64Dst = u64Value;
7892	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
7893	}
7894
7895	/* Commit the new RSP value unless we an access handler made trouble. */
7896	if (rc == VINF_SUCCESS)
7897	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7898
7899	return rc;
7900	}
7901
7902
7903	/**
7904	* Pops a word from the stack.
7905	*
7906	* @returns Strict VBox status code.
7907	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7908	* @param pu16Value Where to store the popped value.
7909	*/
7910	VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT
7911	{
7912	/* Increment the stack pointer. */
7913	uint64_t uNewRsp;
7914	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
7915
7916	/* Write the word the lazy way. */
7917	uint16_t const *pu16Src;
7918	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop,
7919	IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
7920	if (rc == VINF_SUCCESS)
7921	{
7922	pu16Value = pu16Src;
7923	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
7924
7925	/* Commit the new RSP value. */
7926	if (rc == VINF_SUCCESS)
7927	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7928	}
7929
7930	return rc;
7931	}
7932
7933
7934	/**
7935	* Pops a dword from the stack.
7936	*
7937	* @returns Strict VBox status code.
7938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7939	* @param pu32Value Where to store the popped value.
7940	*/
7941	VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT
7942	{
7943	/* Increment the stack pointer. */
7944	uint64_t uNewRsp;
7945	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
7946
7947	/* Write the word the lazy way. */
7948	uint32_t const *pu32Src;
7949	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop,
7950	IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
7951	if (rc == VINF_SUCCESS)
7952	{
7953	pu32Value = pu32Src;
7954	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
7955
7956	/* Commit the new RSP value. */
7957	if (rc == VINF_SUCCESS)
7958	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7959	}
7960
7961	return rc;
7962	}
7963
7964
7965	/**
7966	* Pops a qword from the stack.
7967	*
7968	* @returns Strict VBox status code.
7969	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7970	* @param pu64Value Where to store the popped value.
7971	*/
7972	VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT
7973	{
7974	/* Increment the stack pointer. */
7975	uint64_t uNewRsp;
7976	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
7977
7978	/* Write the word the lazy way. */
7979	uint64_t const *pu64Src;
7980	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop,
7981	IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
7982	if (rc == VINF_SUCCESS)
7983	{
7984	pu64Value = pu64Src;
7985	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
7986
7987	/* Commit the new RSP value. */
7988	if (rc == VINF_SUCCESS)
7989	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7990	}
7991
7992	return rc;
7993	}
7994
7995
7996	/**
7997	* Pushes a word onto the stack, using a temporary stack pointer.
7998	*
7999	* @returns Strict VBox status code.
8000	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8001	* @param u16Value The value to push.
8002	* @param pTmpRsp Pointer to the temporary stack pointer.
8003	*/
8004	VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8005	{
8006	/* Increment the stack pointer. */
8007	RTUINT64U NewRsp = *pTmpRsp;
8008	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
8009
8010	/* Write the word the lazy way. */
8011	uint16_t *pu16Dst;
8012	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop,
8013	IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
8014	if (rc == VINF_SUCCESS)
8015	{
8016	*pu16Dst = u16Value;
8017	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
8018	}
8019
8020	/* Commit the new RSP value unless we an access handler made trouble. */
8021	if (rc == VINF_SUCCESS)
8022	*pTmpRsp = NewRsp;
8023
8024	return rc;
8025	}
8026
8027
8028	/**
8029	* Pushes a dword onto the stack, using a temporary stack pointer.
8030	*
8031	* @returns Strict VBox status code.
8032	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8033	* @param u32Value The value to push.
8034	* @param pTmpRsp Pointer to the temporary stack pointer.
8035	*/
8036	VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8037	{
8038	/* Increment the stack pointer. */
8039	RTUINT64U NewRsp = *pTmpRsp;
8040	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
8041
8042	/* Write the word the lazy way. */
8043	uint32_t *pu32Dst;
8044	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop,
8045	IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
8046	if (rc == VINF_SUCCESS)
8047	{
8048	*pu32Dst = u32Value;
8049	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
8050	}
8051
8052	/* Commit the new RSP value unless we an access handler made trouble. */
8053	if (rc == VINF_SUCCESS)
8054	*pTmpRsp = NewRsp;
8055
8056	return rc;
8057	}
8058
8059
8060	/**
8061	* Pushes a dword onto the stack, using a temporary stack pointer.
8062	*
8063	* @returns Strict VBox status code.
8064	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8065	* @param u64Value The value to push.
8066	* @param pTmpRsp Pointer to the temporary stack pointer.
8067	*/
8068	VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8069	{
8070	/* Increment the stack pointer. */
8071	RTUINT64U NewRsp = *pTmpRsp;
8072	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
8073
8074	/* Write the word the lazy way. */
8075	uint64_t *pu64Dst;
8076	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop,
8077	IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
8078	if (rc == VINF_SUCCESS)
8079	{
8080	*pu64Dst = u64Value;
8081	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
8082	}
8083
8084	/* Commit the new RSP value unless we an access handler made trouble. */
8085	if (rc == VINF_SUCCESS)
8086	*pTmpRsp = NewRsp;
8087
8088	return rc;
8089	}
8090
8091
8092	/**
8093	* Pops a word from the stack, using a temporary stack pointer.
8094	*
8095	* @returns Strict VBox status code.
8096	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8097	* @param pu16Value Where to store the popped value.
8098	* @param pTmpRsp Pointer to the temporary stack pointer.
8099	*/
8100	VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8101	{
8102	/* Increment the stack pointer. */
8103	RTUINT64U NewRsp = *pTmpRsp;
8104	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
8105
8106	/* Write the word the lazy way. */
8107	uint16_t const *pu16Src;
8108	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop,
8109	IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
8110	if (rc == VINF_SUCCESS)
8111	{
8112	pu16Value = pu16Src;
8113	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
8114
8115	/* Commit the new RSP value. */
8116	if (rc == VINF_SUCCESS)
8117	*pTmpRsp = NewRsp;
8118	}
8119
8120	return rc;
8121	}
8122
8123
8124	/**
8125	* Pops a dword from the stack, using a temporary stack pointer.
8126	*
8127	* @returns Strict VBox status code.
8128	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8129	* @param pu32Value Where to store the popped value.
8130	* @param pTmpRsp Pointer to the temporary stack pointer.
8131	*/
8132	VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8133	{
8134	/* Increment the stack pointer. */
8135	RTUINT64U NewRsp = *pTmpRsp;
8136	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
8137
8138	/* Write the word the lazy way. */
8139	uint32_t const *pu32Src;
8140	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop,
8141	IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
8142	if (rc == VINF_SUCCESS)
8143	{
8144	pu32Value = pu32Src;
8145	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
8146
8147	/* Commit the new RSP value. */
8148	if (rc == VINF_SUCCESS)
8149	*pTmpRsp = NewRsp;
8150	}
8151
8152	return rc;
8153	}
8154
8155
8156	/**
8157	* Pops a qword from the stack, using a temporary stack pointer.
8158	*
8159	* @returns Strict VBox status code.
8160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8161	* @param pu64Value Where to store the popped value.
8162	* @param pTmpRsp Pointer to the temporary stack pointer.
8163	*/
8164	VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8165	{
8166	/* Increment the stack pointer. */
8167	RTUINT64U NewRsp = *pTmpRsp;
8168	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
8169
8170	/* Write the word the lazy way. */
8171	uint64_t const *pu64Src;
8172	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop,
8173	IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
8174	if (rcStrict == VINF_SUCCESS)
8175	{
8176	pu64Value = pu64Src;
8177	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
8178
8179	/* Commit the new RSP value. */
8180	if (rcStrict == VINF_SUCCESS)
8181	*pTmpRsp = NewRsp;
8182	}
8183
8184	return rcStrict;
8185	}
8186
8187
8188	/**
8189	* Begin a special stack push (used by interrupt, exceptions and such).
8190	*
8191	* This will raise \#SS or \#PF if appropriate.
8192	*
8193	* @returns Strict VBox status code.
8194	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8195	* @param cbMem The number of bytes to push onto the stack.
8196	* @param cbAlign The alignment mask (7, 3, 1).
8197	* @param ppvMem Where to return the pointer to the stack memory.
8198	* As with the other memory functions this could be
8199	* direct access or bounce buffered access, so
8200	* don't commit register until the commit call
8201	* succeeds.
8202	* @param puNewRsp Where to return the new RSP value. This must be
8203	* passed unchanged to
8204	* iemMemStackPushCommitSpecial().
8205	*/
8206	VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8207	void *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8208	{
8209	Assert(cbMem < UINT8_MAX);
8210	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
8211	return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop,
8212	IEM_ACCESS_STACK_W, cbAlign);
8213	}
8214
8215
8216	/**
8217	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
8218	*
8219	* This will update the rSP.
8220	*
8221	* @returns Strict VBox status code.
8222	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8223	* @param pvMem The pointer returned by
8224	* iemMemStackPushBeginSpecial().
8225	* @param uNewRsp The new RSP value returned by
8226	* iemMemStackPushBeginSpecial().
8227	*/
8228	VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT
8229	{
8230	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
8231	if (rcStrict == VINF_SUCCESS)
8232	pVCpu->cpum.GstCtx.rsp = uNewRsp;
8233	return rcStrict;
8234	}
8235
8236
8237	/**
8238	* Begin a special stack pop (used by iret, retf and such).
8239	*
8240	* This will raise \#SS or \#PF if appropriate.
8241	*
8242	* @returns Strict VBox status code.
8243	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8244	* @param cbMem The number of bytes to pop from the stack.
8245	* @param cbAlign The alignment mask (7, 3, 1).
8246	* @param ppvMem Where to return the pointer to the stack memory.
8247	* @param puNewRsp Where to return the new RSP value. This must be
8248	* assigned to CPUMCTX::rsp manually some time
8249	* after iemMemStackPopDoneSpecial() has been
8250	* called.
8251	*/
8252	VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8253	void const *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8254	{
8255	Assert(cbMem < UINT8_MAX);
8256	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
8257	return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign);
8258	}
8259
8260
8261	/**
8262	* Continue a special stack pop (used by iret and retf).
8263	*
8264	* This will raise \#SS or \#PF if appropriate.
8265	*
8266	* @returns Strict VBox status code.
8267	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8268	* @param cbMem The number of bytes to pop from the stack.
8269	* @param ppvMem Where to return the pointer to the stack memory.
8270	* @param puNewRsp Where to return the new RSP value. This must be
8271	* assigned to CPUMCTX::rsp manually some time
8272	* after iemMemStackPopDoneSpecial() has been
8273	* called.
8274	*/
8275	VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8276	{
8277	Assert(cbMem < UINT8_MAX);
8278	RTUINT64U NewRsp;
8279	NewRsp.u = *puNewRsp;
8280	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
8281	*puNewRsp = NewRsp.u;
8282	return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R,
8283	0 /* checked in iemMemStackPopBeginSpecial */);
8284	}
8285
8286
8287	/**
8288	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
8289	* iemMemStackPopContinueSpecial).
8290	*
8291	* The caller will manually commit the rSP.
8292	*
8293	* @returns Strict VBox status code.
8294	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8295	* @param pvMem The pointer returned by
8296	* iemMemStackPopBeginSpecial() or
8297	* iemMemStackPopContinueSpecial().
8298	*/
8299	VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT
8300	{
8301	return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
8302	}
8303
8304
8305	/**
8306	* Fetches a system table byte.
8307	*
8308	* @returns Strict VBox status code.
8309	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8310	* @param pbDst Where to return the byte.
8311	* @param iSegReg The index of the segment register to use for
8312	* this access. The base and limits are checked.
8313	* @param GCPtrMem The address of the guest memory.
8314	*/
8315	VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8316	{
8317	/* The lazy approach for now... */
8318	uint8_t const *pbSrc;
8319	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pbSrc, sizeof(pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8320	if (rc == VINF_SUCCESS)
8321	{
8322	pbDst = pbSrc;
8323	rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
8324	}
8325	return rc;
8326	}
8327
8328
8329	/**
8330	* Fetches a system table word.
8331	*
8332	* @returns Strict VBox status code.
8333	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8334	* @param pu16Dst Where to return the word.
8335	* @param iSegReg The index of the segment register to use for
8336	* this access. The base and limits are checked.
8337	* @param GCPtrMem The address of the guest memory.
8338	*/
8339	VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8340	{
8341	/* The lazy approach for now... */
8342	uint16_t const *pu16Src;
8343	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8344	if (rc == VINF_SUCCESS)
8345	{
8346	pu16Dst = pu16Src;
8347	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
8348	}
8349	return rc;
8350	}
8351
8352
8353	/**
8354	* Fetches a system table dword.
8355	*
8356	* @returns Strict VBox status code.
8357	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8358	* @param pu32Dst Where to return the dword.
8359	* @param iSegReg The index of the segment register to use for
8360	* this access. The base and limits are checked.
8361	* @param GCPtrMem The address of the guest memory.
8362	*/
8363	VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8364	{
8365	/* The lazy approach for now... */
8366	uint32_t const *pu32Src;
8367	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8368	if (rc == VINF_SUCCESS)
8369	{
8370	pu32Dst = pu32Src;
8371	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
8372	}
8373	return rc;
8374	}
8375
8376
8377	/**
8378	* Fetches a system table qword.
8379	*
8380	* @returns Strict VBox status code.
8381	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8382	* @param pu64Dst Where to return the qword.
8383	* @param iSegReg The index of the segment register to use for
8384	* this access. The base and limits are checked.
8385	* @param GCPtrMem The address of the guest memory.
8386	*/
8387	VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8388	{
8389	/* The lazy approach for now... */
8390	uint64_t const *pu64Src;
8391	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8392	if (rc == VINF_SUCCESS)
8393	{
8394	pu64Dst = pu64Src;
8395	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
8396	}
8397	return rc;
8398	}
8399
8400
8401	/**
8402	* Fetches a descriptor table entry with caller specified error code.
8403	*
8404	* @returns Strict VBox status code.
8405	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8406	* @param pDesc Where to return the descriptor table entry.
8407	* @param uSel The selector which table entry to fetch.
8408	* @param uXcpt The exception to raise on table lookup error.
8409	* @param uErrorCode The error code associated with the exception.
8410	*/
8411	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
8412	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT
8413	{
8414	AssertPtr(pDesc);
8415	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
8416
8417	/** @todo did the 286 require all 8 bytes to be accessible? */
8418	/*
8419	* Get the selector table base and check bounds.
8420	*/
8421	RTGCPTR GCPtrBase;
8422	if (uSel & X86_SEL_LDT)
8423	{
8424	if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
8425	\|\| (uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
8426	{
8427	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
8428	uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
8429	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8430	uErrorCode, 0);
8431	}
8432
8433	Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
8434	GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
8435	}
8436	else
8437	{
8438	if ((uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
8439	{
8440	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
8441	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8442	uErrorCode, 0);
8443	}
8444	GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
8445	}
8446
8447	/*
8448	* Read the legacy descriptor and maybe the long mode extensions if
8449	* required.
8450	*/
8451	VBOXSTRICTRC rcStrict;
8452	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
8453	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
8454	else
8455	{
8456	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
8457	if (rcStrict == VINF_SUCCESS)
8458	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
8459	if (rcStrict == VINF_SUCCESS)
8460	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
8461	if (rcStrict == VINF_SUCCESS)
8462	pDesc->Legacy.au16[3] = 0;
8463	else
8464	return rcStrict;
8465	}
8466
8467	if (rcStrict == VINF_SUCCESS)
8468	{
8469	if ( !IEM_IS_LONG_MODE(pVCpu)
8470	\|\| pDesc->Legacy.Gen.u1DescType)
8471	pDesc->Long.au64[1] = 0;
8472	else if ((uint32_t)(uSel \| X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
8473	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel \| X86_SEL_RPL_LDT) + 1);
8474	else
8475	{
8476	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
8477	/** @todo is this the right exception? */
8478	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
8479	}
8480	}
8481	return rcStrict;
8482	}
8483
8484
8485	/**
8486	* Fetches a descriptor table entry.
8487	*
8488	* @returns Strict VBox status code.
8489	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8490	* @param pDesc Where to return the descriptor table entry.
8491	* @param uSel The selector which table entry to fetch.
8492	* @param uXcpt The exception to raise on table lookup error.
8493	*/
8494	VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT
8495	{
8496	return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
8497	}
8498
8499
8500	/**
8501	* Marks the selector descriptor as accessed (only non-system descriptors).
8502	*
8503	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
8504	* will therefore skip the limit checks.
8505	*
8506	* @returns Strict VBox status code.
8507	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8508	* @param uSel The selector.
8509	*/
8510	VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
8511	{
8512	/*
8513	* Get the selector table base and calculate the entry address.
8514	*/
8515	RTGCPTR GCPtr = uSel & X86_SEL_LDT
8516	? pVCpu->cpum.GstCtx.ldtr.u64Base
8517	: pVCpu->cpum.GstCtx.gdtr.pGdt;
8518	GCPtr += uSel & X86_SEL_MASK;
8519
8520	/*
8521	* ASMAtomicBitSet will assert if the address is misaligned, so do some
8522	* ugly stuff to avoid this. This will make sure it's an atomic access
8523	* as well more or less remove any question about 8-bit or 32-bit accesss.
8524	*/
8525	VBOXSTRICTRC rcStrict;
8526	uint32_t volatile *pu32;
8527	if ((GCPtr & 3) == 0)
8528	{
8529	/* The normal case, map the 32-bit bits around the accessed bit (40). */
8530	GCPtr += 2 + 2;
8531	rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8532	if (rcStrict != VINF_SUCCESS)
8533	return rcStrict;
8534	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
8535	}
8536	else
8537	{
8538	/* The misaligned GDT/LDT case, map the whole thing. */
8539	rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8540	if (rcStrict != VINF_SUCCESS)
8541	return rcStrict;
8542	switch ((uintptr_t)pu32 & 3)
8543	{
8544	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
8545	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
8546	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
8547	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
8548	}
8549	}
8550
8551	return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
8552	}
8553
8554	/** @} */
8555
8556	/** @name Opcode Helpers.
8557	* @{
8558	*/
8559
8560	/**
8561	* Calculates the effective address of a ModR/M memory operand.
8562	*
8563	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8564	*
8565	* @return Strict VBox status code.
8566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8567	* @param bRm The ModRM byte.
8568	* @param cbImm The size of any immediate following the
8569	* effective address opcode bytes. Important for
8570	* RIP relative addressing.
8571	* @param pGCPtrEff Where to return the effective address.
8572	*/
8573	VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff) RT_NOEXCEPT
8574	{
8575	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8576	# define SET_SS_DEF() \
8577	do \
8578	{ \
8579	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8580	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8581	} while (0)
8582
8583	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8584	{
8585	/** @todo Check the effective address size crap! */
8586	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8587	{
8588	uint16_t u16EffAddr;
8589
8590	/* Handle the disp16 form with no registers first. */
8591	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8592	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8593	else
8594	{
8595	/* Get the displacment. */
8596	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8597	{
8598	case 0: u16EffAddr = 0; break;
8599	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8600	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8601	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8602	}
8603
8604	/* Add the base and index registers to the disp. */
8605	switch (bRm & X86_MODRM_RM_MASK)
8606	{
8607	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8608	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8609	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8610	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8611	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8612	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8613	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8614	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8615	}
8616	}
8617
8618	*pGCPtrEff = u16EffAddr;
8619	}
8620	else
8621	{
8622	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8623	uint32_t u32EffAddr;
8624
8625	/* Handle the disp32 form with no registers first. */
8626	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8627	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8628	else
8629	{
8630	/* Get the register (or SIB) value. */
8631	switch ((bRm & X86_MODRM_RM_MASK))
8632	{
8633	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8634	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8635	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8636	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8637	case 4: /* SIB */
8638	{
8639	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8640
8641	/* Get the index and scale it. */
8642	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8643	{
8644	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8645	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8646	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8647	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8648	case 4: u32EffAddr = 0; /none / break;
8649	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8650	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8651	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8652	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8653	}
8654	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8655
8656	/* add base */
8657	switch (bSib & X86_SIB_BASE_MASK)
8658	{
8659	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8660	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8661	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8662	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8663	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
8664	case 5:
8665	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8666	{
8667	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8668	SET_SS_DEF();
8669	}
8670	else
8671	{
8672	uint32_t u32Disp;
8673	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8674	u32EffAddr += u32Disp;
8675	}
8676	break;
8677	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8678	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8679	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8680	}
8681	break;
8682	}
8683	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8684	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8685	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8686	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8687	}
8688
8689	/* Get and add the displacement. */
8690	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8691	{
8692	case 0:
8693	break;
8694	case 1:
8695	{
8696	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8697	u32EffAddr += i8Disp;
8698	break;
8699	}
8700	case 2:
8701	{
8702	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8703	u32EffAddr += u32Disp;
8704	break;
8705	}
8706	default:
8707	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
8708	}
8709
8710	}
8711	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
8712	*pGCPtrEff = u32EffAddr;
8713	else
8714	{
8715	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
8716	*pGCPtrEff = u32EffAddr & UINT16_MAX;
8717	}
8718	}
8719	}
8720	else
8721	{
8722	uint64_t u64EffAddr;
8723
8724	/* Handle the rip+disp32 form with no registers first. */
8725	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8726	{
8727	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8728	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
8729	}
8730	else
8731	{
8732	/* Get the register (or SIB) value. */
8733	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
8734	{
8735	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8736	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8737	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8738	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8739	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8740	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8741	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8742	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8743	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8744	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8745	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8746	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8747	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8748	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8749	/* SIB */
8750	case 4:
8751	case 12:
8752	{
8753	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8754
8755	/* Get the index and scale it. */
8756	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
8757	{
8758	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8759	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8760	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8761	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8762	case 4: u64EffAddr = 0; /none / break;
8763	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8764	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8765	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8766	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8767	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8768	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8769	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8770	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8771	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8772	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8773	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8774	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8775	}
8776	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8777
8778	/* add base */
8779	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
8780	{
8781	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8782	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8783	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8784	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8785	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
8786	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8787	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8788	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8789	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8790	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8791	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8792	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8793	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8794	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8795	/* complicated encodings */
8796	case 5:
8797	case 13:
8798	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8799	{
8800	if (!pVCpu->iem.s.uRexB)
8801	{
8802	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8803	SET_SS_DEF();
8804	}
8805	else
8806	u64EffAddr += pVCpu->cpum.GstCtx.r13;
8807	}
8808	else
8809	{
8810	uint32_t u32Disp;
8811	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8812	u64EffAddr += (int32_t)u32Disp;
8813	}
8814	break;
8815	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8816	}
8817	break;
8818	}
8819	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8820	}
8821
8822	/* Get and add the displacement. */
8823	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8824	{
8825	case 0:
8826	break;
8827	case 1:
8828	{
8829	int8_t i8Disp;
8830	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8831	u64EffAddr += i8Disp;
8832	break;
8833	}
8834	case 2:
8835	{
8836	uint32_t u32Disp;
8837	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8838	u64EffAddr += (int32_t)u32Disp;
8839	break;
8840	}
8841	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
8842	}
8843
8844	}
8845
8846	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8847	*pGCPtrEff = u64EffAddr;
8848	else
8849	{
8850	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8851	*pGCPtrEff = u64EffAddr & UINT32_MAX;
8852	}
8853	}
8854
8855	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
8856	return VINF_SUCCESS;
8857	}
8858
8859
8860	/**
8861	* Calculates the effective address of a ModR/M memory operand.
8862	*
8863	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8864	*
8865	* @return Strict VBox status code.
8866	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8867	* @param bRm The ModRM byte.
8868	* @param cbImm The size of any immediate following the
8869	* effective address opcode bytes. Important for
8870	* RIP relative addressing.
8871	* @param pGCPtrEff Where to return the effective address.
8872	* @param offRsp RSP displacement.
8873	*/
8874	VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp) RT_NOEXCEPT
8875	{
8876	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8877	# define SET_SS_DEF() \
8878	do \
8879	{ \
8880	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8881	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8882	} while (0)
8883
8884	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8885	{
8886	/** @todo Check the effective address size crap! */
8887	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8888	{
8889	uint16_t u16EffAddr;
8890
8891	/* Handle the disp16 form with no registers first. */
8892	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8893	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8894	else
8895	{
8896	/* Get the displacment. */
8897	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8898	{
8899	case 0: u16EffAddr = 0; break;
8900	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8901	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8902	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8903	}
8904
8905	/* Add the base and index registers to the disp. */
8906	switch (bRm & X86_MODRM_RM_MASK)
8907	{
8908	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8909	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8910	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8911	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8912	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8913	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8914	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8915	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8916	}
8917	}
8918
8919	*pGCPtrEff = u16EffAddr;
8920	}
8921	else
8922	{
8923	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8924	uint32_t u32EffAddr;
8925
8926	/* Handle the disp32 form with no registers first. */
8927	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8928	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8929	else
8930	{
8931	/* Get the register (or SIB) value. */
8932	switch ((bRm & X86_MODRM_RM_MASK))
8933	{
8934	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8935	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8936	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8937	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8938	case 4: /* SIB */
8939	{
8940	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8941
8942	/* Get the index and scale it. */
8943	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8944	{
8945	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8946	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8947	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8948	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8949	case 4: u32EffAddr = 0; /none / break;
8950	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8951	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8952	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8953	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8954	}
8955	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8956
8957	/* add base */
8958	switch (bSib & X86_SIB_BASE_MASK)
8959	{
8960	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8961	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8962	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8963	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8964	case 4:
8965	u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
8966	SET_SS_DEF();
8967	break;
8968	case 5:
8969	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8970	{
8971	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8972	SET_SS_DEF();
8973	}
8974	else
8975	{
8976	uint32_t u32Disp;
8977	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8978	u32EffAddr += u32Disp;
8979	}
8980	break;
8981	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8982	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8983	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8984	}
8985	break;
8986	}
8987	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8988	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8989	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8990	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8991	}
8992
8993	/* Get and add the displacement. */
8994	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8995	{
8996	case 0:
8997	break;
8998	case 1:
8999	{
9000	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9001	u32EffAddr += i8Disp;
9002	break;
9003	}
9004	case 2:
9005	{
9006	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9007	u32EffAddr += u32Disp;
9008	break;
9009	}
9010	default:
9011	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
9012	}
9013
9014	}
9015	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9016	*pGCPtrEff = u32EffAddr;
9017	else
9018	{
9019	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9020	*pGCPtrEff = u32EffAddr & UINT16_MAX;
9021	}
9022	}
9023	}
9024	else
9025	{
9026	uint64_t u64EffAddr;
9027
9028	/* Handle the rip+disp32 form with no registers first. */
9029	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9030	{
9031	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9032	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9033	}
9034	else
9035	{
9036	/* Get the register (or SIB) value. */
9037	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
9038	{
9039	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9040	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9041	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9042	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9043	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9044	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9045	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9046	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9047	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9048	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9049	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9050	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9051	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9052	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9053	/* SIB */
9054	case 4:
9055	case 12:
9056	{
9057	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9058
9059	/* Get the index and scale it. */
9060	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
9061	{
9062	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9063	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9064	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9065	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9066	case 4: u64EffAddr = 0; /none / break;
9067	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9068	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9069	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9070	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9071	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9072	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9073	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9074	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9075	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9076	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9077	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9078	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9079	}
9080	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9081
9082	/* add base */
9083	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
9084	{
9085	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9086	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9087	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9088	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9089	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
9090	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9091	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9092	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9093	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9094	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9095	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9096	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9097	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9098	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9099	/* complicated encodings */
9100	case 5:
9101	case 13:
9102	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9103	{
9104	if (!pVCpu->iem.s.uRexB)
9105	{
9106	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9107	SET_SS_DEF();
9108	}
9109	else
9110	u64EffAddr += pVCpu->cpum.GstCtx.r13;
9111	}
9112	else
9113	{
9114	uint32_t u32Disp;
9115	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9116	u64EffAddr += (int32_t)u32Disp;
9117	}
9118	break;
9119	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9120	}
9121	break;
9122	}
9123	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9124	}
9125
9126	/* Get and add the displacement. */
9127	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9128	{
9129	case 0:
9130	break;
9131	case 1:
9132	{
9133	int8_t i8Disp;
9134	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9135	u64EffAddr += i8Disp;
9136	break;
9137	}
9138	case 2:
9139	{
9140	uint32_t u32Disp;
9141	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9142	u64EffAddr += (int32_t)u32Disp;
9143	break;
9144	}
9145	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
9146	}
9147
9148	}
9149
9150	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9151	*pGCPtrEff = u64EffAddr;
9152	else
9153	{
9154	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9155	*pGCPtrEff = u64EffAddr & UINT32_MAX;
9156	}
9157	}
9158
9159	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
9160	return VINF_SUCCESS;
9161	}
9162
9163
9164	#ifdef IEM_WITH_SETJMP
9165	/**
9166	* Calculates the effective address of a ModR/M memory operand.
9167	*
9168	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
9169	*
9170	* May longjmp on internal error.
9171	*
9172	* @return The effective address.
9173	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9174	* @param bRm The ModRM byte.
9175	* @param cbImm The size of any immediate following the
9176	* effective address opcode bytes. Important for
9177	* RIP relative addressing.
9178	*/
9179	RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm) RT_NOEXCEPT
9180	{
9181	Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
9182	# define SET_SS_DEF() \
9183	do \
9184	{ \
9185	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
9186	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
9187	} while (0)
9188
9189	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
9190	{
9191	/** @todo Check the effective address size crap! */
9192	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
9193	{
9194	uint16_t u16EffAddr;
9195
9196	/* Handle the disp16 form with no registers first. */
9197	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
9198	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
9199	else
9200	{
9201	/* Get the displacment. */
9202	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9203	{
9204	case 0: u16EffAddr = 0; break;
9205	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
9206	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
9207	default: AssertFailedStmt(longjmp(pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); / (caller checked for these) */
9208	}
9209
9210	/* Add the base and index registers to the disp. */
9211	switch (bRm & X86_MODRM_RM_MASK)
9212	{
9213	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
9214	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
9215	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
9216	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
9217	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
9218	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
9219	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
9220	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
9221	}
9222	}
9223
9224	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
9225	return u16EffAddr;
9226	}
9227
9228	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9229	uint32_t u32EffAddr;
9230
9231	/* Handle the disp32 form with no registers first. */
9232	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9233	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
9234	else
9235	{
9236	/* Get the register (or SIB) value. */
9237	switch ((bRm & X86_MODRM_RM_MASK))
9238	{
9239	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9240	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9241	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9242	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9243	case 4: /* SIB */
9244	{
9245	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9246
9247	/* Get the index and scale it. */
9248	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
9249	{
9250	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9251	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9252	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9253	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9254	case 4: u32EffAddr = 0; /none / break;
9255	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
9256	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9257	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9258	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9259	}
9260	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9261
9262	/* add base */
9263	switch (bSib & X86_SIB_BASE_MASK)
9264	{
9265	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
9266	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
9267	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
9268	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
9269	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
9270	case 5:
9271	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9272	{
9273	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
9274	SET_SS_DEF();
9275	}
9276	else
9277	{
9278	uint32_t u32Disp;
9279	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9280	u32EffAddr += u32Disp;
9281	}
9282	break;
9283	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
9284	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
9285	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9286	}
9287	break;
9288	}
9289	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
9290	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9291	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9292	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9293	}
9294
9295	/* Get and add the displacement. */
9296	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9297	{
9298	case 0:
9299	break;
9300	case 1:
9301	{
9302	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9303	u32EffAddr += i8Disp;
9304	break;
9305	}
9306	case 2:
9307	{
9308	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9309	u32EffAddr += u32Disp;
9310	break;
9311	}
9312	default:
9313	AssertFailedStmt(longjmp(pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); / (caller checked for these) */
9314	}
9315	}
9316
9317	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9318	{
9319	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
9320	return u32EffAddr;
9321	}
9322	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9323	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
9324	return u32EffAddr & UINT16_MAX;
9325	}
9326
9327	uint64_t u64EffAddr;
9328
9329	/* Handle the rip+disp32 form with no registers first. */
9330	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9331	{
9332	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9333	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9334	}
9335	else
9336	{
9337	/* Get the register (or SIB) value. */
9338	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
9339	{
9340	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9341	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9342	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9343	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9344	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9345	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9346	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9347	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9348	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9349	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9350	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9351	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9352	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9353	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9354	/* SIB */
9355	case 4:
9356	case 12:
9357	{
9358	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9359
9360	/* Get the index and scale it. */
9361	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
9362	{
9363	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9364	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9365	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9366	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9367	case 4: u64EffAddr = 0; /none / break;
9368	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9369	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9370	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9371	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9372	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9373	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9374	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9375	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9376	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9377	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9378	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9379	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9380	}
9381	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9382
9383	/* add base */
9384	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
9385	{
9386	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9387	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9388	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9389	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9390	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
9391	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9392	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9393	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9394	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9395	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9396	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9397	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9398	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9399	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9400	/* complicated encodings */
9401	case 5:
9402	case 13:
9403	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9404	{
9405	if (!pVCpu->iem.s.uRexB)
9406	{
9407	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9408	SET_SS_DEF();
9409	}
9410	else
9411	u64EffAddr += pVCpu->cpum.GstCtx.r13;
9412	}
9413	else
9414	{
9415	uint32_t u32Disp;
9416	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9417	u64EffAddr += (int32_t)u32Disp;
9418	}
9419	break;
9420	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9421	}
9422	break;
9423	}
9424	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9425	}
9426
9427	/* Get and add the displacement. */
9428	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9429	{
9430	case 0:
9431	break;
9432	case 1:
9433	{
9434	int8_t i8Disp;
9435	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9436	u64EffAddr += i8Disp;
9437	break;
9438	}
9439	case 2:
9440	{
9441	uint32_t u32Disp;
9442	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9443	u64EffAddr += (int32_t)u32Disp;
9444	break;
9445	}
9446	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
9447	}
9448
9449	}
9450
9451	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9452	{
9453	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
9454	return u64EffAddr;
9455	}
9456	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9457	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
9458	return u64EffAddr & UINT32_MAX;
9459	}
9460	#endif /* IEM_WITH_SETJMP */
9461
9462	/** @} */
9463
9464
9465	#ifdef LOG_ENABLED
9466	/**
9467	* Logs the current instruction.
9468	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9469	* @param fSameCtx Set if we have the same context information as the VMM,
9470	* clear if we may have already executed an instruction in
9471	* our debug context. When clear, we assume IEMCPU holds
9472	* valid CPU mode info.
9473	*
9474	* The @a fSameCtx parameter is now misleading and obsolete.
9475	* @param pszFunction The IEM function doing the execution.
9476	*/
9477	static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT
9478	{
9479	# ifdef IN_RING3
9480	if (LogIs2Enabled())
9481	{
9482	char szInstr[256];
9483	uint32_t cbInstr = 0;
9484	if (fSameCtx)
9485	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
9486	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
9487	szInstr, sizeof(szInstr), &cbInstr);
9488	else
9489	{
9490	uint32_t fFlags = 0;
9491	switch (pVCpu->iem.s.enmCpuMode)
9492	{
9493	case IEMMODE_64BIT: fFlags \|= DBGF_DISAS_FLAGS_64BIT_MODE; break;
9494	case IEMMODE_32BIT: fFlags \|= DBGF_DISAS_FLAGS_32BIT_MODE; break;
9495	case IEMMODE_16BIT:
9496	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
9497	fFlags \|= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
9498	else
9499	fFlags \|= DBGF_DISAS_FLAGS_16BIT_MODE;
9500	break;
9501	}
9502	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
9503	szInstr, sizeof(szInstr), &cbInstr);
9504	}
9505
9506	PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9507	Log2(("**** %s\n"
9508	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
9509	" eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
9510	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
9511	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
9512	" %s\n"
9513	, pszFunction,
9514	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
9515	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
9516	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
9517	pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
9518	pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
9519	szInstr));
9520
9521	if (LogIs3Enabled())
9522	DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
9523	}
9524	else
9525	# endif
9526	LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
9527	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
9528	RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
9529	}
9530	#endif /* LOG_ENABLED */
9531
9532
9533	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9534	/**
9535	* Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
9536	* VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
9537	*
9538	* @returns Modified rcStrict.
9539	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9540	* @param rcStrict The instruction execution status.
9541	*/
9542	static VBOXSTRICTRC iemHandleNestedInstructionBoundraryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
9543	{
9544	Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
9545	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF))
9546	{
9547	/* VMX preemption timer takes priority over NMI-window exits. */
9548	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
9549	{
9550	rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
9551	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
9552	}
9553	/*
9554	* Check remaining intercepts.
9555	*
9556	* NMI-window and Interrupt-window VM-exits.
9557	* Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
9558	* Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
9559	*
9560	* See Intel spec. 26.7.6 "NMI-Window Exiting".
9561	* See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9562	*/
9563	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW \| VMCPU_FF_VMX_INT_WINDOW)
9564	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9565	&& !TRPMHasTrap(pVCpu))
9566	{
9567	Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
9568	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
9569	&& CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
9570	{
9571	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
9572	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
9573	}
9574	else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
9575	&& CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
9576	{
9577	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
9578	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
9579	}
9580	}
9581	}
9582	/* TPR-below threshold/APIC write has the highest priority. */
9583	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
9584	{
9585	rcStrict = iemVmxApicWriteEmulation(pVCpu);
9586	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9587	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
9588	}
9589	/* MTF takes priority over VMX-preemption timer. */
9590	else
9591	{
9592	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
9593	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9594	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
9595	}
9596	return rcStrict;
9597	}
9598	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9599
9600
9601	/**
9602	* The actual code execution bits of IEMExecOne, IEMExecOneEx, and
9603	* IEMExecOneWithPrefetchedByPC.
9604	*
9605	* Similar code is found in IEMExecLots.
9606	*
9607	* @return Strict VBox status code.
9608	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9609	* @param fExecuteInhibit If set, execute the instruction following CLI,
9610	* POP SS and MOV SS,GR.
9611	* @param pszFunction The calling function name.
9612	*/
9613	DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
9614	{
9615	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9616	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9617	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9618	RT_NOREF_PV(pszFunction);
9619
9620	#ifdef IEM_WITH_SETJMP
9621	VBOXSTRICTRC rcStrict;
9622	jmp_buf JmpBuf;
9623	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
9624	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9625	if ((rcStrict = setjmp(JmpBuf)) == 0)
9626	{
9627	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9628	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9629	}
9630	else
9631	pVCpu->iem.s.cLongJumps++;
9632	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9633	#else
9634	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9635	VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9636	#endif
9637	if (rcStrict == VINF_SUCCESS)
9638	pVCpu->iem.s.cInstructions++;
9639	if (pVCpu->iem.s.cActiveMappings > 0)
9640	{
9641	Assert(rcStrict != VINF_SUCCESS);
9642	iemMemRollback(pVCpu);
9643	}
9644	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9645	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9646	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9647
9648	//#ifdef DEBUG
9649	// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
9650	//#endif
9651
9652	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9653	/*
9654	* Perform any VMX nested-guest instruction boundary actions.
9655	*
9656	* If any of these causes a VM-exit, we must skip executing the next
9657	* instruction (would run into stale page tables). A VM-exit makes sure
9658	* there is no interrupt-inhibition, so that should ensure we don't go
9659	* to try execute the next instruction. Clearing fExecuteInhibit is
9660	* problematic because of the setjmp/longjmp clobbering above.
9661	*/
9662	if ( rcStrict == VINF_SUCCESS
9663	&& VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9664	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW))
9665	rcStrict = iemHandleNestedInstructionBoundraryFFs(pVCpu, rcStrict);
9666	#endif
9667
9668	/* Execute the next instruction as well if a cli, pop ss or
9669	mov ss, Gr has just completed successfully. */
9670	if ( fExecuteInhibit
9671	&& rcStrict == VINF_SUCCESS
9672	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9673	&& EMIsInhibitInterruptsActive(pVCpu))
9674	{
9675	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock);
9676	if (rcStrict == VINF_SUCCESS)
9677	{
9678	#ifdef LOG_ENABLED
9679	iemLogCurInstr(pVCpu, false, pszFunction);
9680	#endif
9681	#ifdef IEM_WITH_SETJMP
9682	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9683	if ((rcStrict = setjmp(JmpBuf)) == 0)
9684	{
9685	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9686	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9687	}
9688	else
9689	pVCpu->iem.s.cLongJumps++;
9690	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9691	#else
9692	IEM_OPCODE_GET_NEXT_U8(&b);
9693	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9694	#endif
9695	if (rcStrict == VINF_SUCCESS)
9696	pVCpu->iem.s.cInstructions++;
9697	if (pVCpu->iem.s.cActiveMappings > 0)
9698	{
9699	Assert(rcStrict != VINF_SUCCESS);
9700	iemMemRollback(pVCpu);
9701	}
9702	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9703	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9704	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9705	}
9706	else if (pVCpu->iem.s.cActiveMappings > 0)
9707	iemMemRollback(pVCpu);
9708	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
9709	}
9710
9711	/*
9712	* Return value fiddling, statistics and sanity assertions.
9713	*/
9714	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9715
9716	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9717	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9718	return rcStrict;
9719	}
9720
9721
9722	/**
9723	* Execute one instruction.
9724	*
9725	* @return Strict VBox status code.
9726	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9727	*/
9728	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
9729	{
9730	AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
9731	#ifdef LOG_ENABLED
9732	iemLogCurInstr(pVCpu, true, "IEMExecOne");
9733	#endif
9734
9735	/*
9736	* Do the decoding and emulation.
9737	*/
9738	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9739	if (rcStrict == VINF_SUCCESS)
9740	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
9741	else if (pVCpu->iem.s.cActiveMappings > 0)
9742	iemMemRollback(pVCpu);
9743
9744	if (rcStrict != VINF_SUCCESS)
9745	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9746	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9747	return rcStrict;
9748	}
9749
9750
9751	VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9752	{
9753	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9754
9755	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9756	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9757	if (rcStrict == VINF_SUCCESS)
9758	{
9759	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
9760	if (pcbWritten)
9761	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9762	}
9763	else if (pVCpu->iem.s.cActiveMappings > 0)
9764	iemMemRollback(pVCpu);
9765
9766	return rcStrict;
9767	}
9768
9769
9770	VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9771	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9772	{
9773	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9774
9775	VBOXSTRICTRC rcStrict;
9776	if ( cbOpcodeBytes
9777	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9778	{
9779	iemInitDecoder(pVCpu, false, false);
9780	#ifdef IEM_WITH_CODE_TLB
9781	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9782	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9783	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9784	pVCpu->iem.s.offCurInstrStart = 0;
9785	pVCpu->iem.s.offInstrNextByte = 0;
9786	#else
9787	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9788	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9789	#endif
9790	rcStrict = VINF_SUCCESS;
9791	}
9792	else
9793	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9794	if (rcStrict == VINF_SUCCESS)
9795	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
9796	else if (pVCpu->iem.s.cActiveMappings > 0)
9797	iemMemRollback(pVCpu);
9798
9799	return rcStrict;
9800	}
9801
9802
9803	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9804	{
9805	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9806
9807	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9808	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9809	if (rcStrict == VINF_SUCCESS)
9810	{
9811	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
9812	if (pcbWritten)
9813	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9814	}
9815	else if (pVCpu->iem.s.cActiveMappings > 0)
9816	iemMemRollback(pVCpu);
9817
9818	return rcStrict;
9819	}
9820
9821
9822	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9823	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9824	{
9825	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9826
9827	VBOXSTRICTRC rcStrict;
9828	if ( cbOpcodeBytes
9829	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9830	{
9831	iemInitDecoder(pVCpu, true, false);
9832	#ifdef IEM_WITH_CODE_TLB
9833	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9834	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9835	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9836	pVCpu->iem.s.offCurInstrStart = 0;
9837	pVCpu->iem.s.offInstrNextByte = 0;
9838	#else
9839	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9840	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9841	#endif
9842	rcStrict = VINF_SUCCESS;
9843	}
9844	else
9845	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9846	if (rcStrict == VINF_SUCCESS)
9847	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
9848	else if (pVCpu->iem.s.cActiveMappings > 0)
9849	iemMemRollback(pVCpu);
9850
9851	return rcStrict;
9852	}
9853
9854
9855	/**
9856	* For debugging DISGetParamSize, may come in handy.
9857	*
9858	* @returns Strict VBox status code.
9859	* @param pVCpu The cross context virtual CPU structure of the
9860	* calling EMT.
9861	* @param pCtxCore The context core structure.
9862	* @param OpcodeBytesPC The PC of the opcode bytes.
9863	* @param pvOpcodeBytes Prefeched opcode bytes.
9864	* @param cbOpcodeBytes Number of prefetched bytes.
9865	* @param pcbWritten Where to return the number of bytes written.
9866	* Optional.
9867	*/
9868	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9869	const void *pvOpcodeBytes, size_t cbOpcodeBytes,
9870	uint32_t *pcbWritten)
9871	{
9872	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9873
9874	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9875	VBOXSTRICTRC rcStrict;
9876	if ( cbOpcodeBytes
9877	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9878	{
9879	iemInitDecoder(pVCpu, true, false);
9880	#ifdef IEM_WITH_CODE_TLB
9881	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9882	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9883	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9884	pVCpu->iem.s.offCurInstrStart = 0;
9885	pVCpu->iem.s.offInstrNextByte = 0;
9886	#else
9887	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9888	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9889	#endif
9890	rcStrict = VINF_SUCCESS;
9891	}
9892	else
9893	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9894	if (rcStrict == VINF_SUCCESS)
9895	{
9896	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
9897	if (pcbWritten)
9898	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9899	}
9900	else if (pVCpu->iem.s.cActiveMappings > 0)
9901	iemMemRollback(pVCpu);
9902
9903	return rcStrict;
9904	}
9905
9906
9907	/**
9908	* For handling split cacheline lock operations when the host has split-lock
9909	* detection enabled.
9910	*
9911	* This will cause the interpreter to disregard the lock prefix and implicit
9912	* locking (xchg).
9913	*
9914	* @returns Strict VBox status code.
9915	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9916	*/
9917	VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
9918	{
9919	/*
9920	* Do the decoding and emulation.
9921	*/
9922	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /fDisregardLock/);
9923	if (rcStrict == VINF_SUCCESS)
9924	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
9925	else if (pVCpu->iem.s.cActiveMappings > 0)
9926	iemMemRollback(pVCpu);
9927
9928	if (rcStrict != VINF_SUCCESS)
9929	LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9930	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9931	return rcStrict;
9932	}
9933
9934
9935	VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
9936	{
9937	uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
9938	AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
9939
9940	/*
9941	* See if there is an interrupt pending in TRPM, inject it if we can.
9942	*/
9943	/** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
9944	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9945	bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
9946	if (fIntrEnabled)
9947	{
9948	if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
9949	fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9950	else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
9951	fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
9952	else
9953	{
9954	Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
9955	fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
9956	}
9957	}
9958	#else
9959	bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9960	#endif
9961
9962	/** @todo What if we are injecting an exception and not an interrupt? Is that
9963	* possible here? For now we assert it is indeed only an interrupt. */
9964	if ( fIntrEnabled
9965	&& TRPMHasTrap(pVCpu)
9966	&& EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
9967	{
9968	uint8_t u8TrapNo;
9969	TRPMEVENT enmType;
9970	uint32_t uErrCode;
9971	RTGCPTR uCr2;
9972	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen /, NULL / fIcebp */);
9973	AssertRC(rc2);
9974	Assert(enmType == TRPM_HARDWARE_INT);
9975	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
9976	TRPMResetTrap(pVCpu);
9977	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9978	/* Injecting an event may cause a VM-exit. */
9979	if ( rcStrict != VINF_SUCCESS
9980	&& rcStrict != VINF_IEM_RAISED_XCPT)
9981	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
9982	#else
9983	NOREF(rcStrict);
9984	#endif
9985	}
9986
9987	/*
9988	* Initial decoder init w/ prefetch, then setup setjmp.
9989	*/
9990	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9991	if (rcStrict == VINF_SUCCESS)
9992	{
9993	#ifdef IEM_WITH_SETJMP
9994	jmp_buf JmpBuf;
9995	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
9996	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9997	pVCpu->iem.s.cActiveMappings = 0;
9998	if ((rcStrict = setjmp(JmpBuf)) == 0)
9999	#endif
10000	{
10001	/*
10002	* The run loop. We limit ourselves to 4096 instructions right now.
10003	*/
10004	uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
10005	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10006	for (;;)
10007	{
10008	/*
10009	* Log the state.
10010	*/
10011	#ifdef LOG_ENABLED
10012	iemLogCurInstr(pVCpu, true, "IEMExecLots");
10013	#endif
10014
10015	/*
10016	* Do the decoding and emulation.
10017	*/
10018	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10019	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10020	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10021	{
10022	Assert(pVCpu->iem.s.cActiveMappings == 0);
10023	pVCpu->iem.s.cInstructions++;
10024	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10025	{
10026	uint64_t fCpu = pVCpu->fLocalForcedActions
10027	& ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10028	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10029	\| VMCPU_FF_TLB_FLUSH
10030	\| VMCPU_FF_INHIBIT_INTERRUPTS
10031	\| VMCPU_FF_BLOCK_NMIS
10032	\| VMCPU_FF_UNHALT ));
10033
10034	if (RT_LIKELY( ( !fCpu
10035	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
10036	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
10037	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
10038	{
10039	if (cMaxInstructionsGccStupidity-- > 0)
10040	{
10041	/* Poll timers every now an then according to the caller's specs. */
10042	if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
10043	\|\| !TMTimerPollBool(pVM, pVCpu))
10044	{
10045	Assert(pVCpu->iem.s.cActiveMappings == 0);
10046	iemReInitDecoder(pVCpu);
10047	continue;
10048	}
10049	}
10050	}
10051	}
10052	Assert(pVCpu->iem.s.cActiveMappings == 0);
10053	}
10054	else if (pVCpu->iem.s.cActiveMappings > 0)
10055	iemMemRollback(pVCpu);
10056	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10057	break;
10058	}
10059	}
10060	#ifdef IEM_WITH_SETJMP
10061	else
10062	{
10063	if (pVCpu->iem.s.cActiveMappings > 0)
10064	iemMemRollback(pVCpu);
10065	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10066	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10067	# endif
10068	pVCpu->iem.s.cLongJumps++;
10069	}
10070	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10071	#endif
10072
10073	/*
10074	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10075	*/
10076	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10077	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10078	}
10079	else
10080	{
10081	if (pVCpu->iem.s.cActiveMappings > 0)
10082	iemMemRollback(pVCpu);
10083
10084	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10085	/*
10086	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10087	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10088	*/
10089	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10090	#endif
10091	}
10092
10093	/*
10094	* Maybe re-enter raw-mode and log.
10095	*/
10096	if (rcStrict != VINF_SUCCESS)
10097	LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10098	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10099	if (pcInstructions)
10100	*pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
10101	return rcStrict;
10102	}
10103
10104
10105	/**
10106	* Interface used by EMExecuteExec, does exit statistics and limits.
10107	*
10108	* @returns Strict VBox status code.
10109	* @param pVCpu The cross context virtual CPU structure.
10110	* @param fWillExit To be defined.
10111	* @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
10112	* @param cMaxInstructions Maximum number of instructions to execute.
10113	* @param cMaxInstructionsWithoutExits
10114	* The max number of instructions without exits.
10115	* @param pStats Where to return statistics.
10116	*/
10117	VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
10118	uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
10119	{
10120	NOREF(fWillExit); /** @todo define flexible exit crits */
10121
10122	/*
10123	* Initialize return stats.
10124	*/
10125	pStats->cInstructions = 0;
10126	pStats->cExits = 0;
10127	pStats->cMaxExitDistance = 0;
10128	pStats->cReserved = 0;
10129
10130	/*
10131	* Initial decoder init w/ prefetch, then setup setjmp.
10132	*/
10133	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10134	if (rcStrict == VINF_SUCCESS)
10135	{
10136	#ifdef IEM_WITH_SETJMP
10137	jmp_buf JmpBuf;
10138	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10139	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10140	pVCpu->iem.s.cActiveMappings = 0;
10141	if ((rcStrict = setjmp(JmpBuf)) == 0)
10142	#endif
10143	{
10144	#ifdef IN_RING0
10145	bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() \|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
10146	#endif
10147	uint32_t cInstructionSinceLastExit = 0;
10148
10149	/*
10150	* The run loop. We limit ourselves to 4096 instructions right now.
10151	*/
10152	PVM pVM = pVCpu->CTX_SUFF(pVM);
10153	for (;;)
10154	{
10155	/*
10156	* Log the state.
10157	*/
10158	#ifdef LOG_ENABLED
10159	iemLogCurInstr(pVCpu, true, "IEMExecForExits");
10160	#endif
10161
10162	/*
10163	* Do the decoding and emulation.
10164	*/
10165	uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
10166
10167	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10168	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10169
10170	if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
10171	&& cInstructionSinceLastExit > 0 /* don't count the first */ )
10172	{
10173	pStats->cExits += 1;
10174	if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
10175	pStats->cMaxExitDistance = cInstructionSinceLastExit;
10176	cInstructionSinceLastExit = 0;
10177	}
10178
10179	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10180	{
10181	Assert(pVCpu->iem.s.cActiveMappings == 0);
10182	pVCpu->iem.s.cInstructions++;
10183	pStats->cInstructions++;
10184	cInstructionSinceLastExit++;
10185	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10186	{
10187	uint64_t fCpu = pVCpu->fLocalForcedActions
10188	& ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10189	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10190	\| VMCPU_FF_TLB_FLUSH
10191	\| VMCPU_FF_INHIBIT_INTERRUPTS
10192	\| VMCPU_FF_BLOCK_NMIS
10193	\| VMCPU_FF_UNHALT ));
10194
10195	if (RT_LIKELY( ( ( !fCpu
10196	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
10197	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
10198	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
10199	\|\| pStats->cInstructions < cMinInstructions))
10200	{
10201	if (pStats->cInstructions < cMaxInstructions)
10202	{
10203	if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
10204	{
10205	#ifdef IN_RING0
10206	if ( !fCheckPreemptionPending
10207	\|\| !RTThreadPreemptIsPending(NIL_RTTHREAD))
10208	#endif
10209	{
10210	Assert(pVCpu->iem.s.cActiveMappings == 0);
10211	iemReInitDecoder(pVCpu);
10212	continue;
10213	}
10214	#ifdef IN_RING0
10215	rcStrict = VINF_EM_RAW_INTERRUPT;
10216	break;
10217	#endif
10218	}
10219	}
10220	}
10221	Assert(!(fCpu & VMCPU_FF_IEM));
10222	}
10223	Assert(pVCpu->iem.s.cActiveMappings == 0);
10224	}
10225	else if (pVCpu->iem.s.cActiveMappings > 0)
10226	iemMemRollback(pVCpu);
10227	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10228	break;
10229	}
10230	}
10231	#ifdef IEM_WITH_SETJMP
10232	else
10233	{
10234	if (pVCpu->iem.s.cActiveMappings > 0)
10235	iemMemRollback(pVCpu);
10236	pVCpu->iem.s.cLongJumps++;
10237	}
10238	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10239	#endif
10240
10241	/*
10242	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10243	*/
10244	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10245	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10246	}
10247	else
10248	{
10249	if (pVCpu->iem.s.cActiveMappings > 0)
10250	iemMemRollback(pVCpu);
10251
10252	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10253	/*
10254	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10255	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10256	*/
10257	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10258	#endif
10259	}
10260
10261	/*
10262	* Maybe re-enter raw-mode and log.
10263	*/
10264	if (rcStrict != VINF_SUCCESS)
10265	LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
10266	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
10267	pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
10268	return rcStrict;
10269	}
10270
10271
10272	/**
10273	* Injects a trap, fault, abort, software interrupt or external interrupt.
10274	*
10275	* The parameter list matches TRPMQueryTrapAll pretty closely.
10276	*
10277	* @returns Strict VBox status code.
10278	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10279	* @param u8TrapNo The trap number.
10280	* @param enmType What type is it (trap/fault/abort), software
10281	* interrupt or hardware interrupt.
10282	* @param uErrCode The error code if applicable.
10283	* @param uCr2 The CR2 value if applicable.
10284	* @param cbInstr The instruction length (only relevant for
10285	* software interrupts).
10286	*/
10287	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
10288	uint8_t cbInstr)
10289	{
10290	iemInitDecoder(pVCpu, false, false);
10291	#ifdef DBGFTRACE_ENABLED
10292	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
10293	u8TrapNo, enmType, uErrCode, uCr2);
10294	#endif
10295
10296	uint32_t fFlags;
10297	switch (enmType)
10298	{
10299	case TRPM_HARDWARE_INT:
10300	Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
10301	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
10302	uErrCode = uCr2 = 0;
10303	break;
10304
10305	case TRPM_SOFTWARE_INT:
10306	Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
10307	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
10308	uErrCode = uCr2 = 0;
10309	break;
10310
10311	case TRPM_TRAP:
10312	Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
10313	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
10314	if (u8TrapNo == X86_XCPT_PF)
10315	fFlags \|= IEM_XCPT_FLAGS_CR2;
10316	switch (u8TrapNo)
10317	{
10318	case X86_XCPT_DF:
10319	case X86_XCPT_TS:
10320	case X86_XCPT_NP:
10321	case X86_XCPT_SS:
10322	case X86_XCPT_PF:
10323	case X86_XCPT_AC:
10324	case X86_XCPT_GP:
10325	fFlags \|= IEM_XCPT_FLAGS_ERR;
10326	break;
10327	}
10328	break;
10329
10330	IEM_NOT_REACHED_DEFAULT_CASE_RET();
10331	}
10332
10333	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
10334
10335	if (pVCpu->iem.s.cActiveMappings > 0)
10336	iemMemRollback(pVCpu);
10337
10338	return rcStrict;
10339	}
10340
10341
10342	/**
10343	* Injects the active TRPM event.
10344	*
10345	* @returns Strict VBox status code.
10346	* @param pVCpu The cross context virtual CPU structure.
10347	*/
10348	VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
10349	{
10350	#ifndef IEM_IMPLEMENTS_TASKSWITCH
10351	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
10352	#else
10353	uint8_t u8TrapNo;
10354	TRPMEVENT enmType;
10355	uint32_t uErrCode;
10356	RTGCUINTPTR uCr2;
10357	uint8_t cbInstr;
10358	int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
10359	if (RT_FAILURE(rc))
10360	return rc;
10361
10362	/** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
10363	* ICEBP \#DB injection as a special case. */
10364	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
10365	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
10366	if (rcStrict == VINF_SVM_VMEXIT)
10367	rcStrict = VINF_SUCCESS;
10368	#endif
10369	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10370	if (rcStrict == VINF_VMX_VMEXIT)
10371	rcStrict = VINF_SUCCESS;
10372	#endif
10373	/** @todo Are there any other codes that imply the event was successfully
10374	* delivered to the guest? See @bugref{6607}. */
10375	if ( rcStrict == VINF_SUCCESS
10376	\|\| rcStrict == VINF_IEM_RAISED_XCPT)
10377	TRPMResetTrap(pVCpu);
10378
10379	return rcStrict;
10380	#endif
10381	}
10382
10383
10384	VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
10385	{
10386	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10387	return VERR_NOT_IMPLEMENTED;
10388	}
10389
10390
10391	VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
10392	{
10393	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10394	return VERR_NOT_IMPLEMENTED;
10395	}
10396
10397
10398	#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
10399	/**
10400	* Executes a IRET instruction with default operand size.
10401	*
10402	* This is for PATM.
10403	*
10404	* @returns VBox status code.
10405	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10406	* @param pCtxCore The register frame.
10407	*/
10408	VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
10409	{
10410	PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
10411
10412	iemCtxCoreToCtx(pCtx, pCtxCore);
10413	iemInitDecoder(pVCpu);
10414	VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
10415	if (rcStrict == VINF_SUCCESS)
10416	iemCtxToCtxCore(pCtxCore, pCtx);
10417	else
10418	LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10419	pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10420	return rcStrict;
10421	}
10422	#endif
10423
10424
10425	/**
10426	* Interface for HM and EM for executing string I/O OUT (write) instructions.
10427	*
10428	* This API ASSUMES that the caller has already verified that the guest code is
10429	* allowed to access the I/O port. (The I/O port is in the DX register in the
10430	* guest state.)
10431	*
10432	* @returns Strict VBox status code.
10433	* @param pVCpu The cross context virtual CPU structure.
10434	* @param cbValue The size of the I/O port access (1, 2, or 4).
10435	* @param enmAddrMode The addressing mode.
10436	* @param fRepPrefix Indicates whether a repeat prefix is used
10437	* (doesn't matter which for this instruction).
10438	* @param cbInstr The instruction length in bytes.
10439	* @param iEffSeg The effective segment address.
10440	* @param fIoChecked Whether the access to the I/O port has been
10441	* checked or not. It's typically checked in the
10442	* HM scenario.
10443	*/
10444	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10445	bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
10446	{
10447	AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
10448	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10449
10450	/*
10451	* State init.
10452	*/
10453	iemInitExec(pVCpu, false /fBypassHandlers/);
10454
10455	/*
10456	* Switch orgy for getting to the right handler.
10457	*/
10458	VBOXSTRICTRC rcStrict;
10459	if (fRepPrefix)
10460	{
10461	switch (enmAddrMode)
10462	{
10463	case IEMMODE_16BIT:
10464	switch (cbValue)
10465	{
10466	case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10467	case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10468	case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10469	default:
10470	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10471	}
10472	break;
10473
10474	case IEMMODE_32BIT:
10475	switch (cbValue)
10476	{
10477	case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10478	case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10479	case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10480	default:
10481	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10482	}
10483	break;
10484
10485	case IEMMODE_64BIT:
10486	switch (cbValue)
10487	{
10488	case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10489	case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10490	case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10491	default:
10492	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10493	}
10494	break;
10495
10496	default:
10497	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10498	}
10499	}
10500	else
10501	{
10502	switch (enmAddrMode)
10503	{
10504	case IEMMODE_16BIT:
10505	switch (cbValue)
10506	{
10507	case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10508	case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10509	case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10510	default:
10511	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10512	}
10513	break;
10514
10515	case IEMMODE_32BIT:
10516	switch (cbValue)
10517	{
10518	case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10519	case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10520	case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10521	default:
10522	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10523	}
10524	break;
10525
10526	case IEMMODE_64BIT:
10527	switch (cbValue)
10528	{
10529	case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10530	case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10531	case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10532	default:
10533	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10534	}
10535	break;
10536
10537	default:
10538	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10539	}
10540	}
10541
10542	if (pVCpu->iem.s.cActiveMappings)
10543	iemMemRollback(pVCpu);
10544
10545	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10546	}
10547
10548
10549	/**
10550	* Interface for HM and EM for executing string I/O IN (read) instructions.
10551	*
10552	* This API ASSUMES that the caller has already verified that the guest code is
10553	* allowed to access the I/O port. (The I/O port is in the DX register in the
10554	* guest state.)
10555	*
10556	* @returns Strict VBox status code.
10557	* @param pVCpu The cross context virtual CPU structure.
10558	* @param cbValue The size of the I/O port access (1, 2, or 4).
10559	* @param enmAddrMode The addressing mode.
10560	* @param fRepPrefix Indicates whether a repeat prefix is used
10561	* (doesn't matter which for this instruction).
10562	* @param cbInstr The instruction length in bytes.
10563	* @param fIoChecked Whether the access to the I/O port has been
10564	* checked or not. It's typically checked in the
10565	* HM scenario.
10566	*/
10567	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10568	bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
10569	{
10570	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10571
10572	/*
10573	* State init.
10574	*/
10575	iemInitExec(pVCpu, false /fBypassHandlers/);
10576
10577	/*
10578	* Switch orgy for getting to the right handler.
10579	*/
10580	VBOXSTRICTRC rcStrict;
10581	if (fRepPrefix)
10582	{
10583	switch (enmAddrMode)
10584	{
10585	case IEMMODE_16BIT:
10586	switch (cbValue)
10587	{
10588	case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10589	case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10590	case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10591	default:
10592	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10593	}
10594	break;
10595
10596	case IEMMODE_32BIT:
10597	switch (cbValue)
10598	{
10599	case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10600	case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10601	case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10602	default:
10603	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10604	}
10605	break;
10606
10607	case IEMMODE_64BIT:
10608	switch (cbValue)
10609	{
10610	case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10611	case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10612	case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10613	default:
10614	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10615	}
10616	break;
10617
10618	default:
10619	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10620	}
10621	}
10622	else
10623	{
10624	switch (enmAddrMode)
10625	{
10626	case IEMMODE_16BIT:
10627	switch (cbValue)
10628	{
10629	case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10630	case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10631	case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10632	default:
10633	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10634	}
10635	break;
10636
10637	case IEMMODE_32BIT:
10638	switch (cbValue)
10639	{
10640	case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10641	case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10642	case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10643	default:
10644	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10645	}
10646	break;
10647
10648	case IEMMODE_64BIT:
10649	switch (cbValue)
10650	{
10651	case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10652	case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10653	case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10654	default:
10655	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10656	}
10657	break;
10658
10659	default:
10660	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10661	}
10662	}
10663
10664	if ( pVCpu->iem.s.cActiveMappings == 0
10665	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
10666	{ /* likely */ }
10667	else
10668	{
10669	AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
10670	iemMemRollback(pVCpu);
10671	}
10672	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10673	}
10674
10675
10676	/**
10677	* Interface for rawmode to write execute an OUT instruction.
10678	*
10679	* @returns Strict VBox status code.
10680	* @param pVCpu The cross context virtual CPU structure.
10681	* @param cbInstr The instruction length in bytes.
10682	* @param u16Port The port to read.
10683	* @param fImm Whether the port is specified using an immediate operand or
10684	* using the implicit DX register.
10685	* @param cbReg The register size.
10686	*
10687	* @remarks In ring-0 not all of the state needs to be synced in.
10688	*/
10689	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10690	{
10691	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10692	Assert(cbReg <= 4 && cbReg != 3);
10693
10694	iemInitExec(pVCpu, false /fBypassHandlers/);
10695	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
10696	Assert(!pVCpu->iem.s.cActiveMappings);
10697	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10698	}
10699
10700
10701	/**
10702	* Interface for rawmode to write execute an IN instruction.
10703	*
10704	* @returns Strict VBox status code.
10705	* @param pVCpu The cross context virtual CPU structure.
10706	* @param cbInstr The instruction length in bytes.
10707	* @param u16Port The port to read.
10708	* @param fImm Whether the port is specified using an immediate operand or
10709	* using the implicit DX.
10710	* @param cbReg The register size.
10711	*/
10712	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10713	{
10714	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10715	Assert(cbReg <= 4 && cbReg != 3);
10716
10717	iemInitExec(pVCpu, false /fBypassHandlers/);
10718	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
10719	Assert(!pVCpu->iem.s.cActiveMappings);
10720	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10721	}
10722
10723
10724	/**
10725	* Interface for HM and EM to write to a CRx register.
10726	*
10727	* @returns Strict VBox status code.
10728	* @param pVCpu The cross context virtual CPU structure.
10729	* @param cbInstr The instruction length in bytes.
10730	* @param iCrReg The control register number (destination).
10731	* @param iGReg The general purpose register number (source).
10732	*
10733	* @remarks In ring-0 not all of the state needs to be synced in.
10734	*/
10735	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
10736	{
10737	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10738	Assert(iCrReg < 16);
10739	Assert(iGReg < 16);
10740
10741	iemInitExec(pVCpu, false /fBypassHandlers/);
10742	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
10743	Assert(!pVCpu->iem.s.cActiveMappings);
10744	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10745	}
10746
10747
10748	/**
10749	* Interface for HM and EM to read from a CRx register.
10750	*
10751	* @returns Strict VBox status code.
10752	* @param pVCpu The cross context virtual CPU structure.
10753	* @param cbInstr The instruction length in bytes.
10754	* @param iGReg The general purpose register number (destination).
10755	* @param iCrReg The control register number (source).
10756	*
10757	* @remarks In ring-0 not all of the state needs to be synced in.
10758	*/
10759	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10760	{
10761	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10762	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
10763	\| CPUMCTX_EXTRN_APIC_TPR);
10764	Assert(iCrReg < 16);
10765	Assert(iGReg < 16);
10766
10767	iemInitExec(pVCpu, false /fBypassHandlers/);
10768	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
10769	Assert(!pVCpu->iem.s.cActiveMappings);
10770	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10771	}
10772
10773
10774	/**
10775	* Interface for HM and EM to clear the CR0[TS] bit.
10776	*
10777	* @returns Strict VBox status code.
10778	* @param pVCpu The cross context virtual CPU structure.
10779	* @param cbInstr The instruction length in bytes.
10780	*
10781	* @remarks In ring-0 not all of the state needs to be synced in.
10782	*/
10783	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
10784	{
10785	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10786
10787	iemInitExec(pVCpu, false /fBypassHandlers/);
10788	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
10789	Assert(!pVCpu->iem.s.cActiveMappings);
10790	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10791	}
10792
10793
10794	/**
10795	* Interface for HM and EM to emulate the LMSW instruction (loads CR0).
10796	*
10797	* @returns Strict VBox status code.
10798	* @param pVCpu The cross context virtual CPU structure.
10799	* @param cbInstr The instruction length in bytes.
10800	* @param uValue The value to load into CR0.
10801	* @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
10802	* memory operand. Otherwise pass NIL_RTGCPTR.
10803	*
10804	* @remarks In ring-0 not all of the state needs to be synced in.
10805	*/
10806	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
10807	{
10808	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10809
10810	iemInitExec(pVCpu, false /fBypassHandlers/);
10811	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
10812	Assert(!pVCpu->iem.s.cActiveMappings);
10813	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10814	}
10815
10816
10817	/**
10818	* Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
10819	*
10820	* Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
10821	*
10822	* @returns Strict VBox status code.
10823	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10824	* @param cbInstr The instruction length in bytes.
10825	* @remarks In ring-0 not all of the state needs to be synced in.
10826	* @thread EMT(pVCpu)
10827	*/
10828	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
10829	{
10830	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10831
10832	iemInitExec(pVCpu, false /fBypassHandlers/);
10833	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
10834	Assert(!pVCpu->iem.s.cActiveMappings);
10835	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10836	}
10837
10838
10839	/**
10840	* Interface for HM and EM to emulate the WBINVD instruction.
10841	*
10842	* @returns Strict VBox status code.
10843	* @param pVCpu The cross context virtual CPU structure.
10844	* @param cbInstr The instruction length in bytes.
10845	*
10846	* @remarks In ring-0 not all of the state needs to be synced in.
10847	*/
10848	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10849	{
10850	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10851
10852	iemInitExec(pVCpu, false /fBypassHandlers/);
10853	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
10854	Assert(!pVCpu->iem.s.cActiveMappings);
10855	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10856	}
10857
10858
10859	/**
10860	* Interface for HM and EM to emulate the INVD instruction.
10861	*
10862	* @returns Strict VBox status code.
10863	* @param pVCpu The cross context virtual CPU structure.
10864	* @param cbInstr The instruction length in bytes.
10865	*
10866	* @remarks In ring-0 not all of the state needs to be synced in.
10867	*/
10868	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10869	{
10870	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10871
10872	iemInitExec(pVCpu, false /fBypassHandlers/);
10873	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
10874	Assert(!pVCpu->iem.s.cActiveMappings);
10875	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10876	}
10877
10878
10879	/**
10880	* Interface for HM and EM to emulate the INVLPG instruction.
10881	*
10882	* @returns Strict VBox status code.
10883	* @retval VINF_PGM_SYNC_CR3
10884	*
10885	* @param pVCpu The cross context virtual CPU structure.
10886	* @param cbInstr The instruction length in bytes.
10887	* @param GCPtrPage The effective address of the page to invalidate.
10888	*
10889	* @remarks In ring-0 not all of the state needs to be synced in.
10890	*/
10891	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
10892	{
10893	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10894
10895	iemInitExec(pVCpu, false /fBypassHandlers/);
10896	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
10897	Assert(!pVCpu->iem.s.cActiveMappings);
10898	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10899	}
10900
10901
10902	/**
10903	* Interface for HM and EM to emulate the INVPCID instruction.
10904	*
10905	* @returns Strict VBox status code.
10906	* @retval VINF_PGM_SYNC_CR3
10907	*
10908	* @param pVCpu The cross context virtual CPU structure.
10909	* @param cbInstr The instruction length in bytes.
10910	* @param iEffSeg The effective segment register.
10911	* @param GCPtrDesc The effective address of the INVPCID descriptor.
10912	* @param uType The invalidation type.
10913	*
10914	* @remarks In ring-0 not all of the state needs to be synced in.
10915	*/
10916	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
10917	uint64_t uType)
10918	{
10919	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
10920
10921	iemInitExec(pVCpu, false /fBypassHandlers/);
10922	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
10923	Assert(!pVCpu->iem.s.cActiveMappings);
10924	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10925	}
10926
10927
10928	/**
10929	* Interface for HM and EM to emulate the CPUID instruction.
10930	*
10931	* @returns Strict VBox status code.
10932	*
10933	* @param pVCpu The cross context virtual CPU structure.
10934	* @param cbInstr The instruction length in bytes.
10935	*
10936	* @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
10937	*/
10938	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
10939	{
10940	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10941	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
10942
10943	iemInitExec(pVCpu, false /fBypassHandlers/);
10944	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
10945	Assert(!pVCpu->iem.s.cActiveMappings);
10946	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10947	}
10948
10949
10950	/**
10951	* Interface for HM and EM to emulate the RDPMC instruction.
10952	*
10953	* @returns Strict VBox status code.
10954	*
10955	* @param pVCpu The cross context virtual CPU structure.
10956	* @param cbInstr The instruction length in bytes.
10957	*
10958	* @remarks Not all of the state needs to be synced in.
10959	*/
10960	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
10961	{
10962	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10963	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
10964
10965	iemInitExec(pVCpu, false /fBypassHandlers/);
10966	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
10967	Assert(!pVCpu->iem.s.cActiveMappings);
10968	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10969	}
10970
10971
10972	/**
10973	* Interface for HM and EM to emulate the RDTSC instruction.
10974	*
10975	* @returns Strict VBox status code.
10976	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10977	*
10978	* @param pVCpu The cross context virtual CPU structure.
10979	* @param cbInstr The instruction length in bytes.
10980	*
10981	* @remarks Not all of the state needs to be synced in.
10982	*/
10983	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
10984	{
10985	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10986	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
10987
10988	iemInitExec(pVCpu, false /fBypassHandlers/);
10989	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
10990	Assert(!pVCpu->iem.s.cActiveMappings);
10991	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10992	}
10993
10994
10995	/**
10996	* Interface for HM and EM to emulate the RDTSCP instruction.
10997	*
10998	* @returns Strict VBox status code.
10999	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11000	*
11001	* @param pVCpu The cross context virtual CPU structure.
11002	* @param cbInstr The instruction length in bytes.
11003	*
11004	* @remarks Not all of the state needs to be synced in. Recommended
11005	* to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
11006	*/
11007	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
11008	{
11009	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11010	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
11011
11012	iemInitExec(pVCpu, false /fBypassHandlers/);
11013	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
11014	Assert(!pVCpu->iem.s.cActiveMappings);
11015	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11016	}
11017
11018
11019	/**
11020	* Interface for HM and EM to emulate the RDMSR instruction.
11021	*
11022	* @returns Strict VBox status code.
11023	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11024	*
11025	* @param pVCpu The cross context virtual CPU structure.
11026	* @param cbInstr The instruction length in bytes.
11027	*
11028	* @remarks Not all of the state needs to be synced in. Requires RCX and
11029	* (currently) all MSRs.
11030	*/
11031	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11032	{
11033	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11034	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_ALL_MSRS);
11035
11036	iemInitExec(pVCpu, false /fBypassHandlers/);
11037	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
11038	Assert(!pVCpu->iem.s.cActiveMappings);
11039	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11040	}
11041
11042
11043	/**
11044	* Interface for HM and EM to emulate the WRMSR instruction.
11045	*
11046	* @returns Strict VBox status code.
11047	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11048	*
11049	* @param pVCpu The cross context virtual CPU structure.
11050	* @param cbInstr The instruction length in bytes.
11051	*
11052	* @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
11053	* and (currently) all MSRs.
11054	*/
11055	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11056	{
11057	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11058	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
11059	\| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_ALL_MSRS);
11060
11061	iemInitExec(pVCpu, false /fBypassHandlers/);
11062	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
11063	Assert(!pVCpu->iem.s.cActiveMappings);
11064	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11065	}
11066
11067
11068	/**
11069	* Interface for HM and EM to emulate the MONITOR instruction.
11070	*
11071	* @returns Strict VBox status code.
11072	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11073	*
11074	* @param pVCpu The cross context virtual CPU structure.
11075	* @param cbInstr The instruction length in bytes.
11076	*
11077	* @remarks Not all of the state needs to be synced in.
11078	* @remarks ASSUMES the default segment of DS and no segment override prefixes
11079	* are used.
11080	*/
11081	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
11082	{
11083	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11084	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
11085
11086	iemInitExec(pVCpu, false /fBypassHandlers/);
11087	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
11088	Assert(!pVCpu->iem.s.cActiveMappings);
11089	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11090	}
11091
11092
11093	/**
11094	* Interface for HM and EM to emulate the MWAIT instruction.
11095	*
11096	* @returns Strict VBox status code.
11097	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11098	*
11099	* @param pVCpu The cross context virtual CPU structure.
11100	* @param cbInstr The instruction length in bytes.
11101	*
11102	* @remarks Not all of the state needs to be synced in.
11103	*/
11104	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
11105	{
11106	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11107	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX);
11108
11109	iemInitExec(pVCpu, false /fBypassHandlers/);
11110	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
11111	Assert(!pVCpu->iem.s.cActiveMappings);
11112	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11113	}
11114
11115
11116	/**
11117	* Interface for HM and EM to emulate the HLT instruction.
11118	*
11119	* @returns Strict VBox status code.
11120	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11121	*
11122	* @param pVCpu The cross context virtual CPU structure.
11123	* @param cbInstr The instruction length in bytes.
11124	*
11125	* @remarks Not all of the state needs to be synced in.
11126	*/
11127	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
11128	{
11129	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
11130
11131	iemInitExec(pVCpu, false /fBypassHandlers/);
11132	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
11133	Assert(!pVCpu->iem.s.cActiveMappings);
11134	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11135	}
11136
11137
11138	/**
11139	* Checks if IEM is in the process of delivering an event (interrupt or
11140	* exception).
11141	*
11142	* @returns true if we're in the process of raising an interrupt or exception,
11143	* false otherwise.
11144	* @param pVCpu The cross context virtual CPU structure.
11145	* @param puVector Where to store the vector associated with the
11146	* currently delivered event, optional.
11147	* @param pfFlags Where to store th event delivery flags (see
11148	* IEM_XCPT_FLAGS_XXX), optional.
11149	* @param puErr Where to store the error code associated with the
11150	* event, optional.
11151	* @param puCr2 Where to store the CR2 associated with the event,
11152	* optional.
11153	* @remarks The caller should check the flags to determine if the error code and
11154	* CR2 are valid for the event.
11155	*/
11156	VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t puVector, uint32_t pfFlags, uint32_t puErr, uint64_t puCr2)
11157	{
11158	bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
11159	if (fRaisingXcpt)
11160	{
11161	if (puVector)
11162	*puVector = pVCpu->iem.s.uCurXcpt;
11163	if (pfFlags)
11164	*pfFlags = pVCpu->iem.s.fCurXcpt;
11165	if (puErr)
11166	*puErr = pVCpu->iem.s.uCurXcptErr;
11167	if (puCr2)
11168	*puCr2 = pVCpu->iem.s.uCurXcptCr2;
11169	}
11170	return fRaisingXcpt;
11171	}
11172
11173	#ifdef IN_RING3
11174
11175	/**
11176	* Handles the unlikely and probably fatal merge cases.
11177	*
11178	* @returns Merged status code.
11179	* @param rcStrict Current EM status code.
11180	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11181	* with @a rcStrict.
11182	* @param iMemMap The memory mapping index. For error reporting only.
11183	* @param pVCpu The cross context virtual CPU structure of the calling
11184	* thread, for error reporting only.
11185	*/
11186	DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
11187	unsigned iMemMap, PVMCPUCC pVCpu)
11188	{
11189	if (RT_FAILURE_NP(rcStrict))
11190	return rcStrict;
11191
11192	if (RT_FAILURE_NP(rcStrictCommit))
11193	return rcStrictCommit;
11194
11195	if (rcStrict == rcStrictCommit)
11196	return rcStrictCommit;
11197
11198	AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
11199	VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
11200	pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
11201	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
11202	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
11203	return VERR_IOM_FF_STATUS_IPE;
11204	}
11205
11206
11207	/**
11208	* Helper for IOMR3ProcessForceFlag.
11209	*
11210	* @returns Merged status code.
11211	* @param rcStrict Current EM status code.
11212	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11213	* with @a rcStrict.
11214	* @param iMemMap The memory mapping index. For error reporting only.
11215	* @param pVCpu The cross context virtual CPU structure of the calling
11216	* thread, for error reporting only.
11217	*/
11218	DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
11219	{
11220	/* Simple. */
11221	if (RT_LIKELY(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RAW_TO_R3))
11222	return rcStrictCommit;
11223
11224	if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
11225	return rcStrict;
11226
11227	/* EM scheduling status codes. */
11228	if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
11229	&& rcStrict <= VINF_EM_LAST))
11230	{
11231	if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
11232	&& rcStrictCommit <= VINF_EM_LAST))
11233	return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
11234	}
11235
11236	/* Unlikely */
11237	return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
11238	}
11239
11240
11241	/**
11242	* Called by force-flag handling code when VMCPU_FF_IEM is set.
11243	*
11244	* @returns Merge between @a rcStrict and what the commit operation returned.
11245	* @param pVM The cross context VM structure.
11246	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11247	* @param rcStrict The status code returned by ring-0 or raw-mode.
11248	*/
11249	VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
11250	{
11251	/*
11252	* Reset the pending commit.
11253	*/
11254	AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess \| pVCpu->iem.s.aMemMappings[1].fAccess \| pVCpu->iem.s.aMemMappings[2].fAccess)
11255	& (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND),
11256	("%#x %#x %#x\n",
11257	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11258	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
11259
11260	/*
11261	* Commit the pending bounce buffers (usually just one).
11262	*/
11263	unsigned cBufs = 0;
11264	unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
11265	while (iMemMap-- > 0)
11266	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND))
11267	{
11268	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
11269	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
11270	Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
11271
11272	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
11273	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
11274	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
11275
11276	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
11277	{
11278	VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
11279	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
11280	pbBuf,
11281	cbFirst,
11282	PGMACCESSORIGIN_IEM);
11283	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
11284	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
11285	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
11286	VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
11287	}
11288
11289	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
11290	{
11291	VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
11292	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
11293	pbBuf + cbFirst,
11294	cbSecond,
11295	PGMACCESSORIGIN_IEM);
11296	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
11297	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
11298	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
11299	VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
11300	}
11301	cBufs++;
11302	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
11303	}
11304
11305	AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
11306	("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
11307	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11308	pVCpu->iem.s.cActiveMappings = 0;
11309	return rcStrict;
11310	}
11311
11312	#endif /* IN_RING3 */
11313

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 96407

Download in other formats: