VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 96745

Last change on this file since 96745 was 96745, checked in by vboxsync, 20 months ago

VMM/IEM: Some micro optimizations around iemHandleNestedInstructionBoundaryFFs. bugref:10092
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 459.4 KB
Line 
1/* $Id: IEMAll.cpp 96745 2022-09-15 11:35:07Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - All Contexts.
4 */
5
6/*
7 * Copyright (C) 2011-2022 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/** @page pg_iem IEM - Interpreted Execution Manager
30 *
31 * The interpreted exeuction manager (IEM) is for executing short guest code
32 * sequences that are causing too many exits / virtualization traps. It will
33 * also be used to interpret single instructions, thus replacing the selective
34 * interpreters in EM and IOM.
35 *
36 * Design goals:
37 * - Relatively small footprint, although we favour speed and correctness
38 * over size.
39 * - Reasonably fast.
40 * - Correctly handle lock prefixed instructions.
41 * - Complete instruction set - eventually.
42 * - Refactorable into a recompiler, maybe.
43 * - Replace EMInterpret*.
44 *
45 * Using the existing disassembler has been considered, however this is thought
46 * to conflict with speed as the disassembler chews things a bit too much while
47 * leaving us with a somewhat complicated state to interpret afterwards.
48 *
49 *
50 * The current code is very much work in progress. You've been warned!
51 *
52 *
53 * @section sec_iem_fpu_instr FPU Instructions
54 *
55 * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
56 * same or equivalent instructions on the host FPU. To make life easy, we also
57 * let the FPU prioritize the unmasked exceptions for us. This however, only
58 * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
59 * for FPU exception delivery, because with CR0.NE=0 there is a window where we
60 * can trigger spurious FPU exceptions.
61 *
62 * The guest FPU state is not loaded into the host CPU and kept there till we
63 * leave IEM because the calling conventions have declared an all year open
64 * season on much of the FPU state. For instance an innocent looking call to
65 * memcpy might end up using a whole bunch of XMM or MM registers if the
66 * particular implementation finds it worthwhile.
67 *
68 *
69 * @section sec_iem_logging Logging
70 *
71 * The IEM code uses the \"IEM\" log group for the main logging. The different
72 * logging levels/flags are generally used for the following purposes:
73 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
74 * - Flow (LogFlow) : Basic enter/exit IEM state info.
75 * - Level 2 (Log2) : ?
76 * - Level 3 (Log3) : More detailed enter/exit IEM state info.
77 * - Level 4 (Log4) : Decoding mnemonics w/ EIP.
78 * - Level 5 (Log5) : Decoding details.
79 * - Level 6 (Log6) : Enables/disables the lockstep comparison with REM.
80 * - Level 7 (Log7) : iret++ execution logging.
81 * - Level 8 (Log8) : Memory writes.
82 * - Level 9 (Log9) : Memory reads.
83 * - Level 10 (Log10): TLBs.
84 * - Level 11 (Log11): Unmasked FPU exceptions.
85 */
86
87/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
88#ifdef _MSC_VER
89# pragma warning(disable:4505)
90#endif
91
92
93/*********************************************************************************************************************************
94* Header Files *
95*********************************************************************************************************************************/
96#define LOG_GROUP LOG_GROUP_IEM
97#define VMCPU_INCL_CPUM_GST_CTX
98#include <VBox/vmm/iem.h>
99#include <VBox/vmm/cpum.h>
100#include <VBox/vmm/apic.h>
101#include <VBox/vmm/pdm.h>
102#include <VBox/vmm/pgm.h>
103#include <VBox/vmm/iom.h>
104#include <VBox/vmm/em.h>
105#include <VBox/vmm/hm.h>
106#include <VBox/vmm/nem.h>
107#include <VBox/vmm/gim.h>
108#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
109# include <VBox/vmm/em.h>
110# include <VBox/vmm/hm_svm.h>
111#endif
112#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
113# include <VBox/vmm/hmvmxinline.h>
114#endif
115#include <VBox/vmm/tm.h>
116#include <VBox/vmm/dbgf.h>
117#include <VBox/vmm/dbgftrace.h>
118#include "IEMInternal.h"
119#include <VBox/vmm/vmcc.h>
120#include <VBox/log.h>
121#include <VBox/err.h>
122#include <VBox/param.h>
123#include <VBox/dis.h>
124#include <VBox/disopcode.h>
125#include <iprt/asm-math.h>
126#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
127# include <iprt/asm-amd64-x86.h>
128#elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32)
129# include <iprt/asm-arm.h>
130#endif
131#include <iprt/assert.h>
132#include <iprt/string.h>
133#include <iprt/x86.h>
134
135#include "IEMInline.h"
136
137
138/*********************************************************************************************************************************
139* Structures and Typedefs *
140*********************************************************************************************************************************/
141/**
142 * CPU exception classes.
143 */
144typedef enum IEMXCPTCLASS
145{
146 IEMXCPTCLASS_BENIGN,
147 IEMXCPTCLASS_CONTRIBUTORY,
148 IEMXCPTCLASS_PAGE_FAULT,
149 IEMXCPTCLASS_DOUBLE_FAULT
150} IEMXCPTCLASS;
151
152
153/*********************************************************************************************************************************
154* Global Variables *
155*********************************************************************************************************************************/
156#if defined(IEM_LOG_MEMORY_WRITES)
157/** What IEM just wrote. */
158uint8_t g_abIemWrote[256];
159/** How much IEM just wrote. */
160size_t g_cbIemWrote;
161#endif
162
163
164/*********************************************************************************************************************************
165* Internal Functions *
166*********************************************************************************************************************************/
167static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
168 uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT;
169
170
171/**
172 * Initializes the decoder state.
173 *
174 * iemReInitDecoder is mostly a copy of this function.
175 *
176 * @param pVCpu The cross context virtual CPU structure of the
177 * calling thread.
178 * @param fBypassHandlers Whether to bypass access handlers.
179 * @param fDisregardLock Whether to disregard the LOCK prefix.
180 */
181DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
182{
183 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
184 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
185 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
186 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
187 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
188 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
189 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
190 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
191 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
192 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
193
194 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
195 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
196 pVCpu->iem.s.enmCpuMode = enmMode;
197 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
198 pVCpu->iem.s.enmEffAddrMode = enmMode;
199 if (enmMode != IEMMODE_64BIT)
200 {
201 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
202 pVCpu->iem.s.enmEffOpSize = enmMode;
203 }
204 else
205 {
206 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
207 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
208 }
209 pVCpu->iem.s.fPrefixes = 0;
210 pVCpu->iem.s.uRexReg = 0;
211 pVCpu->iem.s.uRexB = 0;
212 pVCpu->iem.s.uRexIndex = 0;
213 pVCpu->iem.s.idxPrefix = 0;
214 pVCpu->iem.s.uVex3rdReg = 0;
215 pVCpu->iem.s.uVexLength = 0;
216 pVCpu->iem.s.fEvexStuff = 0;
217 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
218#ifdef IEM_WITH_CODE_TLB
219 pVCpu->iem.s.pbInstrBuf = NULL;
220 pVCpu->iem.s.offInstrNextByte = 0;
221 pVCpu->iem.s.offCurInstrStart = 0;
222# ifdef VBOX_STRICT
223 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
224 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
225 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
226# endif
227#else
228 pVCpu->iem.s.offOpcode = 0;
229 pVCpu->iem.s.cbOpcode = 0;
230#endif
231 pVCpu->iem.s.offModRm = 0;
232 pVCpu->iem.s.cActiveMappings = 0;
233 pVCpu->iem.s.iNextMapping = 0;
234 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
235 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
236 pVCpu->iem.s.fDisregardLock = fDisregardLock;
237
238#ifdef DBGFTRACE_ENABLED
239 switch (enmMode)
240 {
241 case IEMMODE_64BIT:
242 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
243 break;
244 case IEMMODE_32BIT:
245 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
246 break;
247 case IEMMODE_16BIT:
248 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
249 break;
250 }
251#endif
252}
253
254
255/**
256 * Reinitializes the decoder state 2nd+ loop of IEMExecLots.
257 *
258 * This is mostly a copy of iemInitDecoder.
259 *
260 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
261 */
262DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
263{
264 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
265 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
266 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
268 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
269 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
270 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
271 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
272 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
273
274 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
275 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
276 pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
277 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
278 pVCpu->iem.s.enmEffAddrMode = enmMode;
279 if (enmMode != IEMMODE_64BIT)
280 {
281 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
282 pVCpu->iem.s.enmEffOpSize = enmMode;
283 }
284 else
285 {
286 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
287 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
288 }
289 pVCpu->iem.s.fPrefixes = 0;
290 pVCpu->iem.s.uRexReg = 0;
291 pVCpu->iem.s.uRexB = 0;
292 pVCpu->iem.s.uRexIndex = 0;
293 pVCpu->iem.s.idxPrefix = 0;
294 pVCpu->iem.s.uVex3rdReg = 0;
295 pVCpu->iem.s.uVexLength = 0;
296 pVCpu->iem.s.fEvexStuff = 0;
297 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
298#ifdef IEM_WITH_CODE_TLB
299 if (pVCpu->iem.s.pbInstrBuf)
300 {
301 uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
302 - pVCpu->iem.s.uInstrBufPc;
303 if (off < pVCpu->iem.s.cbInstrBufTotal)
304 {
305 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
306 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
307 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
308 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
309 else
310 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
311 }
312 else
313 {
314 pVCpu->iem.s.pbInstrBuf = NULL;
315 pVCpu->iem.s.offInstrNextByte = 0;
316 pVCpu->iem.s.offCurInstrStart = 0;
317 pVCpu->iem.s.cbInstrBuf = 0;
318 pVCpu->iem.s.cbInstrBufTotal = 0;
319 }
320 }
321 else
322 {
323 pVCpu->iem.s.offInstrNextByte = 0;
324 pVCpu->iem.s.offCurInstrStart = 0;
325 pVCpu->iem.s.cbInstrBuf = 0;
326 pVCpu->iem.s.cbInstrBufTotal = 0;
327 }
328#else
329 pVCpu->iem.s.cbOpcode = 0;
330 pVCpu->iem.s.offOpcode = 0;
331#endif
332 pVCpu->iem.s.offModRm = 0;
333 Assert(pVCpu->iem.s.cActiveMappings == 0);
334 pVCpu->iem.s.iNextMapping = 0;
335 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
336 Assert(pVCpu->iem.s.fBypassHandlers == false);
337
338#ifdef DBGFTRACE_ENABLED
339 switch (enmMode)
340 {
341 case IEMMODE_64BIT:
342 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
343 break;
344 case IEMMODE_32BIT:
345 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
346 break;
347 case IEMMODE_16BIT:
348 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
349 break;
350 }
351#endif
352}
353
354
355
356/**
357 * Prefetch opcodes the first time when starting executing.
358 *
359 * @returns Strict VBox status code.
360 * @param pVCpu The cross context virtual CPU structure of the
361 * calling thread.
362 * @param fBypassHandlers Whether to bypass access handlers.
363 * @param fDisregardLock Whether to disregard LOCK prefixes.
364 *
365 * @todo Combine fDisregardLock and fBypassHandlers into a flag parameter and
366 * store them as such.
367 */
368static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) RT_NOEXCEPT
369{
370 iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock);
371
372#ifdef IEM_WITH_CODE_TLB
373 /** @todo Do ITLB lookup here. */
374
375#else /* !IEM_WITH_CODE_TLB */
376
377 /*
378 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
379 *
380 * First translate CS:rIP to a physical address.
381 */
382 uint32_t cbToTryRead;
383 RTGCPTR GCPtrPC;
384 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
385 {
386 cbToTryRead = GUEST_PAGE_SIZE;
387 GCPtrPC = pVCpu->cpum.GstCtx.rip;
388 if (IEM_IS_CANONICAL(GCPtrPC))
389 cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
390 else
391 return iemRaiseGeneralProtectionFault0(pVCpu);
392 }
393 else
394 {
395 uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
396 AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
397 if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
398 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
399 else
400 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
401 if (cbToTryRead) { /* likely */ }
402 else /* overflowed */
403 {
404 Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
405 cbToTryRead = UINT32_MAX;
406 }
407 GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
408 Assert(GCPtrPC <= UINT32_MAX);
409 }
410
411 PGMPTWALK Walk;
412 int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
413 if (RT_SUCCESS(rc))
414 Assert(Walk.fSucceeded); /* probable. */
415 else
416 {
417 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
418#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
419 if (Walk.fFailed & PGM_WALKFAIL_EPT)
420 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
421#endif
422 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
423 }
424 if ((Walk.fEffective & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ }
425 else
426 {
427 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
428#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
429 if (Walk.fFailed & PGM_WALKFAIL_EPT)
430 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
431#endif
432 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
433 }
434 if (!(Walk.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
435 else
436 {
437 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
438#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
439 if (Walk.fFailed & PGM_WALKFAIL_EPT)
440 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
441#endif
442 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
443 }
444 RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
445 /** @todo Check reserved bits and such stuff. PGM is better at doing
446 * that, so do it when implementing the guest virtual address
447 * TLB... */
448
449 /*
450 * Read the bytes at this address.
451 */
452 uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
453 if (cbToTryRead > cbLeftOnPage)
454 cbToTryRead = cbLeftOnPage;
455 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
456 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
457
458 if (!pVCpu->iem.s.fBypassHandlers)
459 {
460 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
461 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
462 { /* likely */ }
463 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
464 {
465 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
466 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
467 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
468 }
469 else
470 {
471 Log((RT_SUCCESS(rcStrict)
472 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
473 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
474 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
475 return rcStrict;
476 }
477 }
478 else
479 {
480 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
481 if (RT_SUCCESS(rc))
482 { /* likely */ }
483 else
484 {
485 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
486 GCPtrPC, GCPhys, rc, cbToTryRead));
487 return rc;
488 }
489 }
490 pVCpu->iem.s.cbOpcode = cbToTryRead;
491#endif /* !IEM_WITH_CODE_TLB */
492 return VINF_SUCCESS;
493}
494
495
496/**
497 * Invalidates the IEM TLBs.
498 *
499 * This is called internally as well as by PGM when moving GC mappings.
500 *
501 * @returns
502 * @param pVCpu The cross context virtual CPU structure of the calling
503 * thread.
504 */
505VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)
506{
507#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
508 Log10(("IEMTlbInvalidateAll\n"));
509# ifdef IEM_WITH_CODE_TLB
510 pVCpu->iem.s.cbInstrBufTotal = 0;
511 pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
512 if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
513 { /* very likely */ }
514 else
515 {
516 pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
517 unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
518 while (i-- > 0)
519 pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
520 }
521# endif
522
523# ifdef IEM_WITH_DATA_TLB
524 pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
525 if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
526 { /* very likely */ }
527 else
528 {
529 pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
530 unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
531 while (i-- > 0)
532 pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
533 }
534# endif
535#else
536 RT_NOREF(pVCpu);
537#endif
538}
539
540
541/**
542 * Invalidates a page in the TLBs.
543 *
544 * @param pVCpu The cross context virtual CPU structure of the calling
545 * thread.
546 * @param GCPtr The address of the page to invalidate
547 * @thread EMT(pVCpu)
548 */
549VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
550{
551#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
552 Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));
553 GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);
554 Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));
555 uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr);
556
557# ifdef IEM_WITH_CODE_TLB
558 if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision))
559 {
560 pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
561 if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc))
562 pVCpu->iem.s.cbInstrBufTotal = 0;
563 }
564# endif
565
566# ifdef IEM_WITH_DATA_TLB
567 if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision))
568 pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
569# endif
570#else
571 NOREF(pVCpu); NOREF(GCPtr);
572#endif
573}
574
575
576#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
577/**
578 * Invalid both TLBs slow fashion following a rollover.
579 *
580 * Worker for IEMTlbInvalidateAllPhysical,
581 * IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,
582 * iemMemMapJmp and others.
583 *
584 * @thread EMT(pVCpu)
585 */
586static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu)
587{
588 Log10(("IEMTlbInvalidateAllPhysicalSlow\n"));
589 ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
590 ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
591
592 unsigned i;
593# ifdef IEM_WITH_CODE_TLB
594 i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
595 while (i-- > 0)
596 {
597 pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
598 pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ
599 | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV);
600 }
601# endif
602# ifdef IEM_WITH_DATA_TLB
603 i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
604 while (i-- > 0)
605 {
606 pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
607 pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ
608 | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV);
609 }
610# endif
611
612}
613#endif
614
615
616/**
617 * Invalidates the host physical aspects of the IEM TLBs.
618 *
619 * This is called internally as well as by PGM when moving GC mappings.
620 *
621 * @param pVCpu The cross context virtual CPU structure of the calling
622 * thread.
623 * @note Currently not used.
624 */
625VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
626{
627#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
628 /* Note! This probably won't end up looking exactly like this, but it give an idea... */
629 Log10(("IEMTlbInvalidateAllPhysical\n"));
630
631# ifdef IEM_WITH_CODE_TLB
632 pVCpu->iem.s.cbInstrBufTotal = 0;
633# endif
634 uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
635 if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))
636 {
637 pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
638 pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
639 }
640 else
641 IEMTlbInvalidateAllPhysicalSlow(pVCpu);
642#else
643 NOREF(pVCpu);
644#endif
645}
646
647
648/**
649 * Invalidates the host physical aspects of the IEM TLBs.
650 *
651 * This is called internally as well as by PGM when moving GC mappings.
652 *
653 * @param pVM The cross context VM structure.
654 * @param idCpuCaller The ID of the calling EMT if available to the caller,
655 * otherwise NIL_VMCPUID.
656 *
657 * @remarks Caller holds the PGM lock.
658 */
659VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller)
660{
661#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
662 PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);
663 if (pVCpuCaller)
664 VMCPU_ASSERT_EMT(pVCpuCaller);
665 Log10(("IEMTlbInvalidateAllPhysicalAllCpus\n"));
666
667 VMCC_FOR_EACH_VMCPU(pVM)
668 {
669# ifdef IEM_WITH_CODE_TLB
670 if (pVCpuCaller == pVCpu)
671 pVCpu->iem.s.cbInstrBufTotal = 0;
672# endif
673
674 uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);
675 uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;
676 if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))
677 { /* likely */}
678 else if (pVCpuCaller == pVCpu)
679 uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;
680 else
681 {
682 IEMTlbInvalidateAllPhysicalSlow(pVCpu);
683 continue;
684 }
685 ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
686 ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
687 }
688 VMCC_FOR_EACH_VMCPU_END(pVM);
689
690#else
691 RT_NOREF(pVM, idCpuCaller);
692#endif
693}
694
695#ifdef IEM_WITH_CODE_TLB
696
697/**
698 * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
699 * failure and jumps.
700 *
701 * We end up here for a number of reasons:
702 * - pbInstrBuf isn't yet initialized.
703 * - Advancing beyond the buffer boundrary (e.g. cross page).
704 * - Advancing beyond the CS segment limit.
705 * - Fetching from non-mappable page (e.g. MMIO).
706 *
707 * @param pVCpu The cross context virtual CPU structure of the
708 * calling thread.
709 * @param pvDst Where to return the bytes.
710 * @param cbDst Number of bytes to read.
711 *
712 * @todo Make cbDst = 0 a way of initializing pbInstrBuf?
713 */
714void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT
715{
716#ifdef IN_RING3
717 for (;;)
718 {
719 Assert(cbDst <= 8);
720 uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
721
722 /*
723 * We might have a partial buffer match, deal with that first to make the
724 * rest simpler. This is the first part of the cross page/buffer case.
725 */
726 if (pVCpu->iem.s.pbInstrBuf != NULL)
727 {
728 if (offBuf < pVCpu->iem.s.cbInstrBuf)
729 {
730 Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
731 uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
732 memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
733
734 cbDst -= cbCopy;
735 pvDst = (uint8_t *)pvDst + cbCopy;
736 offBuf += cbCopy;
737 pVCpu->iem.s.offInstrNextByte += offBuf;
738 }
739 }
740
741 /*
742 * Check segment limit, figuring how much we're allowed to access at this point.
743 *
744 * We will fault immediately if RIP is past the segment limit / in non-canonical
745 * territory. If we do continue, there are one or more bytes to read before we
746 * end up in trouble and we need to do that first before faulting.
747 */
748 RTGCPTR GCPtrFirst;
749 uint32_t cbMaxRead;
750 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
751 {
752 GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
753 if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
754 { /* likely */ }
755 else
756 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
757 cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
758 }
759 else
760 {
761 GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
762 Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
763 if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
764 { /* likely */ }
765 else
766 iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
767 cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
768 if (cbMaxRead != 0)
769 { /* likely */ }
770 else
771 {
772 /* Overflowed because address is 0 and limit is max. */
773 Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
774 cbMaxRead = X86_PAGE_SIZE;
775 }
776 GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
777 uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
778 if (cbMaxRead2 < cbMaxRead)
779 cbMaxRead = cbMaxRead2;
780 /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
781 }
782
783 /*
784 * Get the TLB entry for this piece of code.
785 */
786 uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.CodeTlb, GCPtrFirst);
787 PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag);
788 if (pTlbe->uTag == uTag)
789 {
790 /* likely when executing lots of code, otherwise unlikely */
791# ifdef VBOX_WITH_STATISTICS
792 pVCpu->iem.s.CodeTlb.cTlbHits++;
793# endif
794 }
795 else
796 {
797 pVCpu->iem.s.CodeTlb.cTlbMisses++;
798 PGMPTWALK Walk;
799 int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
800 if (RT_FAILURE(rc))
801 {
802#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
803 /** @todo Nested VMX: Need to handle EPT violation/misconfig here? */
804 Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
805#endif
806 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
807 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
808 }
809
810 AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
811 Assert(Walk.fSucceeded);
812 pTlbe->uTag = uTag;
813 pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A))
814 | (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
815 pTlbe->GCPhys = Walk.GCPhys;
816 pTlbe->pbMappingR3 = NULL;
817 }
818
819 /*
820 * Check TLB page table level access flags.
821 */
822 if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
823 {
824 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
825 {
826 Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
827 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
828 }
829 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
830 {
831 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
832 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
833 }
834 }
835
836 /*
837 * Look up the physical page info if necessary.
838 */
839 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
840 { /* not necessary */ }
841 else
842 {
843 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
844 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
845 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
846 AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
847 if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
848 { /* likely */ }
849 else
850 IEMTlbInvalidateAllPhysicalSlow(pVCpu);
851 pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
852 | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED);
853 int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
854 &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
855 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
856 }
857
858# if defined(IN_RING3) || defined(IN_RING0) /** @todo fixme */
859 /*
860 * Try do a direct read using the pbMappingR3 pointer.
861 */
862 if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
863 == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
864 {
865 uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
866 pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
867 if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
868 {
869 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
870 pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
871 }
872 else
873 {
874 uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
875 Assert(cbInstr < cbMaxRead);
876 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
877 pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
878 }
879 if (cbDst <= cbMaxRead)
880 {
881 pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
882 pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
883 pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
884 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
885 return;
886 }
887 pVCpu->iem.s.pbInstrBuf = NULL;
888
889 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
890 pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
891 }
892 else
893# endif
894#if 0
895 /*
896 * If there is no special read handling, so we can read a bit more and
897 * put it in the prefetch buffer.
898 */
899 if ( cbDst < cbMaxRead
900 && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
901 {
902 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
903 &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
904 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
905 { /* likely */ }
906 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
907 {
908 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
909 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
910 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
911 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
912 }
913 else
914 {
915 Log((RT_SUCCESS(rcStrict)
916 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
917 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
918 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
919 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
920 }
921 }
922 /*
923 * Special read handling, so only read exactly what's needed.
924 * This is a highly unlikely scenario.
925 */
926 else
927#endif
928 {
929 pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
930 uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
931 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
932 pvDst, cbToRead, PGMACCESSORIGIN_IEM);
933 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
934 { /* likely */ }
935 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
936 {
937 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
938 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
939 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
940 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
941 }
942 else
943 {
944 Log((RT_SUCCESS(rcStrict)
945 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
946 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
947 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
948 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
949 }
950 pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
951 if (cbToRead == cbDst)
952 return;
953 }
954
955 /*
956 * More to read, loop.
957 */
958 cbDst -= cbMaxRead;
959 pvDst = (uint8_t *)pvDst + cbMaxRead;
960 }
961#else
962 RT_NOREF(pvDst, cbDst);
963 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
964#endif
965}
966
967#else
968
969/**
970 * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
971 * exception if it fails.
972 *
973 * @returns Strict VBox status code.
974 * @param pVCpu The cross context virtual CPU structure of the
975 * calling thread.
976 * @param cbMin The minimum number of bytes relative offOpcode
977 * that must be read.
978 */
979VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
980{
981 /*
982 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
983 *
984 * First translate CS:rIP to a physical address.
985 */
986 uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
987 uint32_t cbToTryRead;
988 RTGCPTR GCPtrNext;
989 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
990 {
991 cbToTryRead = GUEST_PAGE_SIZE;
992 GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
993 if (!IEM_IS_CANONICAL(GCPtrNext))
994 return iemRaiseGeneralProtectionFault0(pVCpu);
995 }
996 else
997 {
998 uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
999 Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1000 GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1001 if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1002 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1003 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1004 if (!cbToTryRead) /* overflowed */
1005 {
1006 Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1007 cbToTryRead = UINT32_MAX;
1008 /** @todo check out wrapping around the code segment. */
1009 }
1010 if (cbToTryRead < cbMin - cbLeft)
1011 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1012 GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1013 }
1014
1015 /* Only read up to the end of the page, and make sure we don't read more
1016 than the opcode buffer can hold. */
1017 uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1018 if (cbToTryRead > cbLeftOnPage)
1019 cbToTryRead = cbLeftOnPage;
1020 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
1021 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
1022/** @todo r=bird: Convert assertion into undefined opcode exception? */
1023 Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1024
1025 PGMPTWALK Walk;
1026 int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
1027 if (RT_FAILURE(rc))
1028 {
1029 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1030#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1031 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1032 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1033#endif
1034 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1035 }
1036 if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
1037 {
1038 Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1039#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1040 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1041 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1042#endif
1043 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1044 }
1045 if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1046 {
1047 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1048#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1049 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1050 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1051#endif
1052 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1053 }
1054 RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1055 Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
1056 /** @todo Check reserved bits and such stuff. PGM is better at doing
1057 * that, so do it when implementing the guest virtual address
1058 * TLB... */
1059
1060 /*
1061 * Read the bytes at this address.
1062 *
1063 * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1064 * and since PATM should only patch the start of an instruction there
1065 * should be no need to check again here.
1066 */
1067 if (!pVCpu->iem.s.fBypassHandlers)
1068 {
1069 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
1070 cbToTryRead, PGMACCESSORIGIN_IEM);
1071 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1072 { /* likely */ }
1073 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1074 {
1075 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1076 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1077 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1078 }
1079 else
1080 {
1081 Log((RT_SUCCESS(rcStrict)
1082 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1083 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1084 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1085 return rcStrict;
1086 }
1087 }
1088 else
1089 {
1090 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
1091 if (RT_SUCCESS(rc))
1092 { /* likely */ }
1093 else
1094 {
1095 Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
1096 return rc;
1097 }
1098 }
1099 pVCpu->iem.s.cbOpcode += cbToTryRead;
1100 Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
1101
1102 return VINF_SUCCESS;
1103}
1104
1105#endif /* !IEM_WITH_CODE_TLB */
1106#ifndef IEM_WITH_SETJMP
1107
1108/**
1109 * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
1110 *
1111 * @returns Strict VBox status code.
1112 * @param pVCpu The cross context virtual CPU structure of the
1113 * calling thread.
1114 * @param pb Where to return the opcode byte.
1115 */
1116VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
1117{
1118 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1119 if (rcStrict == VINF_SUCCESS)
1120 {
1121 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1122 *pb = pVCpu->iem.s.abOpcode[offOpcode];
1123 pVCpu->iem.s.offOpcode = offOpcode + 1;
1124 }
1125 else
1126 *pb = 0;
1127 return rcStrict;
1128}
1129
1130#else /* IEM_WITH_SETJMP */
1131
1132/**
1133 * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
1134 *
1135 * @returns The opcode byte.
1136 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1137 */
1138uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1139{
1140# ifdef IEM_WITH_CODE_TLB
1141 uint8_t u8;
1142 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
1143 return u8;
1144# else
1145 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1146 if (rcStrict == VINF_SUCCESS)
1147 return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
1148 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1149# endif
1150}
1151
1152#endif /* IEM_WITH_SETJMP */
1153
1154#ifndef IEM_WITH_SETJMP
1155
1156/**
1157 * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
1158 *
1159 * @returns Strict VBox status code.
1160 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1161 * @param pu16 Where to return the opcode dword.
1162 */
1163VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1164{
1165 uint8_t u8;
1166 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1167 if (rcStrict == VINF_SUCCESS)
1168 *pu16 = (int8_t)u8;
1169 return rcStrict;
1170}
1171
1172
1173/**
1174 * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
1175 *
1176 * @returns Strict VBox status code.
1177 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1178 * @param pu32 Where to return the opcode dword.
1179 */
1180VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1181{
1182 uint8_t u8;
1183 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1184 if (rcStrict == VINF_SUCCESS)
1185 *pu32 = (int8_t)u8;
1186 return rcStrict;
1187}
1188
1189
1190/**
1191 * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
1192 *
1193 * @returns Strict VBox status code.
1194 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1195 * @param pu64 Where to return the opcode qword.
1196 */
1197VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1198{
1199 uint8_t u8;
1200 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1201 if (rcStrict == VINF_SUCCESS)
1202 *pu64 = (int8_t)u8;
1203 return rcStrict;
1204}
1205
1206#endif /* !IEM_WITH_SETJMP */
1207
1208
1209#ifndef IEM_WITH_SETJMP
1210
1211/**
1212 * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
1213 *
1214 * @returns Strict VBox status code.
1215 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1216 * @param pu16 Where to return the opcode word.
1217 */
1218VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1219{
1220 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1221 if (rcStrict == VINF_SUCCESS)
1222 {
1223 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1224# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1225 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1226# else
1227 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1228# endif
1229 pVCpu->iem.s.offOpcode = offOpcode + 2;
1230 }
1231 else
1232 *pu16 = 0;
1233 return rcStrict;
1234}
1235
1236#else /* IEM_WITH_SETJMP */
1237
1238/**
1239 * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
1240 *
1241 * @returns The opcode word.
1242 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1243 */
1244uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1245{
1246# ifdef IEM_WITH_CODE_TLB
1247 uint16_t u16;
1248 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
1249 return u16;
1250# else
1251 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1252 if (rcStrict == VINF_SUCCESS)
1253 {
1254 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1255 pVCpu->iem.s.offOpcode += 2;
1256# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1257 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1258# else
1259 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1260# endif
1261 }
1262 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1263# endif
1264}
1265
1266#endif /* IEM_WITH_SETJMP */
1267
1268#ifndef IEM_WITH_SETJMP
1269
1270/**
1271 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1272 *
1273 * @returns Strict VBox status code.
1274 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1275 * @param pu32 Where to return the opcode double word.
1276 */
1277VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1278{
1279 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1280 if (rcStrict == VINF_SUCCESS)
1281 {
1282 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1283 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1284 pVCpu->iem.s.offOpcode = offOpcode + 2;
1285 }
1286 else
1287 *pu32 = 0;
1288 return rcStrict;
1289}
1290
1291
1292/**
1293 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1294 *
1295 * @returns Strict VBox status code.
1296 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1297 * @param pu64 Where to return the opcode quad word.
1298 */
1299VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1300{
1301 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1302 if (rcStrict == VINF_SUCCESS)
1303 {
1304 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1305 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1306 pVCpu->iem.s.offOpcode = offOpcode + 2;
1307 }
1308 else
1309 *pu64 = 0;
1310 return rcStrict;
1311}
1312
1313#endif /* !IEM_WITH_SETJMP */
1314
1315#ifndef IEM_WITH_SETJMP
1316
1317/**
1318 * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1319 *
1320 * @returns Strict VBox status code.
1321 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1322 * @param pu32 Where to return the opcode dword.
1323 */
1324VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1325{
1326 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1327 if (rcStrict == VINF_SUCCESS)
1328 {
1329 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1330# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1331 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1332# else
1333 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1334 pVCpu->iem.s.abOpcode[offOpcode + 1],
1335 pVCpu->iem.s.abOpcode[offOpcode + 2],
1336 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1337# endif
1338 pVCpu->iem.s.offOpcode = offOpcode + 4;
1339 }
1340 else
1341 *pu32 = 0;
1342 return rcStrict;
1343}
1344
1345#else /* IEM_WITH_SETJMP */
1346
1347/**
1348 * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
1349 *
1350 * @returns The opcode dword.
1351 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1352 */
1353uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1354{
1355# ifdef IEM_WITH_CODE_TLB
1356 uint32_t u32;
1357 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
1358 return u32;
1359# else
1360 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1361 if (rcStrict == VINF_SUCCESS)
1362 {
1363 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1364 pVCpu->iem.s.offOpcode = offOpcode + 4;
1365# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1366 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1367# else
1368 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1369 pVCpu->iem.s.abOpcode[offOpcode + 1],
1370 pVCpu->iem.s.abOpcode[offOpcode + 2],
1371 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1372# endif
1373 }
1374 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1375# endif
1376}
1377
1378#endif /* IEM_WITH_SETJMP */
1379
1380#ifndef IEM_WITH_SETJMP
1381
1382/**
1383 * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1384 *
1385 * @returns Strict VBox status code.
1386 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1387 * @param pu64 Where to return the opcode dword.
1388 */
1389VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1390{
1391 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1392 if (rcStrict == VINF_SUCCESS)
1393 {
1394 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1395 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1396 pVCpu->iem.s.abOpcode[offOpcode + 1],
1397 pVCpu->iem.s.abOpcode[offOpcode + 2],
1398 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1399 pVCpu->iem.s.offOpcode = offOpcode + 4;
1400 }
1401 else
1402 *pu64 = 0;
1403 return rcStrict;
1404}
1405
1406
1407/**
1408 * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1409 *
1410 * @returns Strict VBox status code.
1411 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1412 * @param pu64 Where to return the opcode qword.
1413 */
1414VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1415{
1416 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1417 if (rcStrict == VINF_SUCCESS)
1418 {
1419 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1420 *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1421 pVCpu->iem.s.abOpcode[offOpcode + 1],
1422 pVCpu->iem.s.abOpcode[offOpcode + 2],
1423 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1424 pVCpu->iem.s.offOpcode = offOpcode + 4;
1425 }
1426 else
1427 *pu64 = 0;
1428 return rcStrict;
1429}
1430
1431#endif /* !IEM_WITH_SETJMP */
1432
1433#ifndef IEM_WITH_SETJMP
1434
1435/**
1436 * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1437 *
1438 * @returns Strict VBox status code.
1439 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1440 * @param pu64 Where to return the opcode qword.
1441 */
1442VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1443{
1444 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1445 if (rcStrict == VINF_SUCCESS)
1446 {
1447 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1448# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1449 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1450# else
1451 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1452 pVCpu->iem.s.abOpcode[offOpcode + 1],
1453 pVCpu->iem.s.abOpcode[offOpcode + 2],
1454 pVCpu->iem.s.abOpcode[offOpcode + 3],
1455 pVCpu->iem.s.abOpcode[offOpcode + 4],
1456 pVCpu->iem.s.abOpcode[offOpcode + 5],
1457 pVCpu->iem.s.abOpcode[offOpcode + 6],
1458 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1459# endif
1460 pVCpu->iem.s.offOpcode = offOpcode + 8;
1461 }
1462 else
1463 *pu64 = 0;
1464 return rcStrict;
1465}
1466
1467#else /* IEM_WITH_SETJMP */
1468
1469/**
1470 * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
1471 *
1472 * @returns The opcode qword.
1473 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1474 */
1475uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1476{
1477# ifdef IEM_WITH_CODE_TLB
1478 uint64_t u64;
1479 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
1480 return u64;
1481# else
1482 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1483 if (rcStrict == VINF_SUCCESS)
1484 {
1485 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1486 pVCpu->iem.s.offOpcode = offOpcode + 8;
1487# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1488 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1489# else
1490 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1491 pVCpu->iem.s.abOpcode[offOpcode + 1],
1492 pVCpu->iem.s.abOpcode[offOpcode + 2],
1493 pVCpu->iem.s.abOpcode[offOpcode + 3],
1494 pVCpu->iem.s.abOpcode[offOpcode + 4],
1495 pVCpu->iem.s.abOpcode[offOpcode + 5],
1496 pVCpu->iem.s.abOpcode[offOpcode + 6],
1497 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1498# endif
1499 }
1500 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1501# endif
1502}
1503
1504#endif /* IEM_WITH_SETJMP */
1505
1506
1507
1508/** @name Misc Worker Functions.
1509 * @{
1510 */
1511
1512/**
1513 * Gets the exception class for the specified exception vector.
1514 *
1515 * @returns The class of the specified exception.
1516 * @param uVector The exception vector.
1517 */
1518static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT
1519{
1520 Assert(uVector <= X86_XCPT_LAST);
1521 switch (uVector)
1522 {
1523 case X86_XCPT_DE:
1524 case X86_XCPT_TS:
1525 case X86_XCPT_NP:
1526 case X86_XCPT_SS:
1527 case X86_XCPT_GP:
1528 case X86_XCPT_SX: /* AMD only */
1529 return IEMXCPTCLASS_CONTRIBUTORY;
1530
1531 case X86_XCPT_PF:
1532 case X86_XCPT_VE: /* Intel only */
1533 return IEMXCPTCLASS_PAGE_FAULT;
1534
1535 case X86_XCPT_DF:
1536 return IEMXCPTCLASS_DOUBLE_FAULT;
1537 }
1538 return IEMXCPTCLASS_BENIGN;
1539}
1540
1541
1542/**
1543 * Evaluates how to handle an exception caused during delivery of another event
1544 * (exception / interrupt).
1545 *
1546 * @returns How to handle the recursive exception.
1547 * @param pVCpu The cross context virtual CPU structure of the
1548 * calling thread.
1549 * @param fPrevFlags The flags of the previous event.
1550 * @param uPrevVector The vector of the previous event.
1551 * @param fCurFlags The flags of the current exception.
1552 * @param uCurVector The vector of the current exception.
1553 * @param pfXcptRaiseInfo Where to store additional information about the
1554 * exception condition. Optional.
1555 */
1556VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
1557 uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
1558{
1559 /*
1560 * Only CPU exceptions can be raised while delivering other events, software interrupt
1561 * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
1562 */
1563 AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
1564 Assert(pVCpu); RT_NOREF(pVCpu);
1565 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
1566
1567 IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
1568 IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
1569 if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1570 {
1571 IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
1572 if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
1573 {
1574 IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
1575 if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
1576 && ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
1577 || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
1578 {
1579 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1580 fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
1581 : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
1582 Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
1583 uCurVector, pVCpu->cpum.GstCtx.cr2));
1584 }
1585 else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1586 && enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
1587 {
1588 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1589 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
1590 }
1591 else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
1592 && ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1593 || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
1594 {
1595 enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
1596 Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
1597 }
1598 }
1599 else
1600 {
1601 if (uPrevVector == X86_XCPT_NMI)
1602 {
1603 fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
1604 if (uCurVector == X86_XCPT_PF)
1605 {
1606 fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF;
1607 Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
1608 }
1609 }
1610 else if ( uPrevVector == X86_XCPT_AC
1611 && uCurVector == X86_XCPT_AC)
1612 {
1613 enmRaise = IEMXCPTRAISE_CPU_HANG;
1614 fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
1615 Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
1616 }
1617 }
1618 }
1619 else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
1620 {
1621 fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
1622 if (uCurVector == X86_XCPT_PF)
1623 fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF;
1624 }
1625 else
1626 {
1627 Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
1628 fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
1629 }
1630
1631 if (pfXcptRaiseInfo)
1632 *pfXcptRaiseInfo = fRaiseInfo;
1633 return enmRaise;
1634}
1635
1636
1637/**
1638 * Enters the CPU shutdown state initiated by a triple fault or other
1639 * unrecoverable conditions.
1640 *
1641 * @returns Strict VBox status code.
1642 * @param pVCpu The cross context virtual CPU structure of the
1643 * calling thread.
1644 */
1645static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT
1646{
1647 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
1648 IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
1649
1650 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
1651 {
1652 Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
1653 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
1654 }
1655
1656 RT_NOREF(pVCpu);
1657 return VINF_EM_TRIPLE_FAULT;
1658}
1659
1660
1661/**
1662 * Validates a new SS segment.
1663 *
1664 * @returns VBox strict status code.
1665 * @param pVCpu The cross context virtual CPU structure of the
1666 * calling thread.
1667 * @param NewSS The new SS selctor.
1668 * @param uCpl The CPL to load the stack for.
1669 * @param pDesc Where to return the descriptor.
1670 */
1671static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT
1672{
1673 /* Null selectors are not allowed (we're not called for dispatching
1674 interrupts with SS=0 in long mode). */
1675 if (!(NewSS & X86_SEL_MASK_OFF_RPL))
1676 {
1677 Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
1678 return iemRaiseTaskSwitchFault0(pVCpu);
1679 }
1680
1681 /** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
1682 if ((NewSS & X86_SEL_RPL) != uCpl)
1683 {
1684 Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
1685 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1686 }
1687
1688 /*
1689 * Read the descriptor.
1690 */
1691 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
1692 if (rcStrict != VINF_SUCCESS)
1693 return rcStrict;
1694
1695 /*
1696 * Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1697 */
1698 if (!pDesc->Legacy.Gen.u1DescType)
1699 {
1700 Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1701 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1702 }
1703
1704 if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1705 || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1706 {
1707 Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1708 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1709 }
1710 if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1711 {
1712 Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1713 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1714 }
1715
1716 /* Is it there? */
1717 /** @todo testcase: Is this checked before the canonical / limit check below? */
1718 if (!pDesc->Legacy.Gen.u1Present)
1719 {
1720 Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1721 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
1722 }
1723
1724 return VINF_SUCCESS;
1725}
1726
1727/** @} */
1728
1729
1730/** @name Raising Exceptions.
1731 *
1732 * @{
1733 */
1734
1735
1736/**
1737 * Loads the specified stack far pointer from the TSS.
1738 *
1739 * @returns VBox strict status code.
1740 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1741 * @param uCpl The CPL to load the stack for.
1742 * @param pSelSS Where to return the new stack segment.
1743 * @param puEsp Where to return the new stack pointer.
1744 */
1745static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT
1746{
1747 VBOXSTRICTRC rcStrict;
1748 Assert(uCpl < 4);
1749
1750 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
1751 switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
1752 {
1753 /*
1754 * 16-bit TSS (X86TSS16).
1755 */
1756 case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1757 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1758 {
1759 uint32_t off = uCpl * 4 + 2;
1760 if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1761 {
1762 /** @todo check actual access pattern here. */
1763 uint32_t u32Tmp = 0; /* gcc maybe... */
1764 rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1765 if (rcStrict == VINF_SUCCESS)
1766 {
1767 *puEsp = RT_LOWORD(u32Tmp);
1768 *pSelSS = RT_HIWORD(u32Tmp);
1769 return VINF_SUCCESS;
1770 }
1771 }
1772 else
1773 {
1774 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1775 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1776 }
1777 break;
1778 }
1779
1780 /*
1781 * 32-bit TSS (X86TSS32).
1782 */
1783 case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1784 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1785 {
1786 uint32_t off = uCpl * 8 + 4;
1787 if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1788 {
1789/** @todo check actual access pattern here. */
1790 uint64_t u64Tmp;
1791 rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1792 if (rcStrict == VINF_SUCCESS)
1793 {
1794 *puEsp = u64Tmp & UINT32_MAX;
1795 *pSelSS = (RTSEL)(u64Tmp >> 32);
1796 return VINF_SUCCESS;
1797 }
1798 }
1799 else
1800 {
1801 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1802 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1803 }
1804 break;
1805 }
1806
1807 default:
1808 AssertFailed();
1809 rcStrict = VERR_IEM_IPE_4;
1810 break;
1811 }
1812
1813 *puEsp = 0; /* make gcc happy */
1814 *pSelSS = 0; /* make gcc happy */
1815 return rcStrict;
1816}
1817
1818
1819/**
1820 * Loads the specified stack pointer from the 64-bit TSS.
1821 *
1822 * @returns VBox strict status code.
1823 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1824 * @param uCpl The CPL to load the stack for.
1825 * @param uIst The interrupt stack table index, 0 if to use uCpl.
1826 * @param puRsp Where to return the new stack pointer.
1827 */
1828static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT
1829{
1830 Assert(uCpl < 4);
1831 Assert(uIst < 8);
1832 *puRsp = 0; /* make gcc happy */
1833
1834 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
1835 AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
1836
1837 uint32_t off;
1838 if (uIst)
1839 off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
1840 else
1841 off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
1842 if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
1843 {
1844 Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
1845 return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1846 }
1847
1848 return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1849}
1850
1851
1852/**
1853 * Adjust the CPU state according to the exception being raised.
1854 *
1855 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1856 * @param u8Vector The exception that has been raised.
1857 */
1858DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
1859{
1860 switch (u8Vector)
1861 {
1862 case X86_XCPT_DB:
1863 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
1864 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
1865 break;
1866 /** @todo Read the AMD and Intel exception reference... */
1867 }
1868}
1869
1870
1871/**
1872 * Implements exceptions and interrupts for real mode.
1873 *
1874 * @returns VBox strict status code.
1875 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1876 * @param cbInstr The number of bytes to offset rIP by in the return
1877 * address.
1878 * @param u8Vector The interrupt / exception vector number.
1879 * @param fFlags The flags.
1880 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1881 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1882 */
1883static VBOXSTRICTRC
1884iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
1885 uint8_t cbInstr,
1886 uint8_t u8Vector,
1887 uint32_t fFlags,
1888 uint16_t uErr,
1889 uint64_t uCr2) RT_NOEXCEPT
1890{
1891 NOREF(uErr); NOREF(uCr2);
1892 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1893
1894 /*
1895 * Read the IDT entry.
1896 */
1897 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1898 {
1899 Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
1900 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1901 }
1902 RTFAR16 Idte;
1903 VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector);
1904 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1905 {
1906 Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
1907 return rcStrict;
1908 }
1909
1910 /*
1911 * Push the stack frame.
1912 */
1913 uint16_t *pu16Frame;
1914 uint64_t uNewRsp;
1915 rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &uNewRsp);
1916 if (rcStrict != VINF_SUCCESS)
1917 return rcStrict;
1918
1919 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
1920#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
1921 AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
1922 if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
1923 fEfl |= UINT16_C(0xf000);
1924#endif
1925 pu16Frame[2] = (uint16_t)fEfl;
1926 pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
1927 pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
1928 rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
1929 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1930 return rcStrict;
1931
1932 /*
1933 * Load the vector address into cs:ip and make exception specific state
1934 * adjustments.
1935 */
1936 pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
1937 pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
1938 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1939 pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
1940 /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1941 pVCpu->cpum.GstCtx.rip = Idte.off;
1942 fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC);
1943 IEMMISC_SET_EFL(pVCpu, fEfl);
1944
1945 /** @todo do we actually do this in real mode? */
1946 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1947 iemRaiseXcptAdjustState(pVCpu, u8Vector);
1948
1949 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1950}
1951
1952
1953/**
1954 * Loads a NULL data selector into when coming from V8086 mode.
1955 *
1956 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1957 * @param pSReg Pointer to the segment register.
1958 */
1959DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1960{
1961 pSReg->Sel = 0;
1962 pSReg->ValidSel = 0;
1963 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1964 {
1965 /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
1966 pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D;
1967 pSReg->Attr.u |= X86DESCATTR_UNUSABLE;
1968 }
1969 else
1970 {
1971 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1972 /** @todo check this on AMD-V */
1973 pSReg->u64Base = 0;
1974 pSReg->u32Limit = 0;
1975 }
1976}
1977
1978
1979/**
1980 * Loads a segment selector during a task switch in V8086 mode.
1981 *
1982 * @param pSReg Pointer to the segment register.
1983 * @param uSel The selector value to load.
1984 */
1985DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
1986{
1987 /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
1988 pSReg->Sel = uSel;
1989 pSReg->ValidSel = uSel;
1990 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1991 pSReg->u64Base = uSel << 4;
1992 pSReg->u32Limit = 0xffff;
1993 pSReg->Attr.u = 0xf3;
1994}
1995
1996
1997/**
1998 * Loads a segment selector during a task switch in protected mode.
1999 *
2000 * In this task switch scenario, we would throw \#TS exceptions rather than
2001 * \#GPs.
2002 *
2003 * @returns VBox strict status code.
2004 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2005 * @param pSReg Pointer to the segment register.
2006 * @param uSel The new selector value.
2007 *
2008 * @remarks This does _not_ handle CS or SS.
2009 * @remarks This expects pVCpu->iem.s.uCpl to be up to date.
2010 */
2011static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT
2012{
2013 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2014
2015 /* Null data selector. */
2016 if (!(uSel & X86_SEL_MASK_OFF_RPL))
2017 {
2018 iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
2019 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2020 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2021 return VINF_SUCCESS;
2022 }
2023
2024 /* Fetch the descriptor. */
2025 IEMSELDESC Desc;
2026 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
2027 if (rcStrict != VINF_SUCCESS)
2028 {
2029 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
2030 VBOXSTRICTRC_VAL(rcStrict)));
2031 return rcStrict;
2032 }
2033
2034 /* Must be a data segment or readable code segment. */
2035 if ( !Desc.Legacy.Gen.u1DescType
2036 || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2037 {
2038 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
2039 Desc.Legacy.Gen.u4Type));
2040 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2041 }
2042
2043 /* Check privileges for data segments and non-conforming code segments. */
2044 if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
2045 != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
2046 {
2047 /* The RPL and the new CPL must be less than or equal to the DPL. */
2048 if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
2049 || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
2050 {
2051 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
2052 uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
2053 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2054 }
2055 }
2056
2057 /* Is it there? */
2058 if (!Desc.Legacy.Gen.u1Present)
2059 {
2060 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
2061 return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2062 }
2063
2064 /* The base and limit. */
2065 uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2066 uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
2067
2068 /*
2069 * Ok, everything checked out fine. Now set the accessed bit before
2070 * committing the result into the registers.
2071 */
2072 if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2073 {
2074 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2075 if (rcStrict != VINF_SUCCESS)
2076 return rcStrict;
2077 Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
2078 }
2079
2080 /* Commit */
2081 pSReg->Sel = uSel;
2082 pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2083 pSReg->u32Limit = cbLimit;
2084 pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
2085 pSReg->ValidSel = uSel;
2086 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2087 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2088 pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
2089
2090 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2091 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2092 return VINF_SUCCESS;
2093}
2094
2095
2096/**
2097 * Performs a task switch.
2098 *
2099 * If the task switch is the result of a JMP, CALL or IRET instruction, the
2100 * caller is responsible for performing the necessary checks (like DPL, TSS
2101 * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
2102 * reference for JMP, CALL, IRET.
2103 *
2104 * If the task switch is the due to a software interrupt or hardware exception,
2105 * the caller is responsible for validating the TSS selector and descriptor. See
2106 * Intel Instruction reference for INT n.
2107 *
2108 * @returns VBox strict status code.
2109 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2110 * @param enmTaskSwitch The cause of the task switch.
2111 * @param uNextEip The EIP effective after the task switch.
2112 * @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
2113 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2114 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2115 * @param SelTSS The TSS selector of the new task.
2116 * @param pNewDescTSS Pointer to the new TSS descriptor.
2117 */
2118VBOXSTRICTRC
2119iemTaskSwitch(PVMCPUCC pVCpu,
2120 IEMTASKSWITCH enmTaskSwitch,
2121 uint32_t uNextEip,
2122 uint32_t fFlags,
2123 uint16_t uErr,
2124 uint64_t uCr2,
2125 RTSEL SelTSS,
2126 PIEMSELDESC pNewDescTSS) RT_NOEXCEPT
2127{
2128 Assert(!IEM_IS_REAL_MODE(pVCpu));
2129 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2130 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2131
2132 uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
2133 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2134 || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2135 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2136 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2137
2138 bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2139 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2140
2141 Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
2142 fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
2143
2144 /* Update CR2 in case it's a page-fault. */
2145 /** @todo This should probably be done much earlier in IEM/PGM. See
2146 * @bugref{5653#c49}. */
2147 if (fFlags & IEM_XCPT_FLAGS_CR2)
2148 pVCpu->cpum.GstCtx.cr2 = uCr2;
2149
2150 /*
2151 * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
2152 * subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
2153 */
2154 uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
2155 uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
2156 if (uNewTSSLimit < uNewTSSLimitMin)
2157 {
2158 Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
2159 enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
2160 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2161 }
2162
2163 /*
2164 * Task switches in VMX non-root mode always cause task switches.
2165 * The new TSS must have been read and validated (DPL, limits etc.) before a
2166 * task-switch VM-exit commences.
2167 *
2168 * See Intel spec. 25.4.2 "Treatment of Task Switches".
2169 */
2170 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
2171 {
2172 Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
2173 IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
2174 }
2175
2176 /*
2177 * The SVM nested-guest intercept for task-switch takes priority over all exceptions
2178 * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
2179 */
2180 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
2181 {
2182 uint32_t const uExitInfo1 = SelTSS;
2183 uint32_t uExitInfo2 = uErr;
2184 switch (enmTaskSwitch)
2185 {
2186 case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break;
2187 case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break;
2188 default: break;
2189 }
2190 if (fFlags & IEM_XCPT_FLAGS_ERR)
2191 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
2192 if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
2193 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
2194
2195 Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
2196 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
2197 RT_NOREF2(uExitInfo1, uExitInfo2);
2198 }
2199
2200 /*
2201 * Check the current TSS limit. The last written byte to the current TSS during the
2202 * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
2203 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2204 *
2205 * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
2206 * end up with smaller than "legal" TSS limits.
2207 */
2208 uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
2209 uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
2210 if (uCurTSSLimit < uCurTSSLimitMin)
2211 {
2212 Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
2213 enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
2214 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2215 }
2216
2217 /*
2218 * Verify that the new TSS can be accessed and map it. Map only the required contents
2219 * and not the entire TSS.
2220 */
2221 void *pvNewTSS;
2222 uint32_t const cbNewTSS = uNewTSSLimitMin + 1;
2223 RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
2224 AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
2225 /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
2226 * not perform correct translation if this happens. See Intel spec. 7.2.1
2227 * "Task-State Segment". */
2228 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW, 0);
2229 if (rcStrict != VINF_SUCCESS)
2230 {
2231 Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
2232 cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
2233 return rcStrict;
2234 }
2235
2236 /*
2237 * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
2238 */
2239 uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
2240 if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
2241 || enmTaskSwitch == IEMTASKSWITCH_IRET)
2242 {
2243 PX86DESC pDescCurTSS;
2244 rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX,
2245 pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2246 if (rcStrict != VINF_SUCCESS)
2247 {
2248 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2249 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2250 return rcStrict;
2251 }
2252
2253 pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2254 rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
2255 if (rcStrict != VINF_SUCCESS)
2256 {
2257 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2258 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2259 return rcStrict;
2260 }
2261
2262 /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
2263 if (enmTaskSwitch == IEMTASKSWITCH_IRET)
2264 {
2265 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2266 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2267 u32EFlags &= ~X86_EFL_NT;
2268 }
2269 }
2270
2271 /*
2272 * Save the CPU state into the current TSS.
2273 */
2274 RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
2275 if (GCPtrNewTSS == GCPtrCurTSS)
2276 {
2277 Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
2278 Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
2279 pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
2280 pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
2281 pVCpu->cpum.GstCtx.ldtr.Sel));
2282 }
2283 if (fIsNewTSS386)
2284 {
2285 /*
2286 * Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
2287 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2288 */
2289 void *pvCurTSS32;
2290 uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
2291 uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
2292 AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
2293 rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2294 if (rcStrict != VINF_SUCCESS)
2295 {
2296 Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2297 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2298 return rcStrict;
2299 }
2300
2301 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2302 PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
2303 pCurTSS32->eip = uNextEip;
2304 pCurTSS32->eflags = u32EFlags;
2305 pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
2306 pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
2307 pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
2308 pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
2309 pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
2310 pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
2311 pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
2312 pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
2313 pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
2314 pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
2315 pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
2316 pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
2317 pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
2318 pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
2319
2320 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
2321 if (rcStrict != VINF_SUCCESS)
2322 {
2323 Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2324 VBOXSTRICTRC_VAL(rcStrict)));
2325 return rcStrict;
2326 }
2327 }
2328 else
2329 {
2330 /*
2331 * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
2332 */
2333 void *pvCurTSS16;
2334 uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
2335 uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
2336 AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
2337 rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2338 if (rcStrict != VINF_SUCCESS)
2339 {
2340 Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2341 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2342 return rcStrict;
2343 }
2344
2345 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2346 PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
2347 pCurTSS16->ip = uNextEip;
2348 pCurTSS16->flags = u32EFlags;
2349 pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
2350 pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
2351 pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
2352 pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
2353 pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
2354 pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
2355 pCurTSS16->si = pVCpu->cpum.GstCtx.si;
2356 pCurTSS16->di = pVCpu->cpum.GstCtx.di;
2357 pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
2358 pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
2359 pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
2360 pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
2361
2362 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
2363 if (rcStrict != VINF_SUCCESS)
2364 {
2365 Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2366 VBOXSTRICTRC_VAL(rcStrict)));
2367 return rcStrict;
2368 }
2369 }
2370
2371 /*
2372 * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
2373 */
2374 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2375 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2376 {
2377 /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
2378 PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
2379 pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
2380 }
2381
2382 /*
2383 * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
2384 * it's done further below with error handling (e.g. CR3 changes will go through PGM).
2385 */
2386 uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
2387 uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
2388 bool fNewDebugTrap;
2389 if (fIsNewTSS386)
2390 {
2391 PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
2392 uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
2393 uNewEip = pNewTSS32->eip;
2394 uNewEflags = pNewTSS32->eflags;
2395 uNewEax = pNewTSS32->eax;
2396 uNewEcx = pNewTSS32->ecx;
2397 uNewEdx = pNewTSS32->edx;
2398 uNewEbx = pNewTSS32->ebx;
2399 uNewEsp = pNewTSS32->esp;
2400 uNewEbp = pNewTSS32->ebp;
2401 uNewEsi = pNewTSS32->esi;
2402 uNewEdi = pNewTSS32->edi;
2403 uNewES = pNewTSS32->es;
2404 uNewCS = pNewTSS32->cs;
2405 uNewSS = pNewTSS32->ss;
2406 uNewDS = pNewTSS32->ds;
2407 uNewFS = pNewTSS32->fs;
2408 uNewGS = pNewTSS32->gs;
2409 uNewLdt = pNewTSS32->selLdt;
2410 fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
2411 }
2412 else
2413 {
2414 PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
2415 uNewCr3 = 0;
2416 uNewEip = pNewTSS16->ip;
2417 uNewEflags = pNewTSS16->flags;
2418 uNewEax = UINT32_C(0xffff0000) | pNewTSS16->ax;
2419 uNewEcx = UINT32_C(0xffff0000) | pNewTSS16->cx;
2420 uNewEdx = UINT32_C(0xffff0000) | pNewTSS16->dx;
2421 uNewEbx = UINT32_C(0xffff0000) | pNewTSS16->bx;
2422 uNewEsp = UINT32_C(0xffff0000) | pNewTSS16->sp;
2423 uNewEbp = UINT32_C(0xffff0000) | pNewTSS16->bp;
2424 uNewEsi = UINT32_C(0xffff0000) | pNewTSS16->si;
2425 uNewEdi = UINT32_C(0xffff0000) | pNewTSS16->di;
2426 uNewES = pNewTSS16->es;
2427 uNewCS = pNewTSS16->cs;
2428 uNewSS = pNewTSS16->ss;
2429 uNewDS = pNewTSS16->ds;
2430 uNewFS = 0;
2431 uNewGS = 0;
2432 uNewLdt = pNewTSS16->selLdt;
2433 fNewDebugTrap = false;
2434 }
2435
2436 if (GCPtrNewTSS == GCPtrCurTSS)
2437 Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
2438 uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
2439
2440 /*
2441 * We're done accessing the new TSS.
2442 */
2443 rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
2444 if (rcStrict != VINF_SUCCESS)
2445 {
2446 Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
2447 return rcStrict;
2448 }
2449
2450 /*
2451 * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
2452 */
2453 if (enmTaskSwitch != IEMTASKSWITCH_IRET)
2454 {
2455 rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX,
2456 pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2457 if (rcStrict != VINF_SUCCESS)
2458 {
2459 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2460 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2461 return rcStrict;
2462 }
2463
2464 /* Check that the descriptor indicates the new TSS is available (not busy). */
2465 AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2466 || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
2467 ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
2468
2469 pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2470 rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
2471 if (rcStrict != VINF_SUCCESS)
2472 {
2473 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2474 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2475 return rcStrict;
2476 }
2477 }
2478
2479 /*
2480 * From this point on, we're technically in the new task. We will defer exceptions
2481 * until the completion of the task switch but before executing any instructions in the new task.
2482 */
2483 pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
2484 pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
2485 pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
2486 pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
2487 pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
2488 pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
2489 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
2490
2491 /* Set the busy bit in TR. */
2492 pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2493
2494 /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
2495 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2496 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2497 {
2498 uNewEflags |= X86_EFL_NT;
2499 }
2500
2501 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
2502 pVCpu->cpum.GstCtx.cr0 |= X86_CR0_TS;
2503 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
2504
2505 pVCpu->cpum.GstCtx.eip = uNewEip;
2506 pVCpu->cpum.GstCtx.eax = uNewEax;
2507 pVCpu->cpum.GstCtx.ecx = uNewEcx;
2508 pVCpu->cpum.GstCtx.edx = uNewEdx;
2509 pVCpu->cpum.GstCtx.ebx = uNewEbx;
2510 pVCpu->cpum.GstCtx.esp = uNewEsp;
2511 pVCpu->cpum.GstCtx.ebp = uNewEbp;
2512 pVCpu->cpum.GstCtx.esi = uNewEsi;
2513 pVCpu->cpum.GstCtx.edi = uNewEdi;
2514
2515 uNewEflags &= X86_EFL_LIVE_MASK;
2516 uNewEflags |= X86_EFL_RA1_MASK;
2517 IEMMISC_SET_EFL(pVCpu, uNewEflags);
2518
2519 /*
2520 * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
2521 * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
2522 * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
2523 */
2524 pVCpu->cpum.GstCtx.es.Sel = uNewES;
2525 pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
2526
2527 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2528 pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
2529
2530 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2531 pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
2532
2533 pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
2534 pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
2535
2536 pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
2537 pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
2538
2539 pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
2540 pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
2541 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2542
2543 pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
2544 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
2545 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
2546 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
2547
2548 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2549 {
2550 pVCpu->cpum.GstCtx.es.Attr.u |= X86DESCATTR_UNUSABLE;
2551 pVCpu->cpum.GstCtx.cs.Attr.u |= X86DESCATTR_UNUSABLE;
2552 pVCpu->cpum.GstCtx.ss.Attr.u |= X86DESCATTR_UNUSABLE;
2553 pVCpu->cpum.GstCtx.ds.Attr.u |= X86DESCATTR_UNUSABLE;
2554 pVCpu->cpum.GstCtx.fs.Attr.u |= X86DESCATTR_UNUSABLE;
2555 pVCpu->cpum.GstCtx.gs.Attr.u |= X86DESCATTR_UNUSABLE;
2556 pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE;
2557 }
2558
2559 /*
2560 * Switch CR3 for the new task.
2561 */
2562 if ( fIsNewTSS386
2563 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
2564 {
2565 /** @todo Should we update and flush TLBs only if CR3 value actually changes? */
2566 int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
2567 AssertRCSuccessReturn(rc, rc);
2568
2569 /* Inform PGM. */
2570 /** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
2571 rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
2572 AssertRCReturn(rc, rc);
2573 /* ignore informational status codes */
2574
2575 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
2576 }
2577
2578 /*
2579 * Switch LDTR for the new task.
2580 */
2581 if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
2582 iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
2583 else
2584 {
2585 Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
2586
2587 IEMSELDESC DescNewLdt;
2588 rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
2589 if (rcStrict != VINF_SUCCESS)
2590 {
2591 Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
2592 uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
2593 return rcStrict;
2594 }
2595 if ( !DescNewLdt.Legacy.Gen.u1Present
2596 || DescNewLdt.Legacy.Gen.u1DescType
2597 || DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
2598 {
2599 Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
2600 uNewLdt, DescNewLdt.Legacy.u));
2601 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
2602 }
2603
2604 pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
2605 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
2606 pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
2607 pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
2608 pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
2609 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2610 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
2611 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
2612 }
2613
2614 IEMSELDESC DescSS;
2615 if (IEM_IS_V86_MODE(pVCpu))
2616 {
2617 pVCpu->iem.s.uCpl = 3;
2618 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
2619 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
2620 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
2621 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
2622 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
2623 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
2624
2625 /* Quick fix: fake DescSS. */ /** @todo fix the code further down? */
2626 DescSS.Legacy.u = 0;
2627 DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
2628 DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
2629 DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
2630 DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
2631 DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
2632 DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2633 DescSS.Legacy.Gen.u2Dpl = 3;
2634 }
2635 else
2636 {
2637 uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
2638
2639 /*
2640 * Load the stack segment for the new task.
2641 */
2642 if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
2643 {
2644 Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
2645 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2646 }
2647
2648 /* Fetch the descriptor. */
2649 rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
2650 if (rcStrict != VINF_SUCCESS)
2651 {
2652 Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
2653 VBOXSTRICTRC_VAL(rcStrict)));
2654 return rcStrict;
2655 }
2656
2657 /* SS must be a data segment and writable. */
2658 if ( !DescSS.Legacy.Gen.u1DescType
2659 || (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2660 || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
2661 {
2662 Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
2663 uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
2664 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2665 }
2666
2667 /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
2668 if ( (uNewSS & X86_SEL_RPL) != uNewCpl
2669 || DescSS.Legacy.Gen.u2Dpl != uNewCpl)
2670 {
2671 Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
2672 uNewCpl));
2673 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2674 }
2675
2676 /* Is it there? */
2677 if (!DescSS.Legacy.Gen.u1Present)
2678 {
2679 Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
2680 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2681 }
2682
2683 uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
2684 uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
2685
2686 /* Set the accessed bit before committing the result into SS. */
2687 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2688 {
2689 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
2690 if (rcStrict != VINF_SUCCESS)
2691 return rcStrict;
2692 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
2693 }
2694
2695 /* Commit SS. */
2696 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2697 pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
2698 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
2699 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
2700 pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
2701 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2702 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
2703
2704 /* CPL has changed, update IEM before loading rest of segments. */
2705 pVCpu->iem.s.uCpl = uNewCpl;
2706
2707 /*
2708 * Load the data segments for the new task.
2709 */
2710 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
2711 if (rcStrict != VINF_SUCCESS)
2712 return rcStrict;
2713 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
2714 if (rcStrict != VINF_SUCCESS)
2715 return rcStrict;
2716 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
2717 if (rcStrict != VINF_SUCCESS)
2718 return rcStrict;
2719 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
2720 if (rcStrict != VINF_SUCCESS)
2721 return rcStrict;
2722
2723 /*
2724 * Load the code segment for the new task.
2725 */
2726 if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
2727 {
2728 Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
2729 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2730 }
2731
2732 /* Fetch the descriptor. */
2733 IEMSELDESC DescCS;
2734 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
2735 if (rcStrict != VINF_SUCCESS)
2736 {
2737 Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
2738 return rcStrict;
2739 }
2740
2741 /* CS must be a code segment. */
2742 if ( !DescCS.Legacy.Gen.u1DescType
2743 || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2744 {
2745 Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
2746 DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
2747 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2748 }
2749
2750 /* For conforming CS, DPL must be less than or equal to the RPL. */
2751 if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2752 && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
2753 {
2754 Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
2755 DescCS.Legacy.Gen.u2Dpl));
2756 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2757 }
2758
2759 /* For non-conforming CS, DPL must match RPL. */
2760 if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2761 && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
2762 {
2763 Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
2764 DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
2765 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2766 }
2767
2768 /* Is it there? */
2769 if (!DescCS.Legacy.Gen.u1Present)
2770 {
2771 Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
2772 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2773 }
2774
2775 cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
2776 u64Base = X86DESC_BASE(&DescCS.Legacy);
2777
2778 /* Set the accessed bit before committing the result into CS. */
2779 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2780 {
2781 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
2782 if (rcStrict != VINF_SUCCESS)
2783 return rcStrict;
2784 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
2785 }
2786
2787 /* Commit CS. */
2788 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2789 pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
2790 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
2791 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
2792 pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2793 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2794 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
2795 }
2796
2797 /** @todo Debug trap. */
2798 if (fIsNewTSS386 && fNewDebugTrap)
2799 Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
2800
2801 /*
2802 * Construct the error code masks based on what caused this task switch.
2803 * See Intel Instruction reference for INT.
2804 */
2805 uint16_t uExt;
2806 if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
2807 && ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2808 || (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
2809 {
2810 uExt = 1;
2811 }
2812 else
2813 uExt = 0;
2814
2815 /*
2816 * Push any error code on to the new stack.
2817 */
2818 if (fFlags & IEM_XCPT_FLAGS_ERR)
2819 {
2820 Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
2821 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
2822 uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
2823
2824 /* Check that there is sufficient space on the stack. */
2825 /** @todo Factor out segment limit checking for normal/expand down segments
2826 * into a separate function. */
2827 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
2828 {
2829 if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
2830 || pVCpu->cpum.GstCtx.esp < cbStackFrame)
2831 {
2832 /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2833 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
2834 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2835 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2836 }
2837 }
2838 else
2839 {
2840 if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2841 || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
2842 {
2843 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
2844 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2845 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2846 }
2847 }
2848
2849
2850 if (fIsNewTSS386)
2851 rcStrict = iemMemStackPushU32(pVCpu, uErr);
2852 else
2853 rcStrict = iemMemStackPushU16(pVCpu, uErr);
2854 if (rcStrict != VINF_SUCCESS)
2855 {
2856 Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
2857 fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
2858 return rcStrict;
2859 }
2860 }
2861
2862 /* Check the new EIP against the new CS limit. */
2863 if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
2864 {
2865 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
2866 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
2867 /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2868 return iemRaiseGeneralProtectionFault(pVCpu, uExt);
2869 }
2870
2871 Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
2872 pVCpu->cpum.GstCtx.ss.Sel));
2873 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2874}
2875
2876
2877/**
2878 * Implements exceptions and interrupts for protected mode.
2879 *
2880 * @returns VBox strict status code.
2881 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2882 * @param cbInstr The number of bytes to offset rIP by in the return
2883 * address.
2884 * @param u8Vector The interrupt / exception vector number.
2885 * @param fFlags The flags.
2886 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2887 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2888 */
2889static VBOXSTRICTRC
2890iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
2891 uint8_t cbInstr,
2892 uint8_t u8Vector,
2893 uint32_t fFlags,
2894 uint16_t uErr,
2895 uint64_t uCr2) RT_NOEXCEPT
2896{
2897 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2898
2899 /*
2900 * Read the IDT entry.
2901 */
2902 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
2903 {
2904 Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
2905 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2906 }
2907 X86DESC Idte;
2908 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
2909 pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
2910 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2911 {
2912 Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
2913 return rcStrict;
2914 }
2915 Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
2916 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
2917 Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
2918
2919 /*
2920 * Check the descriptor type, DPL and such.
2921 * ASSUMES this is done in the same order as described for call-gate calls.
2922 */
2923 if (Idte.Gate.u1DescType)
2924 {
2925 Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2926 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2927 }
2928 bool fTaskGate = false;
2929 uint8_t f32BitGate = true;
2930 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
2931 switch (Idte.Gate.u4Type)
2932 {
2933 case X86_SEL_TYPE_SYS_UNDEFINED:
2934 case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
2935 case X86_SEL_TYPE_SYS_LDT:
2936 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
2937 case X86_SEL_TYPE_SYS_286_CALL_GATE:
2938 case X86_SEL_TYPE_SYS_UNDEFINED2:
2939 case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
2940 case X86_SEL_TYPE_SYS_UNDEFINED3:
2941 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
2942 case X86_SEL_TYPE_SYS_386_CALL_GATE:
2943 case X86_SEL_TYPE_SYS_UNDEFINED4:
2944 {
2945 /** @todo check what actually happens when the type is wrong...
2946 * esp. call gates. */
2947 Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2948 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2949 }
2950
2951 case X86_SEL_TYPE_SYS_286_INT_GATE:
2952 f32BitGate = false;
2953 RT_FALL_THRU();
2954 case X86_SEL_TYPE_SYS_386_INT_GATE:
2955 fEflToClear |= X86_EFL_IF;
2956 break;
2957
2958 case X86_SEL_TYPE_SYS_TASK_GATE:
2959 fTaskGate = true;
2960#ifndef IEM_IMPLEMENTS_TASKSWITCH
2961 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
2962#endif
2963 break;
2964
2965 case X86_SEL_TYPE_SYS_286_TRAP_GATE:
2966 f32BitGate = false;
2967 case X86_SEL_TYPE_SYS_386_TRAP_GATE:
2968 break;
2969
2970 IEM_NOT_REACHED_DEFAULT_CASE_RET();
2971 }
2972
2973 /* Check DPL against CPL if applicable. */
2974 if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
2975 {
2976 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
2977 {
2978 Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
2979 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2980 }
2981 }
2982
2983 /* Is it there? */
2984 if (!Idte.Gate.u1Present)
2985 {
2986 Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
2987 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2988 }
2989
2990 /* Is it a task-gate? */
2991 if (fTaskGate)
2992 {
2993 /*
2994 * Construct the error code masks based on what caused this task switch.
2995 * See Intel Instruction reference for INT.
2996 */
2997 uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2998 && !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
2999 uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
3000 RTSEL SelTSS = Idte.Gate.u16Sel;
3001
3002 /*
3003 * Fetch the TSS descriptor in the GDT.
3004 */
3005 IEMSELDESC DescTSS;
3006 rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt);
3007 if (rcStrict != VINF_SUCCESS)
3008 {
3009 Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
3010 VBOXSTRICTRC_VAL(rcStrict)));
3011 return rcStrict;
3012 }
3013
3014 /* The TSS descriptor must be a system segment and be available (not busy). */
3015 if ( DescTSS.Legacy.Gen.u1DescType
3016 || ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
3017 && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
3018 {
3019 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
3020 u8Vector, SelTSS, DescTSS.Legacy.au64));
3021 return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt);
3022 }
3023
3024 /* The TSS must be present. */
3025 if (!DescTSS.Legacy.Gen.u1Present)
3026 {
3027 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
3028 return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt);
3029 }
3030
3031 /* Do the actual task switch. */
3032 return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
3033 (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
3034 fFlags, uErr, uCr2, SelTSS, &DescTSS);
3035 }
3036
3037 /* A null CS is bad. */
3038 RTSEL NewCS = Idte.Gate.u16Sel;
3039 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3040 {
3041 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3042 return iemRaiseGeneralProtectionFault0(pVCpu);
3043 }
3044
3045 /* Fetch the descriptor for the new CS. */
3046 IEMSELDESC DescCS;
3047 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
3048 if (rcStrict != VINF_SUCCESS)
3049 {
3050 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3051 return rcStrict;
3052 }
3053
3054 /* Must be a code segment. */
3055 if (!DescCS.Legacy.Gen.u1DescType)
3056 {
3057 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3058 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3059 }
3060 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3061 {
3062 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3063 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3064 }
3065
3066 /* Don't allow lowering the privilege level. */
3067 /** @todo Does the lowering of privileges apply to software interrupts
3068 * only? This has bearings on the more-privileged or
3069 * same-privilege stack behavior further down. A testcase would
3070 * be nice. */
3071 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3072 {
3073 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3074 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3075 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3076 }
3077
3078 /* Make sure the selector is present. */
3079 if (!DescCS.Legacy.Gen.u1Present)
3080 {
3081 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3082 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3083 }
3084
3085 /* Check the new EIP against the new CS limit. */
3086 uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
3087 || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
3088 ? Idte.Gate.u16OffsetLow
3089 : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
3090 uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3091 if (uNewEip > cbLimitCS)
3092 {
3093 Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
3094 u8Vector, uNewEip, cbLimitCS, NewCS));
3095 return iemRaiseGeneralProtectionFault(pVCpu, 0);
3096 }
3097 Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
3098
3099 /* Calc the flag image to push. */
3100 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3101 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
3102 fEfl &= ~X86_EFL_RF;
3103 else
3104 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
3105
3106 /* From V8086 mode only go to CPL 0. */
3107 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3108 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3109 if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
3110 {
3111 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
3112 return iemRaiseGeneralProtectionFault(pVCpu, 0);
3113 }
3114
3115 /*
3116 * If the privilege level changes, we need to get a new stack from the TSS.
3117 * This in turns means validating the new SS and ESP...
3118 */
3119 if (uNewCpl != pVCpu->iem.s.uCpl)
3120 {
3121 RTSEL NewSS;
3122 uint32_t uNewEsp;
3123 rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
3124 if (rcStrict != VINF_SUCCESS)
3125 return rcStrict;
3126
3127 IEMSELDESC DescSS;
3128 rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
3129 if (rcStrict != VINF_SUCCESS)
3130 return rcStrict;
3131 /* If the new SS is 16-bit, we are only going to use SP, not ESP. */
3132 if (!DescSS.Legacy.Gen.u1DefBig)
3133 {
3134 Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
3135 uNewEsp = (uint16_t)uNewEsp;
3136 }
3137
3138 Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3139
3140 /* Check that there is sufficient space for the stack frame. */
3141 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3142 uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
3143 ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
3144 : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
3145
3146 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
3147 {
3148 if ( uNewEsp - 1 > cbLimitSS
3149 || uNewEsp < cbStackFrame)
3150 {
3151 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
3152 u8Vector, NewSS, uNewEsp, cbStackFrame));
3153 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3154 }
3155 }
3156 else
3157 {
3158 if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
3159 || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
3160 {
3161 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
3162 u8Vector, NewSS, uNewEsp, cbStackFrame));
3163 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3164 }
3165 }
3166
3167 /*
3168 * Start making changes.
3169 */
3170
3171 /* Set the new CPL so that stack accesses use it. */
3172 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3173 pVCpu->iem.s.uCpl = uNewCpl;
3174
3175 /* Create the stack frame. */
3176 RTPTRUNION uStackFrame;
3177 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3178 uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy),
3179 IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3180 if (rcStrict != VINF_SUCCESS)
3181 return rcStrict;
3182 void * const pvStackFrame = uStackFrame.pv;
3183 if (f32BitGate)
3184 {
3185 if (fFlags & IEM_XCPT_FLAGS_ERR)
3186 *uStackFrame.pu32++ = uErr;
3187 uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3188 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
3189 uStackFrame.pu32[2] = fEfl;
3190 uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
3191 uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
3192 Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3193 if (fEfl & X86_EFL_VM)
3194 {
3195 uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
3196 uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
3197 uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
3198 uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
3199 uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
3200 }
3201 }
3202 else
3203 {
3204 if (fFlags & IEM_XCPT_FLAGS_ERR)
3205 *uStackFrame.pu16++ = uErr;
3206 uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3207 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
3208 uStackFrame.pu16[2] = fEfl;
3209 uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
3210 uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
3211 Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
3212 if (fEfl & X86_EFL_VM)
3213 {
3214 uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
3215 uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
3216 uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
3217 uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
3218 uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
3219 }
3220 }
3221 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
3222 if (rcStrict != VINF_SUCCESS)
3223 return rcStrict;
3224
3225 /* Mark the selectors 'accessed' (hope this is the correct time). */
3226 /** @todo testcase: excatly _when_ are the accessed bits set - before or
3227 * after pushing the stack frame? (Write protect the gdt + stack to
3228 * find out.) */
3229 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3230 {
3231 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3232 if (rcStrict != VINF_SUCCESS)
3233 return rcStrict;
3234 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3235 }
3236
3237 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3238 {
3239 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
3240 if (rcStrict != VINF_SUCCESS)
3241 return rcStrict;
3242 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3243 }
3244
3245 /*
3246 * Start comitting the register changes (joins with the DPL=CPL branch).
3247 */
3248 pVCpu->cpum.GstCtx.ss.Sel = NewSS;
3249 pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
3250 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3251 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
3252 pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3253 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3254 /** @todo When coming from 32-bit code and operating with a 16-bit TSS and
3255 * 16-bit handler, the high word of ESP remains unchanged (i.e. only
3256 * SP is loaded).
3257 * Need to check the other combinations too:
3258 * - 16-bit TSS, 32-bit handler
3259 * - 32-bit TSS, 16-bit handler */
3260 if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3261 pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
3262 else
3263 pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
3264
3265 if (fEfl & X86_EFL_VM)
3266 {
3267 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
3268 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
3269 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
3270 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
3271 }
3272 }
3273 /*
3274 * Same privilege, no stack change and smaller stack frame.
3275 */
3276 else
3277 {
3278 uint64_t uNewRsp;
3279 RTPTRUNION uStackFrame;
3280 uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
3281 rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1, &uStackFrame.pv, &uNewRsp);
3282 if (rcStrict != VINF_SUCCESS)
3283 return rcStrict;
3284 void * const pvStackFrame = uStackFrame.pv;
3285
3286 if (f32BitGate)
3287 {
3288 if (fFlags & IEM_XCPT_FLAGS_ERR)
3289 *uStackFrame.pu32++ = uErr;
3290 uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3291 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
3292 uStackFrame.pu32[2] = fEfl;
3293 }
3294 else
3295 {
3296 if (fFlags & IEM_XCPT_FLAGS_ERR)
3297 *uStackFrame.pu16++ = uErr;
3298 uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3299 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
3300 uStackFrame.pu16[2] = fEfl;
3301 }
3302 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
3303 if (rcStrict != VINF_SUCCESS)
3304 return rcStrict;
3305
3306 /* Mark the CS selector as 'accessed'. */
3307 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3308 {
3309 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3310 if (rcStrict != VINF_SUCCESS)
3311 return rcStrict;
3312 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3313 }
3314
3315 /*
3316 * Start committing the register changes (joins with the other branch).
3317 */
3318 pVCpu->cpum.GstCtx.rsp = uNewRsp;
3319 }
3320
3321 /* ... register committing continues. */
3322 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
3323 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
3324 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3325 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3326 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3327 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3328
3329 pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
3330 fEfl &= ~fEflToClear;
3331 IEMMISC_SET_EFL(pVCpu, fEfl);
3332
3333 if (fFlags & IEM_XCPT_FLAGS_CR2)
3334 pVCpu->cpum.GstCtx.cr2 = uCr2;
3335
3336 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3337 iemRaiseXcptAdjustState(pVCpu, u8Vector);
3338
3339 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3340}
3341
3342
3343/**
3344 * Implements exceptions and interrupts for long mode.
3345 *
3346 * @returns VBox strict status code.
3347 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3348 * @param cbInstr The number of bytes to offset rIP by in the return
3349 * address.
3350 * @param u8Vector The interrupt / exception vector number.
3351 * @param fFlags The flags.
3352 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3353 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3354 */
3355static VBOXSTRICTRC
3356iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
3357 uint8_t cbInstr,
3358 uint8_t u8Vector,
3359 uint32_t fFlags,
3360 uint16_t uErr,
3361 uint64_t uCr2) RT_NOEXCEPT
3362{
3363 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3364
3365 /*
3366 * Read the IDT entry.
3367 */
3368 uint16_t offIdt = (uint16_t)u8Vector << 4;
3369 if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
3370 {
3371 Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3372 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3373 }
3374 X86DESC64 Idte;
3375#ifdef _MSC_VER /* Shut up silly compiler warning. */
3376 Idte.au64[0] = 0;
3377 Idte.au64[1] = 0;
3378#endif
3379 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
3380 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3381 rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
3382 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3383 {
3384 Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3385 return rcStrict;
3386 }
3387 Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
3388 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3389 Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3390
3391 /*
3392 * Check the descriptor type, DPL and such.
3393 * ASSUMES this is done in the same order as described for call-gate calls.
3394 */
3395 if (Idte.Gate.u1DescType)
3396 {
3397 Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3398 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3399 }
3400 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
3401 switch (Idte.Gate.u4Type)
3402 {
3403 case AMD64_SEL_TYPE_SYS_INT_GATE:
3404 fEflToClear |= X86_EFL_IF;
3405 break;
3406 case AMD64_SEL_TYPE_SYS_TRAP_GATE:
3407 break;
3408
3409 default:
3410 Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3411 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3412 }
3413
3414 /* Check DPL against CPL if applicable. */
3415 if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
3416 {
3417 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
3418 {
3419 Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
3420 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3421 }
3422 }
3423
3424 /* Is it there? */
3425 if (!Idte.Gate.u1Present)
3426 {
3427 Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
3428 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3429 }
3430
3431 /* A null CS is bad. */
3432 RTSEL NewCS = Idte.Gate.u16Sel;
3433 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3434 {
3435 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3436 return iemRaiseGeneralProtectionFault0(pVCpu);
3437 }
3438
3439 /* Fetch the descriptor for the new CS. */
3440 IEMSELDESC DescCS;
3441 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
3442 if (rcStrict != VINF_SUCCESS)
3443 {
3444 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3445 return rcStrict;
3446 }
3447
3448 /* Must be a 64-bit code segment. */
3449 if (!DescCS.Long.Gen.u1DescType)
3450 {
3451 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3452 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3453 }
3454 if ( !DescCS.Long.Gen.u1Long
3455 || DescCS.Long.Gen.u1DefBig
3456 || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
3457 {
3458 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
3459 u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
3460 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3461 }
3462
3463 /* Don't allow lowering the privilege level. For non-conforming CS
3464 selectors, the CS.DPL sets the privilege level the trap/interrupt
3465 handler runs at. For conforming CS selectors, the CPL remains
3466 unchanged, but the CS.DPL must be <= CPL. */
3467 /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
3468 * when CPU in Ring-0. Result \#GP? */
3469 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3470 {
3471 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3472 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3473 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3474 }
3475
3476
3477 /* Make sure the selector is present. */
3478 if (!DescCS.Legacy.Gen.u1Present)
3479 {
3480 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3481 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3482 }
3483
3484 /* Check that the new RIP is canonical. */
3485 uint64_t const uNewRip = Idte.Gate.u16OffsetLow
3486 | ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
3487 | ((uint64_t)Idte.Gate.u32OffsetTop << 32);
3488 if (!IEM_IS_CANONICAL(uNewRip))
3489 {
3490 Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
3491 return iemRaiseGeneralProtectionFault0(pVCpu);
3492 }
3493
3494 /*
3495 * If the privilege level changes or if the IST isn't zero, we need to get
3496 * a new stack from the TSS.
3497 */
3498 uint64_t uNewRsp;
3499 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3500 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3501 if ( uNewCpl != pVCpu->iem.s.uCpl
3502 || Idte.Gate.u3IST != 0)
3503 {
3504 rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
3505 if (rcStrict != VINF_SUCCESS)
3506 return rcStrict;
3507 }
3508 else
3509 uNewRsp = pVCpu->cpum.GstCtx.rsp;
3510 uNewRsp &= ~(uint64_t)0xf;
3511
3512 /*
3513 * Calc the flag image to push.
3514 */
3515 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3516 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
3517 fEfl &= ~X86_EFL_RF;
3518 else
3519 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
3520
3521 /*
3522 * Start making changes.
3523 */
3524 /* Set the new CPL so that stack accesses use it. */
3525 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3526 pVCpu->iem.s.uCpl = uNewCpl;
3527
3528 /* Create the stack frame. */
3529 uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
3530 RTPTRUNION uStackFrame;
3531 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3532 uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3533 if (rcStrict != VINF_SUCCESS)
3534 return rcStrict;
3535 void * const pvStackFrame = uStackFrame.pv;
3536
3537 if (fFlags & IEM_XCPT_FLAGS_ERR)
3538 *uStackFrame.pu64++ = uErr;
3539 uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
3540 uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */
3541 uStackFrame.pu64[2] = fEfl;
3542 uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
3543 uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
3544 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
3545 if (rcStrict != VINF_SUCCESS)
3546 return rcStrict;
3547
3548 /* Mark the CS selectors 'accessed' (hope this is the correct time). */
3549 /** @todo testcase: excatly _when_ are the accessed bits set - before or
3550 * after pushing the stack frame? (Write protect the gdt + stack to
3551 * find out.) */
3552 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3553 {
3554 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3555 if (rcStrict != VINF_SUCCESS)
3556 return rcStrict;
3557 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3558 }
3559
3560 /*
3561 * Start comitting the register changes.
3562 */
3563 /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
3564 * hidden registers when interrupting 32-bit or 16-bit code! */
3565 if (uNewCpl != uOldCpl)
3566 {
3567 pVCpu->cpum.GstCtx.ss.Sel = 0 | uNewCpl;
3568 pVCpu->cpum.GstCtx.ss.ValidSel = 0 | uNewCpl;
3569 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3570 pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3571 pVCpu->cpum.GstCtx.ss.u64Base = 0;
3572 pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE;
3573 }
3574 pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
3575 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
3576 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
3577 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3578 pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
3579 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3580 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3581 pVCpu->cpum.GstCtx.rip = uNewRip;
3582
3583 fEfl &= ~fEflToClear;
3584 IEMMISC_SET_EFL(pVCpu, fEfl);
3585
3586 if (fFlags & IEM_XCPT_FLAGS_CR2)
3587 pVCpu->cpum.GstCtx.cr2 = uCr2;
3588
3589 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3590 iemRaiseXcptAdjustState(pVCpu, u8Vector);
3591
3592 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3593}
3594
3595
3596/**
3597 * Implements exceptions and interrupts.
3598 *
3599 * All exceptions and interrupts goes thru this function!
3600 *
3601 * @returns VBox strict status code.
3602 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3603 * @param cbInstr The number of bytes to offset rIP by in the return
3604 * address.
3605 * @param u8Vector The interrupt / exception vector number.
3606 * @param fFlags The flags.
3607 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3608 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3609 */
3610VBOXSTRICTRC
3611iemRaiseXcptOrInt(PVMCPUCC pVCpu,
3612 uint8_t cbInstr,
3613 uint8_t u8Vector,
3614 uint32_t fFlags,
3615 uint16_t uErr,
3616 uint64_t uCr2) RT_NOEXCEPT
3617{
3618 /*
3619 * Get all the state that we might need here.
3620 */
3621 IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3622 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3623
3624#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
3625 /*
3626 * Flush prefetch buffer
3627 */
3628 pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3629#endif
3630
3631 /*
3632 * Perform the V8086 IOPL check and upgrade the fault without nesting.
3633 */
3634 if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
3635 && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
3636 && (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
3637 | IEM_XCPT_FLAGS_BP_INSTR
3638 | IEM_XCPT_FLAGS_ICEBP_INSTR
3639 | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
3640 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
3641 {
3642 Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
3643 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
3644 u8Vector = X86_XCPT_GP;
3645 uErr = 0;
3646 }
3647#ifdef DBGFTRACE_ENABLED
3648 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
3649 pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
3650 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
3651#endif
3652
3653 /*
3654 * Evaluate whether NMI blocking should be in effect.
3655 * Normally, NMI blocking is in effect whenever we inject an NMI.
3656 */
3657 bool fBlockNmi;
3658 if ( u8Vector == X86_XCPT_NMI
3659 && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3660 fBlockNmi = true;
3661 else
3662 fBlockNmi = false;
3663
3664#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3665 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3666 {
3667 VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
3668 if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3669 return rcStrict0;
3670
3671 /* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
3672 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
3673 {
3674 Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
3675 fBlockNmi = false;
3676 }
3677 }
3678#endif
3679
3680#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3681 if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
3682 {
3683 /*
3684 * If the event is being injected as part of VMRUN, it isn't subject to event
3685 * intercepts in the nested-guest. However, secondary exceptions that occur
3686 * during injection of any event -are- subject to exception intercepts.
3687 *
3688 * See AMD spec. 15.20 "Event Injection".
3689 */
3690 if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
3691 pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
3692 else
3693 {
3694 /*
3695 * Check and handle if the event being raised is intercepted.
3696 */
3697 VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
3698 if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
3699 return rcStrict0;
3700 }
3701 }
3702#endif
3703
3704 /*
3705 * Set NMI blocking if necessary.
3706 */
3707 if ( fBlockNmi
3708 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3709 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3710
3711 /*
3712 * Do recursion accounting.
3713 */
3714 uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
3715 uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
3716 if (pVCpu->iem.s.cXcptRecursions == 0)
3717 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
3718 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
3719 else
3720 {
3721 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
3722 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
3723 pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
3724
3725 if (pVCpu->iem.s.cXcptRecursions >= 4)
3726 {
3727#ifdef DEBUG_bird
3728 AssertFailed();
3729#endif
3730 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
3731 }
3732
3733 /*
3734 * Evaluate the sequence of recurring events.
3735 */
3736 IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
3737 NULL /* pXcptRaiseInfo */);
3738 if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
3739 { /* likely */ }
3740 else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
3741 {
3742 Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
3743 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
3744 u8Vector = X86_XCPT_DF;
3745 uErr = 0;
3746#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3747 /* VMX nested-guest #DF intercept needs to be checked here. */
3748 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3749 {
3750 VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
3751 if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3752 return rcStrict0;
3753 }
3754#endif
3755 /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
3756 if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
3757 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
3758 }
3759 else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
3760 {
3761 Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
3762 return iemInitiateCpuShutdown(pVCpu);
3763 }
3764 else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
3765 {
3766 /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
3767 Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
3768 if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
3769 && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
3770 return VERR_EM_GUEST_CPU_HANG;
3771 }
3772 else
3773 {
3774 AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
3775 enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
3776 return VERR_IEM_IPE_9;
3777 }
3778
3779 /*
3780 * The 'EXT' bit is set when an exception occurs during deliver of an external
3781 * event (such as an interrupt or earlier exception)[1]. Privileged software
3782 * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
3783 * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
3784 *
3785 * [1] - Intel spec. 6.13 "Error Code"
3786 * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
3787 * [3] - Intel Instruction reference for INT n.
3788 */
3789 if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR))
3790 && (fFlags & IEM_XCPT_FLAGS_ERR)
3791 && u8Vector != X86_XCPT_PF
3792 && u8Vector != X86_XCPT_DF)
3793 {
3794 uErr |= X86_TRAP_ERR_EXTERNAL;
3795 }
3796 }
3797
3798 pVCpu->iem.s.cXcptRecursions++;
3799 pVCpu->iem.s.uCurXcpt = u8Vector;
3800 pVCpu->iem.s.fCurXcpt = fFlags;
3801 pVCpu->iem.s.uCurXcptErr = uErr;
3802 pVCpu->iem.s.uCurXcptCr2 = uCr2;
3803
3804 /*
3805 * Extensive logging.
3806 */
3807#if defined(LOG_ENABLED) && defined(IN_RING3)
3808 if (LogIs3Enabled())
3809 {
3810 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
3811 PVM pVM = pVCpu->CTX_SUFF(pVM);
3812 char szRegs[4096];
3813 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
3814 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
3815 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
3816 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
3817 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
3818 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
3819 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
3820 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
3821 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
3822 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
3823 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
3824 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
3825 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
3826 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
3827 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
3828 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
3829 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
3830 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
3831 " efer=%016VR{efer}\n"
3832 " pat=%016VR{pat}\n"
3833 " sf_mask=%016VR{sf_mask}\n"
3834 "krnl_gs_base=%016VR{krnl_gs_base}\n"
3835 " lstar=%016VR{lstar}\n"
3836 " star=%016VR{star} cstar=%016VR{cstar}\n"
3837 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
3838 );
3839
3840 char szInstr[256];
3841 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
3842 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
3843 szInstr, sizeof(szInstr), NULL);
3844 Log3(("%s%s\n", szRegs, szInstr));
3845 }
3846#endif /* LOG_ENABLED */
3847
3848 /*
3849 * Stats.
3850 */
3851 if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3852 STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; });
3853 else if (u8Vector <= X86_XCPT_LAST)
3854 {
3855 STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]);
3856 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector),
3857 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, ASMReadTSC());
3858 }
3859
3860 /*
3861 * #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3)
3862 * to ensure that a stale TLB or paging cache entry will only cause one
3863 * spurious #PF.
3864 */
3865 if ( u8Vector == X86_XCPT_PF
3866 && (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_CR2))
3867 IEMTlbInvalidatePage(pVCpu, uCr2);
3868
3869 /*
3870 * Call the mode specific worker function.
3871 */
3872 VBOXSTRICTRC rcStrict;
3873 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
3874 rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3875 else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
3876 rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3877 else
3878 rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3879
3880 /* Flush the prefetch buffer. */
3881#ifdef IEM_WITH_CODE_TLB
3882 pVCpu->iem.s.pbInstrBuf = NULL;
3883#else
3884 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
3885#endif
3886
3887 /*
3888 * Unwind.
3889 */
3890 pVCpu->iem.s.cXcptRecursions--;
3891 pVCpu->iem.s.uCurXcpt = uPrevXcpt;
3892 pVCpu->iem.s.fCurXcpt = fPrevXcpt;
3893 Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
3894 VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
3895 pVCpu->iem.s.cXcptRecursions + 1));
3896 return rcStrict;
3897}
3898
3899#ifdef IEM_WITH_SETJMP
3900/**
3901 * See iemRaiseXcptOrInt. Will not return.
3902 */
3903DECL_NO_RETURN(void)
3904iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
3905 uint8_t cbInstr,
3906 uint8_t u8Vector,
3907 uint32_t fFlags,
3908 uint16_t uErr,
3909 uint64_t uCr2) RT_NOEXCEPT
3910{
3911 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3912 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3913}
3914#endif
3915
3916
3917/** \#DE - 00. */
3918VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT
3919{
3920 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3921}
3922
3923
3924/** \#DB - 01.
3925 * @note This automatically clear DR7.GD. */
3926VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT
3927{
3928 /** @todo set/clear RF. */
3929 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3930 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3931}
3932
3933
3934/** \#BR - 05. */
3935VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT
3936{
3937 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3938}
3939
3940
3941/** \#UD - 06. */
3942VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT
3943{
3944 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3945}
3946
3947
3948/** \#NM - 07. */
3949VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT
3950{
3951 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3952}
3953
3954
3955/** \#TS(err) - 0a. */
3956VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3957{
3958 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
3959}
3960
3961
3962/** \#TS(tr) - 0a. */
3963VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT
3964{
3965 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
3966 pVCpu->cpum.GstCtx.tr.Sel, 0);
3967}
3968
3969
3970/** \#TS(0) - 0a. */
3971VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
3972{
3973 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
3974 0, 0);
3975}
3976
3977
3978/** \#TS(err) - 0a. */
3979VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3980{
3981 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
3982 uSel & X86_SEL_MASK_OFF_RPL, 0);
3983}
3984
3985
3986/** \#NP(err) - 0b. */
3987VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3988{
3989 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
3990}
3991
3992
3993/** \#NP(sel) - 0b. */
3994VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3995{
3996 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
3997 uSel & ~X86_SEL_RPL, 0);
3998}
3999
4000
4001/** \#SS(seg) - 0c. */
4002VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4003{
4004 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
4005 uSel & ~X86_SEL_RPL, 0);
4006}
4007
4008
4009/** \#SS(err) - 0c. */
4010VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4011{
4012 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
4013}
4014
4015
4016/** \#GP(n) - 0d. */
4017VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4018{
4019 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
4020}
4021
4022
4023/** \#GP(0) - 0d. */
4024VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
4025{
4026 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4027}
4028
4029#ifdef IEM_WITH_SETJMP
4030/** \#GP(0) - 0d. */
4031DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4032{
4033 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4034}
4035#endif
4036
4037
4038/** \#GP(sel) - 0d. */
4039VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4040{
4041 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
4042 Sel & ~X86_SEL_RPL, 0);
4043}
4044
4045
4046/** \#GP(0) - 0d. */
4047VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT
4048{
4049 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4050}
4051
4052
4053/** \#GP(sel) - 0d. */
4054VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4055{
4056 NOREF(iSegReg); NOREF(fAccess);
4057 return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4058 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4059}
4060
4061#ifdef IEM_WITH_SETJMP
4062/** \#GP(sel) - 0d, longjmp. */
4063DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4064{
4065 NOREF(iSegReg); NOREF(fAccess);
4066 iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4067 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4068}
4069#endif
4070
4071/** \#GP(sel) - 0d. */
4072VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4073{
4074 NOREF(Sel);
4075 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4076}
4077
4078#ifdef IEM_WITH_SETJMP
4079/** \#GP(sel) - 0d, longjmp. */
4080DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4081{
4082 NOREF(Sel);
4083 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4084}
4085#endif
4086
4087
4088/** \#GP(sel) - 0d. */
4089VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4090{
4091 NOREF(iSegReg); NOREF(fAccess);
4092 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4093}
4094
4095#ifdef IEM_WITH_SETJMP
4096/** \#GP(sel) - 0d, longjmp. */
4097DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4098{
4099 NOREF(iSegReg); NOREF(fAccess);
4100 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4101}
4102#endif
4103
4104
4105/** \#PF(n) - 0e. */
4106VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4107{
4108 uint16_t uErr;
4109 switch (rc)
4110 {
4111 case VERR_PAGE_NOT_PRESENT:
4112 case VERR_PAGE_TABLE_NOT_PRESENT:
4113 case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
4114 case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
4115 uErr = 0;
4116 break;
4117
4118 default:
4119 AssertMsgFailed(("%Rrc\n", rc));
4120 RT_FALL_THRU();
4121 case VERR_ACCESS_DENIED:
4122 uErr = X86_TRAP_PF_P;
4123 break;
4124
4125 /** @todo reserved */
4126 }
4127
4128 if (pVCpu->iem.s.uCpl == 3)
4129 uErr |= X86_TRAP_PF_US;
4130
4131 if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
4132 && ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
4133 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
4134 uErr |= X86_TRAP_PF_ID;
4135
4136#if 0 /* This is so much non-sense, really. Why was it done like that? */
4137 /* Note! RW access callers reporting a WRITE protection fault, will clear
4138 the READ flag before calling. So, read-modify-write accesses (RW)
4139 can safely be reported as READ faults. */
4140 if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
4141 uErr |= X86_TRAP_PF_RW;
4142#else
4143 if (fAccess & IEM_ACCESS_TYPE_WRITE)
4144 {
4145 /// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
4146 /// (regardless of outcome of the comparison in the latter case).
4147 //if (!(fAccess & IEM_ACCESS_TYPE_READ))
4148 uErr |= X86_TRAP_PF_RW;
4149 }
4150#endif
4151
4152 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2,
4153 uErr, GCPtrWhere);
4154}
4155
4156#ifdef IEM_WITH_SETJMP
4157/** \#PF(n) - 0e, longjmp. */
4158DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4159{
4160 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
4161}
4162#endif
4163
4164
4165/** \#MF(0) - 10. */
4166VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu)
4167{
4168 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4169}
4170
4171
4172/** \#AC(0) - 11. */
4173VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
4174{
4175 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
4176}
4177
4178#ifdef IEM_WITH_SETJMP
4179/** \#AC(0) - 11, longjmp. */
4180DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4181{
4182 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu)));
4183}
4184#endif
4185
4186
4187/** \#XF(0)/\#XM(0) - 19. */
4188VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT
4189{
4190 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4191}
4192
4193
4194/** Accessed via IEMOP_RAISE_DIVIDE_ERROR. */
4195IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
4196{
4197 NOREF(cbInstr);
4198 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4199}
4200
4201
4202/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */
4203IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
4204{
4205 NOREF(cbInstr);
4206 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4207}
4208
4209
4210/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */
4211IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
4212{
4213 NOREF(cbInstr);
4214 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4215}
4216
4217
4218/** @} */
4219
4220/** @name Common opcode decoders.
4221 * @{
4222 */
4223//#include <iprt/mem.h>
4224
4225/**
4226 * Used to add extra details about a stub case.
4227 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4228 */
4229void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT
4230{
4231#if defined(LOG_ENABLED) && defined(IN_RING3)
4232 PVM pVM = pVCpu->CTX_SUFF(pVM);
4233 char szRegs[4096];
4234 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4235 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4236 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4237 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4238 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4239 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4240 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4241 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4242 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4243 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4244 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4245 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4246 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4247 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4248 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4249 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4250 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4251 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4252 " efer=%016VR{efer}\n"
4253 " pat=%016VR{pat}\n"
4254 " sf_mask=%016VR{sf_mask}\n"
4255 "krnl_gs_base=%016VR{krnl_gs_base}\n"
4256 " lstar=%016VR{lstar}\n"
4257 " star=%016VR{star} cstar=%016VR{cstar}\n"
4258 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4259 );
4260
4261 char szInstr[256];
4262 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4263 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
4264 szInstr, sizeof(szInstr), NULL);
4265
4266 RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
4267#else
4268 RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
4269#endif
4270}
4271
4272/** @} */
4273
4274
4275
4276/** @name Register Access.
4277 * @{
4278 */
4279
4280/**
4281 * Adds a 8-bit signed jump offset to RIP/EIP/IP.
4282 *
4283 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4284 * segment limit.
4285 *
4286 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4287 * @param offNextInstr The offset of the next instruction.
4288 */
4289VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT
4290{
4291 switch (pVCpu->iem.s.enmEffOpSize)
4292 {
4293 case IEMMODE_16BIT:
4294 {
4295 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4296 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4297 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4298 return iemRaiseGeneralProtectionFault0(pVCpu);
4299 pVCpu->cpum.GstCtx.rip = uNewIp;
4300 break;
4301 }
4302
4303 case IEMMODE_32BIT:
4304 {
4305 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4306 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4307
4308 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4309 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4310 return iemRaiseGeneralProtectionFault0(pVCpu);
4311 pVCpu->cpum.GstCtx.rip = uNewEip;
4312 break;
4313 }
4314
4315 case IEMMODE_64BIT:
4316 {
4317 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4318
4319 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4320 if (!IEM_IS_CANONICAL(uNewRip))
4321 return iemRaiseGeneralProtectionFault0(pVCpu);
4322 pVCpu->cpum.GstCtx.rip = uNewRip;
4323 break;
4324 }
4325
4326 IEM_NOT_REACHED_DEFAULT_CASE_RET();
4327 }
4328
4329 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4330
4331#ifndef IEM_WITH_CODE_TLB
4332 /* Flush the prefetch buffer. */
4333 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4334#endif
4335
4336 return VINF_SUCCESS;
4337}
4338
4339
4340/**
4341 * Adds a 16-bit signed jump offset to RIP/EIP/IP.
4342 *
4343 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4344 * segment limit.
4345 *
4346 * @returns Strict VBox status code.
4347 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4348 * @param offNextInstr The offset of the next instruction.
4349 */
4350VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT
4351{
4352 Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
4353
4354 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4355 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4356 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4357 return iemRaiseGeneralProtectionFault0(pVCpu);
4358 /** @todo Test 16-bit jump in 64-bit mode. possible? */
4359 pVCpu->cpum.GstCtx.rip = uNewIp;
4360 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4361
4362#ifndef IEM_WITH_CODE_TLB
4363 /* Flush the prefetch buffer. */
4364 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4365#endif
4366
4367 return VINF_SUCCESS;
4368}
4369
4370
4371/**
4372 * Adds a 32-bit signed jump offset to RIP/EIP/IP.
4373 *
4374 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4375 * segment limit.
4376 *
4377 * @returns Strict VBox status code.
4378 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4379 * @param offNextInstr The offset of the next instruction.
4380 */
4381VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT
4382{
4383 Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
4384
4385 if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
4386 {
4387 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4388
4389 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4390 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4391 return iemRaiseGeneralProtectionFault0(pVCpu);
4392 pVCpu->cpum.GstCtx.rip = uNewEip;
4393 }
4394 else
4395 {
4396 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4397
4398 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4399 if (!IEM_IS_CANONICAL(uNewRip))
4400 return iemRaiseGeneralProtectionFault0(pVCpu);
4401 pVCpu->cpum.GstCtx.rip = uNewRip;
4402 }
4403 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4404
4405#ifndef IEM_WITH_CODE_TLB
4406 /* Flush the prefetch buffer. */
4407 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4408#endif
4409
4410 return VINF_SUCCESS;
4411}
4412
4413
4414/**
4415 * Performs a near jump to the specified address.
4416 *
4417 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4418 * segment limit.
4419 *
4420 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4421 * @param uNewRip The new RIP value.
4422 */
4423VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
4424{
4425 switch (pVCpu->iem.s.enmEffOpSize)
4426 {
4427 case IEMMODE_16BIT:
4428 {
4429 Assert(uNewRip <= UINT16_MAX);
4430 if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
4431 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4432 return iemRaiseGeneralProtectionFault0(pVCpu);
4433 /** @todo Test 16-bit jump in 64-bit mode. */
4434 pVCpu->cpum.GstCtx.rip = uNewRip;
4435 break;
4436 }
4437
4438 case IEMMODE_32BIT:
4439 {
4440 Assert(uNewRip <= UINT32_MAX);
4441 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4442 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4443
4444 if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
4445 return iemRaiseGeneralProtectionFault0(pVCpu);
4446 pVCpu->cpum.GstCtx.rip = uNewRip;
4447 break;
4448 }
4449
4450 case IEMMODE_64BIT:
4451 {
4452 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4453
4454 if (!IEM_IS_CANONICAL(uNewRip))
4455 return iemRaiseGeneralProtectionFault0(pVCpu);
4456 pVCpu->cpum.GstCtx.rip = uNewRip;
4457 break;
4458 }
4459
4460 IEM_NOT_REACHED_DEFAULT_CASE_RET();
4461 }
4462
4463 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4464
4465#ifndef IEM_WITH_CODE_TLB
4466 /* Flush the prefetch buffer. */
4467 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4468#endif
4469
4470 return VINF_SUCCESS;
4471}
4472
4473/** @} */
4474
4475
4476/** @name FPU access and helpers.
4477 *
4478 * @{
4479 */
4480
4481/**
4482 * Updates the x87.DS and FPUDP registers.
4483 *
4484 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4485 * @param pFpuCtx The FPU context.
4486 * @param iEffSeg The effective segment register.
4487 * @param GCPtrEff The effective address relative to @a iEffSeg.
4488 */
4489DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
4490{
4491 RTSEL sel;
4492 switch (iEffSeg)
4493 {
4494 case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
4495 case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
4496 case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
4497 case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
4498 case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
4499 case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
4500 default:
4501 AssertMsgFailed(("%d\n", iEffSeg));
4502 sel = pVCpu->cpum.GstCtx.ds.Sel;
4503 }
4504 /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
4505 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4506 {
4507 pFpuCtx->DS = 0;
4508 pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
4509 }
4510 else if (!IEM_IS_LONG_MODE(pVCpu))
4511 {
4512 pFpuCtx->DS = sel;
4513 pFpuCtx->FPUDP = GCPtrEff;
4514 }
4515 else
4516 *(uint64_t *)&pFpuCtx->FPUDP = GCPtrEff;
4517}
4518
4519
4520/**
4521 * Rotates the stack registers in the push direction.
4522 *
4523 * @param pFpuCtx The FPU context.
4524 * @remarks This is a complete waste of time, but fxsave stores the registers in
4525 * stack order.
4526 */
4527DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
4528{
4529 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
4530 pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
4531 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
4532 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
4533 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
4534 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
4535 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
4536 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
4537 pFpuCtx->aRegs[0].r80 = r80Tmp;
4538}
4539
4540
4541/**
4542 * Rotates the stack registers in the pop direction.
4543 *
4544 * @param pFpuCtx The FPU context.
4545 * @remarks This is a complete waste of time, but fxsave stores the registers in
4546 * stack order.
4547 */
4548DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
4549{
4550 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
4551 pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
4552 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
4553 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
4554 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
4555 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
4556 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
4557 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
4558 pFpuCtx->aRegs[7].r80 = r80Tmp;
4559}
4560
4561
4562/**
4563 * Updates FSW and pushes a FPU result onto the FPU stack if no pending
4564 * exception prevents it.
4565 *
4566 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4567 * @param pResult The FPU operation result to push.
4568 * @param pFpuCtx The FPU context.
4569 */
4570static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4571{
4572 /* Update FSW and bail if there are pending exceptions afterwards. */
4573 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4574 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
4575 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
4576 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
4577 {
4578 if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES))
4579 Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n",
4580 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4581 pFpuCtx->FSW = fFsw;
4582 return;
4583 }
4584
4585 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4586 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4587 {
4588 /* All is fine, push the actual value. */
4589 pFpuCtx->FTW |= RT_BIT(iNewTop);
4590 pFpuCtx->aRegs[7].r80 = pResult->r80Result;
4591 }
4592 else if (pFpuCtx->FCW & X86_FCW_IM)
4593 {
4594 /* Masked stack overflow, push QNaN. */
4595 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
4596 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4597 }
4598 else
4599 {
4600 /* Raise stack overflow, don't push anything. */
4601 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
4602 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
4603 Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4604 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4605 return;
4606 }
4607
4608 fFsw &= ~X86_FSW_TOP_MASK;
4609 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
4610 pFpuCtx->FSW = fFsw;
4611
4612 iemFpuRotateStackPush(pFpuCtx);
4613 RT_NOREF(pVCpu);
4614}
4615
4616
4617/**
4618 * Stores a result in a FPU register and updates the FSW and FTW.
4619 *
4620 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4621 * @param pFpuCtx The FPU context.
4622 * @param pResult The result to store.
4623 * @param iStReg Which FPU register to store it in.
4624 */
4625static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4626{
4627 Assert(iStReg < 8);
4628 uint16_t fNewFsw = pFpuCtx->FSW;
4629 uint16_t const iReg = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK;
4630 fNewFsw &= ~X86_FSW_C_MASK;
4631 fNewFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
4632 if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4633 Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4634 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4635 pFpuCtx->FSW = fNewFsw;
4636 pFpuCtx->FTW |= RT_BIT(iReg);
4637 pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
4638 RT_NOREF(pVCpu);
4639}
4640
4641
4642/**
4643 * Only updates the FPU status word (FSW) with the result of the current
4644 * instruction.
4645 *
4646 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4647 * @param pFpuCtx The FPU context.
4648 * @param u16FSW The FSW output of the current instruction.
4649 */
4650static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT
4651{
4652 uint16_t fNewFsw = pFpuCtx->FSW;
4653 fNewFsw &= ~X86_FSW_C_MASK;
4654 fNewFsw |= u16FSW & ~X86_FSW_TOP_MASK;
4655 if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4656 Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4657 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4658 pFpuCtx->FSW = fNewFsw;
4659 RT_NOREF(pVCpu);
4660}
4661
4662
4663/**
4664 * Pops one item off the FPU stack if no pending exception prevents it.
4665 *
4666 * @param pFpuCtx The FPU context.
4667 */
4668static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4669{
4670 /* Check pending exceptions. */
4671 uint16_t uFSW = pFpuCtx->FSW;
4672 if ( (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
4673 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
4674 return;
4675
4676 /* TOP--. */
4677 uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
4678 uFSW &= ~X86_FSW_TOP_MASK;
4679 uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
4680 pFpuCtx->FSW = uFSW;
4681
4682 /* Mark the previous ST0 as empty. */
4683 iOldTop >>= X86_FSW_TOP_SHIFT;
4684 pFpuCtx->FTW &= ~RT_BIT(iOldTop);
4685
4686 /* Rotate the registers. */
4687 iemFpuRotateStackPop(pFpuCtx);
4688}
4689
4690
4691/**
4692 * Pushes a FPU result onto the FPU stack if no pending exception prevents it.
4693 *
4694 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4695 * @param pResult The FPU operation result to push.
4696 */
4697void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT
4698{
4699 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4700 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4701 iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4702}
4703
4704
4705/**
4706 * Pushes a FPU result onto the FPU stack if no pending exception prevents it,
4707 * and sets FPUDP and FPUDS.
4708 *
4709 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4710 * @param pResult The FPU operation result to push.
4711 * @param iEffSeg The effective segment register.
4712 * @param GCPtrEff The effective address relative to @a iEffSeg.
4713 */
4714void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4715{
4716 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4717 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4718 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4719 iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4720}
4721
4722
4723/**
4724 * Replace ST0 with the first value and push the second onto the FPU stack,
4725 * unless a pending exception prevents it.
4726 *
4727 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4728 * @param pResult The FPU operation result to store and push.
4729 */
4730void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT
4731{
4732 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4733 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4734
4735 /* Update FSW and bail if there are pending exceptions afterwards. */
4736 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4737 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
4738 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
4739 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
4740 {
4741 if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4742 Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n",
4743 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4744 pFpuCtx->FSW = fFsw;
4745 return;
4746 }
4747
4748 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4749 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4750 {
4751 /* All is fine, push the actual value. */
4752 pFpuCtx->FTW |= RT_BIT(iNewTop);
4753 pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
4754 pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
4755 }
4756 else if (pFpuCtx->FCW & X86_FCW_IM)
4757 {
4758 /* Masked stack overflow, push QNaN. */
4759 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
4760 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
4761 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4762 }
4763 else
4764 {
4765 /* Raise stack overflow, don't push anything. */
4766 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
4767 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
4768 Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4769 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4770 return;
4771 }
4772
4773 fFsw &= ~X86_FSW_TOP_MASK;
4774 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
4775 pFpuCtx->FSW = fFsw;
4776
4777 iemFpuRotateStackPush(pFpuCtx);
4778}
4779
4780
4781/**
4782 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4783 * FOP.
4784 *
4785 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4786 * @param pResult The result to store.
4787 * @param iStReg Which FPU register to store it in.
4788 */
4789void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4790{
4791 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4792 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4793 iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4794}
4795
4796
4797/**
4798 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4799 * FOP, and then pops the stack.
4800 *
4801 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4802 * @param pResult The result to store.
4803 * @param iStReg Which FPU register to store it in.
4804 */
4805void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4806{
4807 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4808 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4809 iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4810 iemFpuMaybePopOne(pFpuCtx);
4811}
4812
4813
4814/**
4815 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4816 * FPUDP, and FPUDS.
4817 *
4818 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4819 * @param pResult The result to store.
4820 * @param iStReg Which FPU register to store it in.
4821 * @param iEffSeg The effective memory operand selector register.
4822 * @param GCPtrEff The effective memory operand offset.
4823 */
4824void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
4825 uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4826{
4827 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4828 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4829 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4830 iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4831}
4832
4833
4834/**
4835 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4836 * FPUDP, and FPUDS, and then pops the stack.
4837 *
4838 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4839 * @param pResult The result to store.
4840 * @param iStReg Which FPU register to store it in.
4841 * @param iEffSeg The effective memory operand selector register.
4842 * @param GCPtrEff The effective memory operand offset.
4843 */
4844void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
4845 uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4846{
4847 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4848 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4849 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4850 iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4851 iemFpuMaybePopOne(pFpuCtx);
4852}
4853
4854
4855/**
4856 * Updates the FOP, FPUIP, and FPUCS. For FNOP.
4857 *
4858 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4859 */
4860void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT
4861{
4862 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4863 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4864}
4865
4866
4867/**
4868 * Updates the FSW, FOP, FPUIP, and FPUCS.
4869 *
4870 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4871 * @param u16FSW The FSW from the current instruction.
4872 */
4873void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4874{
4875 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4876 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4877 iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4878}
4879
4880
4881/**
4882 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
4883 *
4884 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4885 * @param u16FSW The FSW from the current instruction.
4886 */
4887void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4888{
4889 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4890 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4891 iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4892 iemFpuMaybePopOne(pFpuCtx);
4893}
4894
4895
4896/**
4897 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
4898 *
4899 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4900 * @param u16FSW The FSW from the current instruction.
4901 * @param iEffSeg The effective memory operand selector register.
4902 * @param GCPtrEff The effective memory operand offset.
4903 */
4904void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4905{
4906 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4907 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4908 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4909 iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4910}
4911
4912
4913/**
4914 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
4915 *
4916 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4917 * @param u16FSW The FSW from the current instruction.
4918 */
4919void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4920{
4921 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4922 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4923 iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4924 iemFpuMaybePopOne(pFpuCtx);
4925 iemFpuMaybePopOne(pFpuCtx);
4926}
4927
4928
4929/**
4930 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
4931 *
4932 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4933 * @param u16FSW The FSW from the current instruction.
4934 * @param iEffSeg The effective memory operand selector register.
4935 * @param GCPtrEff The effective memory operand offset.
4936 */
4937void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4938{
4939 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4940 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4941 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4942 iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4943 iemFpuMaybePopOne(pFpuCtx);
4944}
4945
4946
4947/**
4948 * Worker routine for raising an FPU stack underflow exception.
4949 *
4950 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4951 * @param pFpuCtx The FPU context.
4952 * @param iStReg The stack register being accessed.
4953 */
4954static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg)
4955{
4956 Assert(iStReg < 8 || iStReg == UINT8_MAX);
4957 if (pFpuCtx->FCW & X86_FCW_IM)
4958 {
4959 /* Masked underflow. */
4960 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4961 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
4962 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
4963 if (iStReg != UINT8_MAX)
4964 {
4965 pFpuCtx->FTW |= RT_BIT(iReg);
4966 iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
4967 }
4968 }
4969 else
4970 {
4971 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4972 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
4973 Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n",
4974 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4975 }
4976 RT_NOREF(pVCpu);
4977}
4978
4979
4980/**
4981 * Raises a FPU stack underflow exception.
4982 *
4983 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4984 * @param iStReg The destination register that should be loaded
4985 * with QNaN if \#IS is not masked. Specify
4986 * UINT8_MAX if none (like for fcom).
4987 */
4988void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
4989{
4990 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4991 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4992 iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
4993}
4994
4995
4996void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4997{
4998 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4999 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5000 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5001 iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5002}
5003
5004
5005void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
5006{
5007 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5008 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5009 iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5010 iemFpuMaybePopOne(pFpuCtx);
5011}
5012
5013
5014void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5015{
5016 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5017 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5018 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5019 iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5020 iemFpuMaybePopOne(pFpuCtx);
5021}
5022
5023
5024void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT
5025{
5026 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5027 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5028 iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX);
5029 iemFpuMaybePopOne(pFpuCtx);
5030 iemFpuMaybePopOne(pFpuCtx);
5031}
5032
5033
5034void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5035{
5036 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5037 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5038
5039 if (pFpuCtx->FCW & X86_FCW_IM)
5040 {
5041 /* Masked overflow - Push QNaN. */
5042 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5043 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
5044 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
5045 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
5046 pFpuCtx->FTW |= RT_BIT(iNewTop);
5047 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5048 iemFpuRotateStackPush(pFpuCtx);
5049 }
5050 else
5051 {
5052 /* Exception pending - don't change TOP or the register stack. */
5053 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5054 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
5055 Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n",
5056 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5057 }
5058}
5059
5060
5061void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT
5062{
5063 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5064 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5065
5066 if (pFpuCtx->FCW & X86_FCW_IM)
5067 {
5068 /* Masked overflow - Push QNaN. */
5069 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5070 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
5071 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
5072 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
5073 pFpuCtx->FTW |= RT_BIT(iNewTop);
5074 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
5075 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5076 iemFpuRotateStackPush(pFpuCtx);
5077 }
5078 else
5079 {
5080 /* Exception pending - don't change TOP or the register stack. */
5081 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5082 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
5083 Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n",
5084 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5085 }
5086}
5087
5088
5089/**
5090 * Worker routine for raising an FPU stack overflow exception on a push.
5091 *
5092 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5093 * @param pFpuCtx The FPU context.
5094 */
5095static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
5096{
5097 if (pFpuCtx->FCW & X86_FCW_IM)
5098 {
5099 /* Masked overflow. */
5100 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5101 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
5102 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
5103 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
5104 pFpuCtx->FTW |= RT_BIT(iNewTop);
5105 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5106 iemFpuRotateStackPush(pFpuCtx);
5107 }
5108 else
5109 {
5110 /* Exception pending - don't change TOP or the register stack. */
5111 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5112 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
5113 Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n",
5114 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5115 }
5116 RT_NOREF(pVCpu);
5117}
5118
5119
5120/**
5121 * Raises a FPU stack overflow exception on a push.
5122 *
5123 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5124 */
5125void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5126{
5127 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5128 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5129 iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5130}
5131
5132
5133/**
5134 * Raises a FPU stack overflow exception on a push with a memory operand.
5135 *
5136 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5137 * @param iEffSeg The effective memory operand selector register.
5138 * @param GCPtrEff The effective memory operand offset.
5139 */
5140void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5141{
5142 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5143 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5144 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5145 iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5146}
5147
5148/** @} */
5149
5150
5151/** @name SSE+AVX SIMD access and helpers.
5152 *
5153 * @{
5154 */
5155/**
5156 * Stores a result in a SIMD XMM register, updates the MXCSR.
5157 *
5158 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5159 * @param pResult The result to store.
5160 * @param iXmmReg Which SIMD XMM register to store the result in.
5161 */
5162void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT
5163{
5164 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5165 pFpuCtx->MXCSR |= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS;
5166
5167 /* The result is only updated if there is no unmasked exception pending. */
5168 if (( ~((pFpuCtx->MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
5169 & (pFpuCtx->MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0)
5170 pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult;
5171}
5172
5173
5174/**
5175 * Updates the MXCSR.
5176 *
5177 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5178 * @param fMxcsr The new MXCSR value.
5179 */
5180void iemSseUpdateMxcsr(PVMCPUCC pVCpu, uint32_t fMxcsr) RT_NOEXCEPT
5181{
5182 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5183 pFpuCtx->MXCSR |= fMxcsr & X86_MXCSR_XCPT_FLAGS;
5184}
5185/** @} */
5186
5187
5188/** @name Memory access.
5189 *
5190 * @{
5191 */
5192
5193
5194/**
5195 * Updates the IEMCPU::cbWritten counter if applicable.
5196 *
5197 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5198 * @param fAccess The access being accounted for.
5199 * @param cbMem The access size.
5200 */
5201DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
5202{
5203 if ( (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE)
5204 || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA | IEM_ACCESS_TYPE_WRITE) )
5205 pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
5206}
5207
5208
5209/**
5210 * Applies the segment limit, base and attributes.
5211 *
5212 * This may raise a \#GP or \#SS.
5213 *
5214 * @returns VBox strict status code.
5215 *
5216 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5217 * @param fAccess The kind of access which is being performed.
5218 * @param iSegReg The index of the segment register to apply.
5219 * This is UINT8_MAX if none (for IDT, GDT, LDT,
5220 * TSS, ++).
5221 * @param cbMem The access size.
5222 * @param pGCPtrMem Pointer to the guest memory address to apply
5223 * segmentation to. Input and output parameter.
5224 */
5225VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT
5226{
5227 if (iSegReg == UINT8_MAX)
5228 return VINF_SUCCESS;
5229
5230 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
5231 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
5232 switch (pVCpu->iem.s.enmCpuMode)
5233 {
5234 case IEMMODE_16BIT:
5235 case IEMMODE_32BIT:
5236 {
5237 RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
5238 RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
5239
5240 if ( pSel->Attr.n.u1Present
5241 && !pSel->Attr.n.u1Unusable)
5242 {
5243 Assert(pSel->Attr.n.u1DescType);
5244 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5245 {
5246 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5247 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
5248 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5249
5250 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5251 {
5252 /** @todo CPL check. */
5253 }
5254
5255 /*
5256 * There are two kinds of data selectors, normal and expand down.
5257 */
5258 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5259 {
5260 if ( GCPtrFirst32 > pSel->u32Limit
5261 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5262 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5263 }
5264 else
5265 {
5266 /*
5267 * The upper boundary is defined by the B bit, not the G bit!
5268 */
5269 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5270 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5271 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5272 }
5273 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5274 }
5275 else
5276 {
5277 /*
5278 * Code selector and usually be used to read thru, writing is
5279 * only permitted in real and V8086 mode.
5280 */
5281 if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5282 || ( (fAccess & IEM_ACCESS_TYPE_READ)
5283 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
5284 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
5285 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5286
5287 if ( GCPtrFirst32 > pSel->u32Limit
5288 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5289 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5290
5291 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5292 {
5293 /** @todo CPL check. */
5294 }
5295
5296 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5297 }
5298 }
5299 else
5300 return iemRaiseGeneralProtectionFault0(pVCpu);
5301 return VINF_SUCCESS;
5302 }
5303
5304 case IEMMODE_64BIT:
5305 {
5306 RTGCPTR GCPtrMem = *pGCPtrMem;
5307 if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS)
5308 *pGCPtrMem = GCPtrMem + pSel->u64Base;
5309
5310 Assert(cbMem >= 1);
5311 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
5312 return VINF_SUCCESS;
5313 /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
5314 * 4.12.2 "Data Limit Checks in 64-bit Mode". */
5315 return iemRaiseGeneralProtectionFault0(pVCpu);
5316 }
5317
5318 default:
5319 AssertFailedReturn(VERR_IEM_IPE_7);
5320 }
5321}
5322
5323
5324/**
5325 * Translates a virtual address to a physical physical address and checks if we
5326 * can access the page as specified.
5327 *
5328 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5329 * @param GCPtrMem The virtual address.
5330 * @param fAccess The intended access.
5331 * @param pGCPhysMem Where to return the physical address.
5332 */
5333VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT
5334{
5335 /** @todo Need a different PGM interface here. We're currently using
5336 * generic / REM interfaces. this won't cut it for R0. */
5337 /** @todo If/when PGM handles paged real-mode, we can remove the hack in
5338 * iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
5339 * here. */
5340 PGMPTWALK Walk;
5341 int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5342 if (RT_FAILURE(rc))
5343 {
5344 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
5345 /** @todo Check unassigned memory in unpaged mode. */
5346 /** @todo Reserved bits in page tables. Requires new PGM interface. */
5347#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5348 if (Walk.fFailed & PGM_WALKFAIL_EPT)
5349 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
5350#endif
5351 *pGCPhysMem = NIL_RTGCPHYS;
5352 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
5353 }
5354
5355 /* If the page is writable and does not have the no-exec bit set, all
5356 access is allowed. Otherwise we'll have to check more carefully... */
5357 if ((Walk.fEffective & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US))
5358 {
5359 /* Write to read only memory? */
5360 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5361 && !(Walk.fEffective & X86_PTE_RW)
5362 && ( ( pVCpu->iem.s.uCpl == 3
5363 && !(fAccess & IEM_ACCESS_WHAT_SYS))
5364 || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
5365 {
5366 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
5367 *pGCPhysMem = NIL_RTGCPHYS;
5368#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5369 if (Walk.fFailed & PGM_WALKFAIL_EPT)
5370 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5371#endif
5372 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
5373 }
5374
5375 /* Kernel memory accessed by userland? */
5376 if ( !(Walk.fEffective & X86_PTE_US)
5377 && pVCpu->iem.s.uCpl == 3
5378 && !(fAccess & IEM_ACCESS_WHAT_SYS))
5379 {
5380 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
5381 *pGCPhysMem = NIL_RTGCPHYS;
5382#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5383 if (Walk.fFailed & PGM_WALKFAIL_EPT)
5384 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5385#endif
5386 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
5387 }
5388
5389 /* Executing non-executable memory? */
5390 if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
5391 && (Walk.fEffective & X86_PTE_PAE_NX)
5392 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
5393 {
5394 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
5395 *pGCPhysMem = NIL_RTGCPHYS;
5396#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5397 if (Walk.fFailed & PGM_WALKFAIL_EPT)
5398 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5399#endif
5400 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE),
5401 VERR_ACCESS_DENIED);
5402 }
5403 }
5404
5405 /*
5406 * Set the dirty / access flags.
5407 * ASSUMES this is set when the address is translated rather than on committ...
5408 */
5409 /** @todo testcase: check when A and D bits are actually set by the CPU. */
5410 uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
5411 if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
5412 {
5413 int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
5414 AssertRC(rc2);
5415 /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
5416 Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
5417 }
5418
5419 RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
5420 *pGCPhysMem = GCPhys;
5421 return VINF_SUCCESS;
5422}
5423
5424
5425/**
5426 * Looks up a memory mapping entry.
5427 *
5428 * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
5429 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5430 * @param pvMem The memory address.
5431 * @param fAccess The access to.
5432 */
5433DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
5434{
5435 Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5436 fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;
5437 if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
5438 && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
5439 return 0;
5440 if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
5441 && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
5442 return 1;
5443 if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
5444 && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
5445 return 2;
5446 return VERR_NOT_FOUND;
5447}
5448
5449
5450/**
5451 * Finds a free memmap entry when using iNextMapping doesn't work.
5452 *
5453 * @returns Memory mapping index, 1024 on failure.
5454 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5455 */
5456static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
5457{
5458 /*
5459 * The easy case.
5460 */
5461 if (pVCpu->iem.s.cActiveMappings == 0)
5462 {
5463 pVCpu->iem.s.iNextMapping = 1;
5464 return 0;
5465 }
5466
5467 /* There should be enough mappings for all instructions. */
5468 AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
5469
5470 for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
5471 if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
5472 return i;
5473
5474 AssertFailedReturn(1024);
5475}
5476
5477
5478/**
5479 * Commits a bounce buffer that needs writing back and unmaps it.
5480 *
5481 * @returns Strict VBox status code.
5482 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5483 * @param iMemMap The index of the buffer to commit.
5484 * @param fPostponeFail Whether we can postpone writer failures to ring-3.
5485 * Always false in ring-3, obviously.
5486 */
5487static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
5488{
5489 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
5490 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
5491#ifdef IN_RING3
5492 Assert(!fPostponeFail);
5493 RT_NOREF_PV(fPostponeFail);
5494#endif
5495
5496 /*
5497 * Do the writing.
5498 */
5499 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5500 if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
5501 {
5502 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
5503 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5504 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5505 if (!pVCpu->iem.s.fBypassHandlers)
5506 {
5507 /*
5508 * Carefully and efficiently dealing with access handler return
5509 * codes make this a little bloated.
5510 */
5511 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
5512 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5513 pbBuf,
5514 cbFirst,
5515 PGMACCESSORIGIN_IEM);
5516 if (rcStrict == VINF_SUCCESS)
5517 {
5518 if (cbSecond)
5519 {
5520 rcStrict = PGMPhysWrite(pVM,
5521 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5522 pbBuf + cbFirst,
5523 cbSecond,
5524 PGMACCESSORIGIN_IEM);
5525 if (rcStrict == VINF_SUCCESS)
5526 { /* nothing */ }
5527 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5528 {
5529 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
5530 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5531 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5532 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5533 }
5534#ifndef IN_RING3
5535 else if (fPostponeFail)
5536 {
5537 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5538 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5539 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5540 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5541 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5542 return iemSetPassUpStatus(pVCpu, rcStrict);
5543 }
5544#endif
5545 else
5546 {
5547 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5548 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5549 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5550 return rcStrict;
5551 }
5552 }
5553 }
5554 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5555 {
5556 if (!cbSecond)
5557 {
5558 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
5559 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5560 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5561 }
5562 else
5563 {
5564 VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
5565 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5566 pbBuf + cbFirst,
5567 cbSecond,
5568 PGMACCESSORIGIN_IEM);
5569 if (rcStrict2 == VINF_SUCCESS)
5570 {
5571 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
5572 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5573 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5574 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5575 }
5576 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5577 {
5578 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
5579 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5580 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5581 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5582 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5583 }
5584#ifndef IN_RING3
5585 else if (fPostponeFail)
5586 {
5587 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5588 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5589 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5590 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5591 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5592 return iemSetPassUpStatus(pVCpu, rcStrict);
5593 }
5594#endif
5595 else
5596 {
5597 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5598 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5599 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5600 return rcStrict2;
5601 }
5602 }
5603 }
5604#ifndef IN_RING3
5605 else if (fPostponeFail)
5606 {
5607 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5608 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5609 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5610 if (!cbSecond)
5611 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;
5612 else
5613 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;
5614 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5615 return iemSetPassUpStatus(pVCpu, rcStrict);
5616 }
5617#endif
5618 else
5619 {
5620 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5621 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5622 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5623 return rcStrict;
5624 }
5625 }
5626 else
5627 {
5628 /*
5629 * No access handlers, much simpler.
5630 */
5631 int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
5632 if (RT_SUCCESS(rc))
5633 {
5634 if (cbSecond)
5635 {
5636 rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
5637 if (RT_SUCCESS(rc))
5638 { /* likely */ }
5639 else
5640 {
5641 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5642 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5643 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
5644 return rc;
5645 }
5646 }
5647 }
5648 else
5649 {
5650 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5651 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
5652 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5653 return rc;
5654 }
5655 }
5656 }
5657
5658#if defined(IEM_LOG_MEMORY_WRITES)
5659 Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5660 RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
5661 if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
5662 Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5663 RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
5664 &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
5665
5666 size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5667 g_cbIemWrote = cbWrote;
5668 memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
5669#endif
5670
5671 /*
5672 * Free the mapping entry.
5673 */
5674 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
5675 Assert(pVCpu->iem.s.cActiveMappings != 0);
5676 pVCpu->iem.s.cActiveMappings--;
5677 return VINF_SUCCESS;
5678}
5679
5680
5681/**
5682 * iemMemMap worker that deals with a request crossing pages.
5683 */
5684static VBOXSTRICTRC
5685iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
5686{
5687 /*
5688 * Do the address translations.
5689 */
5690 RTGCPHYS GCPhysFirst;
5691 VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
5692 if (rcStrict != VINF_SUCCESS)
5693 return rcStrict;
5694
5695 RTGCPHYS GCPhysSecond;
5696 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
5697 fAccess, &GCPhysSecond);
5698 if (rcStrict != VINF_SUCCESS)
5699 return rcStrict;
5700 GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
5701
5702 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5703
5704 /*
5705 * Read in the current memory content if it's a read, execute or partial
5706 * write access.
5707 */
5708 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5709 uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (GCPhysFirst & GUEST_PAGE_OFFSET_MASK);
5710 uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
5711
5712 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
5713 {
5714 if (!pVCpu->iem.s.fBypassHandlers)
5715 {
5716 /*
5717 * Must carefully deal with access handler status codes here,
5718 * makes the code a bit bloated.
5719 */
5720 rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
5721 if (rcStrict == VINF_SUCCESS)
5722 {
5723 rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5724 if (rcStrict == VINF_SUCCESS)
5725 { /*likely */ }
5726 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5727 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5728 else
5729 {
5730 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
5731 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5732 return rcStrict;
5733 }
5734 }
5735 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5736 {
5737 VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5738 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5739 {
5740 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5741 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5742 }
5743 else
5744 {
5745 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
5746 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
5747 return rcStrict2;
5748 }
5749 }
5750 else
5751 {
5752 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5753 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5754 return rcStrict;
5755 }
5756 }
5757 else
5758 {
5759 /*
5760 * No informational status codes here, much more straight forward.
5761 */
5762 int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
5763 if (RT_SUCCESS(rc))
5764 {
5765 Assert(rc == VINF_SUCCESS);
5766 rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
5767 if (RT_SUCCESS(rc))
5768 Assert(rc == VINF_SUCCESS);
5769 else
5770 {
5771 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
5772 return rc;
5773 }
5774 }
5775 else
5776 {
5777 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
5778 return rc;
5779 }
5780 }
5781 }
5782#ifdef VBOX_STRICT
5783 else
5784 memset(pbBuf, 0xcc, cbMem);
5785 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5786 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5787#endif
5788 AssertCompileMemberAlignment(VMCPU, iem.s.aBounceBuffers, 64);
5789
5790 /*
5791 * Commit the bounce buffer entry.
5792 */
5793 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5794 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
5795 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
5796 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
5797 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
5798 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5799 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
5800 pVCpu->iem.s.iNextMapping = iMemMap + 1;
5801 pVCpu->iem.s.cActiveMappings++;
5802
5803 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5804 *ppvMem = pbBuf;
5805 return VINF_SUCCESS;
5806}
5807
5808
5809/**
5810 * iemMemMap woker that deals with iemMemPageMap failures.
5811 */
5812static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
5813 RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
5814{
5815 /*
5816 * Filter out conditions we can handle and the ones which shouldn't happen.
5817 */
5818 if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
5819 && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
5820 && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
5821 {
5822 AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
5823 return rcMap;
5824 }
5825 pVCpu->iem.s.cPotentialExits++;
5826
5827 /*
5828 * Read in the current memory content if it's a read, execute or partial
5829 * write access.
5830 */
5831 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5832 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
5833 {
5834 if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
5835 memset(pbBuf, 0xff, cbMem);
5836 else
5837 {
5838 int rc;
5839 if (!pVCpu->iem.s.fBypassHandlers)
5840 {
5841 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
5842 if (rcStrict == VINF_SUCCESS)
5843 { /* nothing */ }
5844 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5845 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5846 else
5847 {
5848 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5849 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5850 return rcStrict;
5851 }
5852 }
5853 else
5854 {
5855 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
5856 if (RT_SUCCESS(rc))
5857 { /* likely */ }
5858 else
5859 {
5860 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5861 GCPhysFirst, rc));
5862 return rc;
5863 }
5864 }
5865 }
5866 }
5867#ifdef VBOX_STRICT
5868 else
5869 memset(pbBuf, 0xcc, cbMem);
5870#endif
5871#ifdef VBOX_STRICT
5872 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5873 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5874#endif
5875
5876 /*
5877 * Commit the bounce buffer entry.
5878 */
5879 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5880 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
5881 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
5882 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
5883 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
5884 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5885 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
5886 pVCpu->iem.s.iNextMapping = iMemMap + 1;
5887 pVCpu->iem.s.cActiveMappings++;
5888
5889 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5890 *ppvMem = pbBuf;
5891 return VINF_SUCCESS;
5892}
5893
5894
5895
5896/**
5897 * Maps the specified guest memory for the given kind of access.
5898 *
5899 * This may be using bounce buffering of the memory if it's crossing a page
5900 * boundary or if there is an access handler installed for any of it. Because
5901 * of lock prefix guarantees, we're in for some extra clutter when this
5902 * happens.
5903 *
5904 * This may raise a \#GP, \#SS, \#PF or \#AC.
5905 *
5906 * @returns VBox strict status code.
5907 *
5908 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5909 * @param ppvMem Where to return the pointer to the mapped memory.
5910 * @param cbMem The number of bytes to map. This is usually 1, 2, 4, 6,
5911 * 8, 12, 16, 32 or 512. When used by string operations
5912 * it can be up to a page.
5913 * @param iSegReg The index of the segment register to use for this
5914 * access. The base and limits are checked. Use UINT8_MAX
5915 * to indicate that no segmentation is required (for IDT,
5916 * GDT and LDT accesses).
5917 * @param GCPtrMem The address of the guest memory.
5918 * @param fAccess How the memory is being accessed. The
5919 * IEM_ACCESS_TYPE_XXX bit is used to figure out how to map
5920 * the memory, while the IEM_ACCESS_WHAT_XXX bit is used
5921 * when raising exceptions.
5922 * @param uAlignCtl Alignment control:
5923 * - Bits 15:0 is the alignment mask.
5924 * - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
5925 * IEM_MEMMAP_F_ALIGN_SSE, and
5926 * IEM_MEMMAP_F_ALIGN_GP_OR_AC.
5927 * Pass zero to skip alignment.
5928 */
5929VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
5930 uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT
5931{
5932 /*
5933 * Check the input and figure out which mapping entry to use.
5934 */
5935 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94); /* 512 is the max! */
5936 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
5937 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5938
5939 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
5940 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
5941 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
5942 {
5943 iMemMap = iemMemMapFindFree(pVCpu);
5944 AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
5945 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
5946 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
5947 pVCpu->iem.s.aMemMappings[2].fAccess),
5948 VERR_IEM_IPE_9);
5949 }
5950
5951 /*
5952 * Map the memory, checking that we can actually access it. If something
5953 * slightly complicated happens, fall back on bounce buffering.
5954 */
5955 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
5956 if (rcStrict == VINF_SUCCESS)
5957 { /* likely */ }
5958 else
5959 return rcStrict;
5960
5961 if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */
5962 { /* likely */ }
5963 else
5964 return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
5965
5966 /*
5967 * Alignment check.
5968 */
5969 if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
5970 { /* likelyish */ }
5971 else
5972 {
5973 /* Misaligned access. */
5974 if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
5975 {
5976 if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
5977 || ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
5978 && (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
5979 {
5980 AssertCompile(X86_CR0_AM == X86_EFL_AC);
5981
5982 if (iemMemAreAlignmentChecksEnabled(pVCpu))
5983 return iemRaiseAlignmentCheckException(pVCpu);
5984 }
5985 else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
5986 && iemMemAreAlignmentChecksEnabled(pVCpu)
5987/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
5988 * implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
5989 )
5990 return iemRaiseAlignmentCheckException(pVCpu);
5991 else
5992 return iemRaiseGeneralProtectionFault0(pVCpu);
5993 }
5994 }
5995
5996#ifdef IEM_WITH_DATA_TLB
5997 Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
5998
5999 /*
6000 * Get the TLB entry for this page.
6001 */
6002 uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6003 PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6004 if (pTlbe->uTag == uTag)
6005 {
6006# ifdef VBOX_WITH_STATISTICS
6007 pVCpu->iem.s.DataTlb.cTlbHits++;
6008# endif
6009 }
6010 else
6011 {
6012 pVCpu->iem.s.DataTlb.cTlbMisses++;
6013 PGMPTWALK Walk;
6014 int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6015 if (RT_FAILURE(rc))
6016 {
6017 Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6018# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6019 if (Walk.fFailed & PGM_WALKFAIL_EPT)
6020 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6021# endif
6022 iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6023 }
6024
6025 Assert(Walk.fSucceeded);
6026 pTlbe->uTag = uTag;
6027 pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A); /* skipping NX */
6028 pTlbe->GCPhys = Walk.GCPhys;
6029 pTlbe->pbMappingR3 = NULL;
6030 }
6031
6032 /*
6033 * Check TLB page table level access flags.
6034 */
6035 /* If the page is either supervisor only or non-writable, we need to do
6036 more careful access checks. */
6037 if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_WRITE))
6038 {
6039 /* Write to read only memory? */
6040 if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
6041 && (fAccess & IEM_ACCESS_TYPE_WRITE)
6042 && ( ( pVCpu->iem.s.uCpl == 3
6043 && !(fAccess & IEM_ACCESS_WHAT_SYS))
6044 || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
6045 {
6046 Log(("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6047# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6048 if (Walk.fFailed & PGM_WALKFAIL_EPT)
6049 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6050# endif
6051 iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6052 }
6053
6054 /* Kernel memory accessed by userland? */
6055 if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
6056 && pVCpu->iem.s.uCpl == 3
6057 && !(fAccess & IEM_ACCESS_WHAT_SYS))
6058 {
6059 Log(("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6060# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6061 if (Walk.fFailed & PGM_WALKFAIL_EPT)
6062 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6063# endif
6064 iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6065 }
6066 }
6067
6068 /*
6069 * Set the dirty / access flags.
6070 * ASSUMES this is set when the address is translated rather than on commit...
6071 */
6072 /** @todo testcase: check when A and D bits are actually set by the CPU. */
6073 uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) | IEMTLBE_F_PT_NO_ACCESSED;
6074 if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty)
6075 {
6076 uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
6077 int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6078 AssertRC(rc2);
6079 /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6080 Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6081 pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty;
6082 }
6083
6084 /*
6085 * Look up the physical page info if necessary.
6086 */
6087 uint8_t *pbMem = NULL;
6088 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6089# ifdef IN_RING3
6090 pbMem = pTlbe->pbMappingR3;
6091# else
6092 pbMem = NULL;
6093# endif
6094 else
6095 {
6096 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6097 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6098 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6099 AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6100 if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
6101 { /* likely */ }
6102 else
6103 IEMTlbInvalidateAllPhysicalSlow(pVCpu);
6104 pTlbe->pbMappingR3 = NULL;
6105 pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6106 | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED);
6107 int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6108 &pbMem, &pTlbe->fFlagsAndPhysRev);
6109 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6110# ifdef IN_RING3
6111 pTlbe->pbMappingR3 = pbMem;
6112# endif
6113 }
6114
6115 /*
6116 * Check the physical page level access and mapping.
6117 */
6118 if ( !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ))
6119 || !(pTlbe->fFlagsAndPhysRev & ( (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0)
6120 | (fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0))) )
6121 { /* probably likely */ }
6122 else
6123 return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem,
6124 pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6125 pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6126 : pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6127 : VERR_PGM_PHYS_TLB_CATCH_WRITE);
6128 Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6129
6130 if (pbMem)
6131 {
6132 Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6133 pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6134 fAccess |= IEM_ACCESS_NOT_LOCKED;
6135 }
6136 else
6137 {
6138 Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6139 RTGCPHYS const GCPhysFirst = pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6140 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6141 if (rcStrict != VINF_SUCCESS)
6142 return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6143 }
6144
6145 void * const pvMem = pbMem;
6146
6147 if (fAccess & IEM_ACCESS_TYPE_WRITE)
6148 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6149 if (fAccess & IEM_ACCESS_TYPE_READ)
6150 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6151
6152#else /* !IEM_WITH_DATA_TLB */
6153
6154 RTGCPHYS GCPhysFirst;
6155 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6156 if (rcStrict != VINF_SUCCESS)
6157 return rcStrict;
6158
6159 if (fAccess & IEM_ACCESS_TYPE_WRITE)
6160 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6161 if (fAccess & IEM_ACCESS_TYPE_READ)
6162 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6163
6164 void *pvMem;
6165 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6166 if (rcStrict != VINF_SUCCESS)
6167 return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6168
6169#endif /* !IEM_WITH_DATA_TLB */
6170
6171 /*
6172 * Fill in the mapping table entry.
6173 */
6174 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6175 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6176 pVCpu->iem.s.iNextMapping = iMemMap + 1;
6177 pVCpu->iem.s.cActiveMappings += 1;
6178
6179 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6180 *ppvMem = pvMem;
6181
6182 return VINF_SUCCESS;
6183}
6184
6185
6186/**
6187 * Commits the guest memory if bounce buffered and unmaps it.
6188 *
6189 * @returns Strict VBox status code.
6190 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6191 * @param pvMem The mapping.
6192 * @param fAccess The kind of access.
6193 */
6194VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6195{
6196 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6197 AssertReturn(iMemMap >= 0, iMemMap);
6198
6199 /* If it's bounce buffered, we may need to write back the buffer. */
6200 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6201 {
6202 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6203 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
6204 }
6205 /* Otherwise unlock it. */
6206 else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6207 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6208
6209 /* Free the entry. */
6210 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6211 Assert(pVCpu->iem.s.cActiveMappings != 0);
6212 pVCpu->iem.s.cActiveMappings--;
6213 return VINF_SUCCESS;
6214}
6215
6216#ifdef IEM_WITH_SETJMP
6217
6218/**
6219 * Maps the specified guest memory for the given kind of access, longjmp on
6220 * error.
6221 *
6222 * This may be using bounce buffering of the memory if it's crossing a page
6223 * boundary or if there is an access handler installed for any of it. Because
6224 * of lock prefix guarantees, we're in for some extra clutter when this
6225 * happens.
6226 *
6227 * This may raise a \#GP, \#SS, \#PF or \#AC.
6228 *
6229 * @returns Pointer to the mapped memory.
6230 *
6231 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6232 * @param cbMem The number of bytes to map. This is usually 1,
6233 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
6234 * string operations it can be up to a page.
6235 * @param iSegReg The index of the segment register to use for
6236 * this access. The base and limits are checked.
6237 * Use UINT8_MAX to indicate that no segmentation
6238 * is required (for IDT, GDT and LDT accesses).
6239 * @param GCPtrMem The address of the guest memory.
6240 * @param fAccess How the memory is being accessed. The
6241 * IEM_ACCESS_TYPE_XXX bit is used to figure out
6242 * how to map the memory, while the
6243 * IEM_ACCESS_WHAT_XXX bit is used when raising
6244 * exceptions.
6245 * @param uAlignCtl Alignment control:
6246 * - Bits 15:0 is the alignment mask.
6247 * - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
6248 * IEM_MEMMAP_F_ALIGN_SSE, and
6249 * IEM_MEMMAP_F_ALIGN_GP_OR_AC.
6250 * Pass zero to skip alignment.
6251 */
6252void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess,
6253 uint32_t uAlignCtl) RT_NOEXCEPT
6254{
6255 /*
6256 * Check the input, check segment access and adjust address
6257 * with segment base.
6258 */
6259 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
6260 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
6261 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
6262
6263 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6264 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
6265 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6266
6267 /*
6268 * Alignment check.
6269 */
6270 if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
6271 { /* likelyish */ }
6272 else
6273 {
6274 /* Misaligned access. */
6275 if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6276 {
6277 if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
6278 || ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
6279 && (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
6280 {
6281 AssertCompile(X86_CR0_AM == X86_EFL_AC);
6282
6283 if (iemMemAreAlignmentChecksEnabled(pVCpu))
6284 iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6285 }
6286 else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
6287 && iemMemAreAlignmentChecksEnabled(pVCpu)
6288/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
6289 * implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
6290 )
6291 iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6292 else
6293 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
6294 }
6295 }
6296
6297 /*
6298 * Figure out which mapping entry to use.
6299 */
6300 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
6301 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6302 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6303 {
6304 iMemMap = iemMemMapFindFree(pVCpu);
6305 AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
6306 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
6307 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
6308 pVCpu->iem.s.aMemMappings[2].fAccess),
6309 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
6310 }
6311
6312 /*
6313 * Crossing a page boundary?
6314 */
6315 if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
6316 { /* No (likely). */ }
6317 else
6318 {
6319 void *pvMem;
6320 rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
6321 if (rcStrict == VINF_SUCCESS)
6322 return pvMem;
6323 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6324 }
6325
6326#ifdef IEM_WITH_DATA_TLB
6327 Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
6328
6329 /*
6330 * Get the TLB entry for this page.
6331 */
6332 uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6333 PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6334 if (pTlbe->uTag == uTag)
6335 STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6336 else
6337 {
6338 pVCpu->iem.s.DataTlb.cTlbMisses++;
6339 PGMPTWALK Walk;
6340 int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6341 if (RT_FAILURE(rc))
6342 {
6343 Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6344# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6345 if (Walk.fFailed & PGM_WALKFAIL_EPT)
6346 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6347# endif
6348 iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6349 }
6350
6351 Assert(Walk.fSucceeded);
6352 pTlbe->uTag = uTag;
6353 pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A); /* skipping NX */
6354 pTlbe->GCPhys = Walk.GCPhys;
6355 pTlbe->pbMappingR3 = NULL;
6356 }
6357
6358 /*
6359 * Check the flags and physical revision.
6360 */
6361 /** @todo make the caller pass these in with fAccess. */
6362 uint64_t const fNoUser = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && pVCpu->iem.s.uCpl == 3
6363 ? IEMTLBE_F_PT_NO_USER : 0;
6364 uint64_t const fNoWriteNoDirty = fAccess & IEM_ACCESS_TYPE_WRITE
6365 ? IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PT_NO_DIRTY
6366 | ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)
6367 || (pVCpu->iem.s.uCpl == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6368 ? IEMTLBE_F_PT_NO_WRITE : 0)
6369 : 0;
6370 uint64_t const fNoRead = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0;
6371 uint8_t *pbMem = NULL;
6372 if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PT_NO_ACCESSED | fNoRead | fNoWriteNoDirty | fNoUser))
6373 == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6374# ifdef IN_RING3
6375 pbMem = pTlbe->pbMappingR3;
6376# else
6377 pbMem = NULL;
6378# endif
6379 else
6380 {
6381 /*
6382 * Okay, something isn't quite right or needs refreshing.
6383 */
6384 /* Write to read only memory? */
6385 if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
6386 {
6387 Log(("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6388# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6389 if (Walk.fFailed & PGM_WALKFAIL_EPT)
6390 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6391# endif
6392 iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6393 }
6394
6395 /* Kernel memory accessed by userland? */
6396 if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER)
6397 {
6398 Log(("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6399# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6400 if (Walk.fFailed & PGM_WALKFAIL_EPT)
6401 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6402# endif
6403 iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6404 }
6405
6406 /* Set the dirty / access flags.
6407 ASSUMES this is set when the address is translated rather than on commit... */
6408 /** @todo testcase: check when A and D bits are actually set by the CPU. */
6409 if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED))
6410 {
6411 uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
6412 int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6413 AssertRC(rc2);
6414 /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6415 Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6416 pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED);
6417 }
6418
6419 /*
6420 * Check if the physical page info needs updating.
6421 */
6422 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6423# ifdef IN_RING3
6424 pbMem = pTlbe->pbMappingR3;
6425# else
6426 pbMem = NULL;
6427# endif
6428 else
6429 {
6430 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6431 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6432 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6433 AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6434 pTlbe->pbMappingR3 = NULL;
6435 pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6436 | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED);
6437 int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6438 &pbMem, &pTlbe->fFlagsAndPhysRev);
6439 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6440# ifdef IN_RING3
6441 pTlbe->pbMappingR3 = pbMem;
6442# endif
6443 }
6444
6445 /*
6446 * Check the physical page level access and mapping.
6447 */
6448 if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty | fNoRead) & (IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ))))
6449 { /* probably likely */ }
6450 else
6451 {
6452 rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, cbMem,
6453 pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6454 pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6455 : pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6456 : VERR_PGM_PHYS_TLB_CATCH_WRITE);
6457 if (rcStrict == VINF_SUCCESS)
6458 return pbMem;
6459 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6460 }
6461 }
6462 Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6463
6464 if (pbMem)
6465 {
6466 Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6467 pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6468 fAccess |= IEM_ACCESS_NOT_LOCKED;
6469 }
6470 else
6471 {
6472 Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6473 RTGCPHYS const GCPhysFirst = pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6474 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6475 if (rcStrict == VINF_SUCCESS)
6476 return pbMem;
6477 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6478 }
6479
6480 void * const pvMem = pbMem;
6481
6482 if (fAccess & IEM_ACCESS_TYPE_WRITE)
6483 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6484 if (fAccess & IEM_ACCESS_TYPE_READ)
6485 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6486
6487#else /* !IEM_WITH_DATA_TLB */
6488
6489
6490 RTGCPHYS GCPhysFirst;
6491 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6492 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
6493 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6494
6495 if (fAccess & IEM_ACCESS_TYPE_WRITE)
6496 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6497 if (fAccess & IEM_ACCESS_TYPE_READ)
6498 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6499
6500 void *pvMem;
6501 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6502 if (rcStrict == VINF_SUCCESS)
6503 { /* likely */ }
6504 else
6505 {
6506 rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6507 if (rcStrict == VINF_SUCCESS)
6508 return pvMem;
6509 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6510 }
6511
6512#endif /* !IEM_WITH_DATA_TLB */
6513
6514 /*
6515 * Fill in the mapping table entry.
6516 */
6517 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6518 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6519 pVCpu->iem.s.iNextMapping = iMemMap + 1;
6520 pVCpu->iem.s.cActiveMappings++;
6521
6522 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6523 return pvMem;
6524}
6525
6526
6527/**
6528 * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
6529 *
6530 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6531 * @param pvMem The mapping.
6532 * @param fAccess The kind of access.
6533 */
6534void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6535{
6536 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6537 AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
6538
6539 /* If it's bounce buffered, we may need to write back the buffer. */
6540 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6541 {
6542 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6543 {
6544 VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
6545 if (rcStrict == VINF_SUCCESS)
6546 return;
6547 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6548 }
6549 }
6550 /* Otherwise unlock it. */
6551 else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6552 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6553
6554 /* Free the entry. */
6555 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6556 Assert(pVCpu->iem.s.cActiveMappings != 0);
6557 pVCpu->iem.s.cActiveMappings--;
6558}
6559
6560#endif /* IEM_WITH_SETJMP */
6561
6562#ifndef IN_RING3
6563/**
6564 * Commits the guest memory if bounce buffered and unmaps it, if any bounce
6565 * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
6566 *
6567 * Allows the instruction to be completed and retired, while the IEM user will
6568 * return to ring-3 immediately afterwards and do the postponed writes there.
6569 *
6570 * @returns VBox status code (no strict statuses). Caller must check
6571 * VMCPU_FF_IEM before repeating string instructions and similar stuff.
6572 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6573 * @param pvMem The mapping.
6574 * @param fAccess The kind of access.
6575 */
6576VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6577{
6578 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6579 AssertReturn(iMemMap >= 0, iMemMap);
6580
6581 /* If it's bounce buffered, we may need to write back the buffer. */
6582 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6583 {
6584 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6585 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);
6586 }
6587 /* Otherwise unlock it. */
6588 else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6589 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6590
6591 /* Free the entry. */
6592 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6593 Assert(pVCpu->iem.s.cActiveMappings != 0);
6594 pVCpu->iem.s.cActiveMappings--;
6595 return VINF_SUCCESS;
6596}
6597#endif
6598
6599
6600/**
6601 * Rollbacks mappings, releasing page locks and such.
6602 *
6603 * The caller shall only call this after checking cActiveMappings.
6604 *
6605 * @returns Strict VBox status code to pass up.
6606 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6607 */
6608void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT
6609{
6610 Assert(pVCpu->iem.s.cActiveMappings > 0);
6611
6612 uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
6613 while (iMemMap-- > 0)
6614 {
6615 uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
6616 if (fAccess != IEM_ACCESS_INVALID)
6617 {
6618 AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
6619 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6620 if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED | IEM_ACCESS_NOT_LOCKED)))
6621 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6622 AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
6623 ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
6624 iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
6625 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
6626 pVCpu->iem.s.cActiveMappings--;
6627 }
6628 }
6629}
6630
6631
6632/**
6633 * Fetches a data byte.
6634 *
6635 * @returns Strict VBox status code.
6636 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6637 * @param pu8Dst Where to return the byte.
6638 * @param iSegReg The index of the segment register to use for
6639 * this access. The base and limits are checked.
6640 * @param GCPtrMem The address of the guest memory.
6641 */
6642VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6643{
6644 /* The lazy approach for now... */
6645 uint8_t const *pu8Src;
6646 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6647 if (rc == VINF_SUCCESS)
6648 {
6649 *pu8Dst = *pu8Src;
6650 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6651 }
6652 return rc;
6653}
6654
6655
6656#ifdef IEM_WITH_SETJMP
6657/**
6658 * Fetches a data byte, longjmp on error.
6659 *
6660 * @returns The byte.
6661 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6662 * @param iSegReg The index of the segment register to use for
6663 * this access. The base and limits are checked.
6664 * @param GCPtrMem The address of the guest memory.
6665 */
6666uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6667{
6668 /* The lazy approach for now... */
6669 uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6670 uint8_t const bRet = *pu8Src;
6671 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6672 return bRet;
6673}
6674#endif /* IEM_WITH_SETJMP */
6675
6676
6677/**
6678 * Fetches a data word.
6679 *
6680 * @returns Strict VBox status code.
6681 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6682 * @param pu16Dst Where to return the word.
6683 * @param iSegReg The index of the segment register to use for
6684 * this access. The base and limits are checked.
6685 * @param GCPtrMem The address of the guest memory.
6686 */
6687VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6688{
6689 /* The lazy approach for now... */
6690 uint16_t const *pu16Src;
6691 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem,
6692 IEM_ACCESS_DATA_R, sizeof(*pu16Src) - 1);
6693 if (rc == VINF_SUCCESS)
6694 {
6695 *pu16Dst = *pu16Src;
6696 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6697 }
6698 return rc;
6699}
6700
6701
6702#ifdef IEM_WITH_SETJMP
6703/**
6704 * Fetches a data word, longjmp on error.
6705 *
6706 * @returns The word
6707 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6708 * @param iSegReg The index of the segment register to use for
6709 * this access. The base and limits are checked.
6710 * @param GCPtrMem The address of the guest memory.
6711 */
6712uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6713{
6714 /* The lazy approach for now... */
6715 uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6716 sizeof(*pu16Src) - 1);
6717 uint16_t const u16Ret = *pu16Src;
6718 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6719 return u16Ret;
6720}
6721#endif
6722
6723
6724/**
6725 * Fetches a data dword.
6726 *
6727 * @returns Strict VBox status code.
6728 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6729 * @param pu32Dst Where to return the dword.
6730 * @param iSegReg The index of the segment register to use for
6731 * this access. The base and limits are checked.
6732 * @param GCPtrMem The address of the guest memory.
6733 */
6734VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6735{
6736 /* The lazy approach for now... */
6737 uint32_t const *pu32Src;
6738 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem,
6739 IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6740 if (rc == VINF_SUCCESS)
6741 {
6742 *pu32Dst = *pu32Src;
6743 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6744 }
6745 return rc;
6746}
6747
6748
6749/**
6750 * Fetches a data dword and zero extends it to a qword.
6751 *
6752 * @returns Strict VBox status code.
6753 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6754 * @param pu64Dst Where to return the qword.
6755 * @param iSegReg The index of the segment register to use for
6756 * this access. The base and limits are checked.
6757 * @param GCPtrMem The address of the guest memory.
6758 */
6759VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6760{
6761 /* The lazy approach for now... */
6762 uint32_t const *pu32Src;
6763 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem,
6764 IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6765 if (rc == VINF_SUCCESS)
6766 {
6767 *pu64Dst = *pu32Src;
6768 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6769 }
6770 return rc;
6771}
6772
6773
6774#ifdef IEM_WITH_SETJMP
6775
6776/**
6777 * Fetches a data dword, longjmp on error, fallback/safe version.
6778 *
6779 * @returns The dword
6780 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6781 * @param iSegReg The index of the segment register to use for
6782 * this access. The base and limits are checked.
6783 * @param GCPtrMem The address of the guest memory.
6784 */
6785uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6786{
6787 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6788 sizeof(*pu32Src) - 1);
6789 uint32_t const u32Ret = *pu32Src;
6790 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6791 return u32Ret;
6792}
6793
6794
6795/**
6796 * Fetches a data dword, longjmp on error.
6797 *
6798 * @returns The dword
6799 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6800 * @param iSegReg The index of the segment register to use for
6801 * this access. The base and limits are checked.
6802 * @param GCPtrMem The address of the guest memory.
6803 */
6804uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6805{
6806# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
6807 /*
6808 * Convert from segmented to flat address and check that it doesn't cross a page boundrary.
6809 */
6810 RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
6811 if (RT_LIKELY((GCPtrEff & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(uint32_t)))
6812 {
6813 /*
6814 * TLB lookup.
6815 */
6816 uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrEff);
6817 PIEMTLBENTRY pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6818 if (pTlbe->uTag == uTag)
6819 {
6820 /*
6821 * Check TLB page table level access flags.
6822 */
6823 uint64_t const fNoUser = pVCpu->iem.s.uCpl == 3 ? IEMTLBE_F_PT_NO_USER : 0;
6824 if ( (pTlbe->fFlagsAndPhysRev & ( IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PG_NO_READ
6825 | IEMTLBE_F_PT_NO_ACCESSED | IEMTLBE_F_NO_MAPPINGR3 | fNoUser))
6826 == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6827 {
6828 STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6829
6830 /*
6831 * Alignment check:
6832 */
6833 /** @todo check priority \#AC vs \#PF */
6834 if ( !(GCPtrEff & (sizeof(uint32_t) - 1))
6835 || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
6836 || !pVCpu->cpum.GstCtx.eflags.Bits.u1AC
6837 || pVCpu->iem.s.uCpl != 3)
6838 {
6839 /*
6840 * Fetch and return the dword
6841 */
6842 Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
6843 Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
6844 return *(uint32_t const *)&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
6845 }
6846 Log10(("iemMemFetchDataU32Jmp: Raising #AC for %RGv\n", GCPtrEff));
6847 iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6848 }
6849 }
6850 }
6851
6852 /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
6853 outdated page pointer, or other troubles. */
6854 Log10(("iemMemFetchDataU32Jmp: %u:%RGv fallback\n", iSegReg, GCPtrMem));
6855 return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
6856
6857# else
6858 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem,
6859 IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6860 uint32_t const u32Ret = *pu32Src;
6861 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6862 return u32Ret;
6863# endif
6864}
6865#endif
6866
6867
6868#ifdef SOME_UNUSED_FUNCTION
6869/**
6870 * Fetches a data dword and sign extends it to a qword.
6871 *
6872 * @returns Strict VBox status code.
6873 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6874 * @param pu64Dst Where to return the sign extended value.
6875 * @param iSegReg The index of the segment register to use for
6876 * this access. The base and limits are checked.
6877 * @param GCPtrMem The address of the guest memory.
6878 */
6879VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6880{
6881 /* The lazy approach for now... */
6882 int32_t const *pi32Src;
6883 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem,
6884 IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1);
6885 if (rc == VINF_SUCCESS)
6886 {
6887 *pu64Dst = *pi32Src;
6888 rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
6889 }
6890#ifdef __GNUC__ /* warning: GCC may be a royal pain */
6891 else
6892 *pu64Dst = 0;
6893#endif
6894 return rc;
6895}
6896#endif
6897
6898
6899/**
6900 * Fetches a data qword.
6901 *
6902 * @returns Strict VBox status code.
6903 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6904 * @param pu64Dst Where to return the qword.
6905 * @param iSegReg The index of the segment register to use for
6906 * this access. The base and limits are checked.
6907 * @param GCPtrMem The address of the guest memory.
6908 */
6909VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6910{
6911 /* The lazy approach for now... */
6912 uint64_t const *pu64Src;
6913 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem,
6914 IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6915 if (rc == VINF_SUCCESS)
6916 {
6917 *pu64Dst = *pu64Src;
6918 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6919 }
6920 return rc;
6921}
6922
6923
6924#ifdef IEM_WITH_SETJMP
6925/**
6926 * Fetches a data qword, longjmp on error.
6927 *
6928 * @returns The qword.
6929 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6930 * @param iSegReg The index of the segment register to use for
6931 * this access. The base and limits are checked.
6932 * @param GCPtrMem The address of the guest memory.
6933 */
6934uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6935{
6936 /* The lazy approach for now... */
6937 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem,
6938 IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6939 uint64_t const u64Ret = *pu64Src;
6940 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6941 return u64Ret;
6942}
6943#endif
6944
6945
6946/**
6947 * Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
6948 *
6949 * @returns Strict VBox status code.
6950 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6951 * @param pu64Dst Where to return the qword.
6952 * @param iSegReg The index of the segment register to use for
6953 * this access. The base and limits are checked.
6954 * @param GCPtrMem The address of the guest memory.
6955 */
6956VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6957{
6958 /* The lazy approach for now... */
6959 uint64_t const *pu64Src;
6960 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem,
6961 IEM_ACCESS_DATA_R, 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
6962 if (rc == VINF_SUCCESS)
6963 {
6964 *pu64Dst = *pu64Src;
6965 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6966 }
6967 return rc;
6968}
6969
6970
6971#ifdef IEM_WITH_SETJMP
6972/**
6973 * Fetches a data qword, longjmp on error.
6974 *
6975 * @returns The qword.
6976 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6977 * @param iSegReg The index of the segment register to use for
6978 * this access. The base and limits are checked.
6979 * @param GCPtrMem The address of the guest memory.
6980 */
6981uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6982{
6983 /* The lazy approach for now... */
6984 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6985 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
6986 uint64_t const u64Ret = *pu64Src;
6987 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6988 return u64Ret;
6989}
6990#endif
6991
6992
6993/**
6994 * Fetches a data tword.
6995 *
6996 * @returns Strict VBox status code.
6997 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6998 * @param pr80Dst Where to return the tword.
6999 * @param iSegReg The index of the segment register to use for
7000 * this access. The base and limits are checked.
7001 * @param GCPtrMem The address of the guest memory.
7002 */
7003VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7004{
7005 /* The lazy approach for now... */
7006 PCRTFLOAT80U pr80Src;
7007 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
7008 if (rc == VINF_SUCCESS)
7009 {
7010 *pr80Dst = *pr80Src;
7011 rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7012 }
7013 return rc;
7014}
7015
7016
7017#ifdef IEM_WITH_SETJMP
7018/**
7019 * Fetches a data tword, longjmp on error.
7020 *
7021 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7022 * @param pr80Dst Where to return the tword.
7023 * @param iSegReg The index of the segment register to use for
7024 * this access. The base and limits are checked.
7025 * @param GCPtrMem The address of the guest memory.
7026 */
7027void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7028{
7029 /* The lazy approach for now... */
7030 PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
7031 *pr80Dst = *pr80Src;
7032 iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7033}
7034#endif
7035
7036
7037/**
7038 * Fetches a data decimal tword.
7039 *
7040 * @returns Strict VBox status code.
7041 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7042 * @param pd80Dst Where to return the tword.
7043 * @param iSegReg The index of the segment register to use for
7044 * this access. The base and limits are checked.
7045 * @param GCPtrMem The address of the guest memory.
7046 */
7047VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7048{
7049 /* The lazy approach for now... */
7050 PCRTPBCD80U pd80Src;
7051 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pd80Src, sizeof(*pd80Src), iSegReg, GCPtrMem,
7052 IEM_ACCESS_DATA_R, 7 /** @todo FBLD alignment check */);
7053 if (rc == VINF_SUCCESS)
7054 {
7055 *pd80Dst = *pd80Src;
7056 rc = iemMemCommitAndUnmap(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7057 }
7058 return rc;
7059}
7060
7061
7062#ifdef IEM_WITH_SETJMP
7063/**
7064 * Fetches a data decimal tword, longjmp on error.
7065 *
7066 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7067 * @param pd80Dst Where to return the tword.
7068 * @param iSegReg The index of the segment register to use for
7069 * this access. The base and limits are checked.
7070 * @param GCPtrMem The address of the guest memory.
7071 */
7072void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7073{
7074 /* The lazy approach for now... */
7075 PCRTPBCD80U pd80Src = (PCRTPBCD80U)iemMemMapJmp(pVCpu, sizeof(*pd80Src), iSegReg, GCPtrMem,
7076 IEM_ACCESS_DATA_R, 7 /** @todo FBSTP alignment check */);
7077 *pd80Dst = *pd80Src;
7078 iemMemCommitAndUnmapJmp(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7079}
7080#endif
7081
7082
7083/**
7084 * Fetches a data dqword (double qword), generally SSE related.
7085 *
7086 * @returns Strict VBox status code.
7087 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7088 * @param pu128Dst Where to return the qword.
7089 * @param iSegReg The index of the segment register to use for
7090 * this access. The base and limits are checked.
7091 * @param GCPtrMem The address of the guest memory.
7092 */
7093VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7094{
7095 /* The lazy approach for now... */
7096 PCRTUINT128U pu128Src;
7097 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem,
7098 IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7099 if (rc == VINF_SUCCESS)
7100 {
7101 pu128Dst->au64[0] = pu128Src->au64[0];
7102 pu128Dst->au64[1] = pu128Src->au64[1];
7103 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7104 }
7105 return rc;
7106}
7107
7108
7109#ifdef IEM_WITH_SETJMP
7110/**
7111 * Fetches a data dqword (double qword), generally SSE related.
7112 *
7113 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7114 * @param pu128Dst Where to return the qword.
7115 * @param iSegReg The index of the segment register to use for
7116 * this access. The base and limits are checked.
7117 * @param GCPtrMem The address of the guest memory.
7118 */
7119void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7120{
7121 /* The lazy approach for now... */
7122 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem,
7123 IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7124 pu128Dst->au64[0] = pu128Src->au64[0];
7125 pu128Dst->au64[1] = pu128Src->au64[1];
7126 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7127}
7128#endif
7129
7130
7131/**
7132 * Fetches a data dqword (double qword) at an aligned address, generally SSE
7133 * related.
7134 *
7135 * Raises \#GP(0) if not aligned.
7136 *
7137 * @returns Strict VBox status code.
7138 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7139 * @param pu128Dst Where to return the qword.
7140 * @param iSegReg The index of the segment register to use for
7141 * this access. The base and limits are checked.
7142 * @param GCPtrMem The address of the guest memory.
7143 */
7144VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7145{
7146 /* The lazy approach for now... */
7147 PCRTUINT128U pu128Src;
7148 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem,
7149 IEM_ACCESS_DATA_R, (sizeof(*pu128Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
7150 if (rc == VINF_SUCCESS)
7151 {
7152 pu128Dst->au64[0] = pu128Src->au64[0];
7153 pu128Dst->au64[1] = pu128Src->au64[1];
7154 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7155 }
7156 return rc;
7157}
7158
7159
7160#ifdef IEM_WITH_SETJMP
7161/**
7162 * Fetches a data dqword (double qword) at an aligned address, generally SSE
7163 * related, longjmp on error.
7164 *
7165 * Raises \#GP(0) if not aligned.
7166 *
7167 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7168 * @param pu128Dst Where to return the qword.
7169 * @param iSegReg The index of the segment register to use for
7170 * this access. The base and limits are checked.
7171 * @param GCPtrMem The address of the guest memory.
7172 */
7173void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7174{
7175 /* The lazy approach for now... */
7176 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7177 (sizeof(*pu128Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
7178 pu128Dst->au64[0] = pu128Src->au64[0];
7179 pu128Dst->au64[1] = pu128Src->au64[1];
7180 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7181}
7182#endif
7183
7184
7185/**
7186 * Fetches a data oword (octo word), generally AVX related.
7187 *
7188 * @returns Strict VBox status code.
7189 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7190 * @param pu256Dst Where to return the qword.
7191 * @param iSegReg The index of the segment register to use for
7192 * this access. The base and limits are checked.
7193 * @param GCPtrMem The address of the guest memory.
7194 */
7195VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7196{
7197 /* The lazy approach for now... */
7198 PCRTUINT256U pu256Src;
7199 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem,
7200 IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7201 if (rc == VINF_SUCCESS)
7202 {
7203 pu256Dst->au64[0] = pu256Src->au64[0];
7204 pu256Dst->au64[1] = pu256Src->au64[1];
7205 pu256Dst->au64[2] = pu256Src->au64[2];
7206 pu256Dst->au64[3] = pu256Src->au64[3];
7207 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7208 }
7209 return rc;
7210}
7211
7212
7213#ifdef IEM_WITH_SETJMP
7214/**
7215 * Fetches a data oword (octo word), generally AVX related.
7216 *
7217 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7218 * @param pu256Dst Where to return the qword.
7219 * @param iSegReg The index of the segment register to use for
7220 * this access. The base and limits are checked.
7221 * @param GCPtrMem The address of the guest memory.
7222 */
7223void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7224{
7225 /* The lazy approach for now... */
7226 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem,
7227 IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7228 pu256Dst->au64[0] = pu256Src->au64[0];
7229 pu256Dst->au64[1] = pu256Src->au64[1];
7230 pu256Dst->au64[2] = pu256Src->au64[2];
7231 pu256Dst->au64[3] = pu256Src->au64[3];
7232 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7233}
7234#endif
7235
7236
7237/**
7238 * Fetches a data oword (octo word) at an aligned address, generally AVX
7239 * related.
7240 *
7241 * Raises \#GP(0) if not aligned.
7242 *
7243 * @returns Strict VBox status code.
7244 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7245 * @param pu256Dst Where to return the qword.
7246 * @param iSegReg The index of the segment register to use for
7247 * this access. The base and limits are checked.
7248 * @param GCPtrMem The address of the guest memory.
7249 */
7250VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7251{
7252 /* The lazy approach for now... */
7253 PCRTUINT256U pu256Src;
7254 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem,
7255 IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
7256 if (rc == VINF_SUCCESS)
7257 {
7258 pu256Dst->au64[0] = pu256Src->au64[0];
7259 pu256Dst->au64[1] = pu256Src->au64[1];
7260 pu256Dst->au64[2] = pu256Src->au64[2];
7261 pu256Dst->au64[3] = pu256Src->au64[3];
7262 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7263 }
7264 return rc;
7265}
7266
7267
7268#ifdef IEM_WITH_SETJMP
7269/**
7270 * Fetches a data oword (octo word) at an aligned address, generally AVX
7271 * related, longjmp on error.
7272 *
7273 * Raises \#GP(0) if not aligned.
7274 *
7275 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7276 * @param pu256Dst Where to return the qword.
7277 * @param iSegReg The index of the segment register to use for
7278 * this access. The base and limits are checked.
7279 * @param GCPtrMem The address of the guest memory.
7280 */
7281void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7282{
7283 /* The lazy approach for now... */
7284 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7285 (sizeof(*pu256Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
7286 pu256Dst->au64[0] = pu256Src->au64[0];
7287 pu256Dst->au64[1] = pu256Src->au64[1];
7288 pu256Dst->au64[2] = pu256Src->au64[2];
7289 pu256Dst->au64[3] = pu256Src->au64[3];
7290 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7291}
7292#endif
7293
7294
7295
7296/**
7297 * Fetches a descriptor register (lgdt, lidt).
7298 *
7299 * @returns Strict VBox status code.
7300 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7301 * @param pcbLimit Where to return the limit.
7302 * @param pGCPtrBase Where to return the base.
7303 * @param iSegReg The index of the segment register to use for
7304 * this access. The base and limits are checked.
7305 * @param GCPtrMem The address of the guest memory.
7306 * @param enmOpSize The effective operand size.
7307 */
7308VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
7309 RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT
7310{
7311 /*
7312 * Just like SIDT and SGDT, the LIDT and LGDT instructions are a
7313 * little special:
7314 * - The two reads are done separately.
7315 * - Operand size override works in 16-bit and 32-bit code, but 64-bit.
7316 * - We suspect the 386 to actually commit the limit before the base in
7317 * some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
7318 * don't try emulate this eccentric behavior, because it's not well
7319 * enough understood and rather hard to trigger.
7320 * - The 486 seems to do a dword limit read when the operand size is 32-bit.
7321 */
7322 VBOXSTRICTRC rcStrict;
7323 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7324 {
7325 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7326 if (rcStrict == VINF_SUCCESS)
7327 rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
7328 }
7329 else
7330 {
7331 uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
7332 if (enmOpSize == IEMMODE_32BIT)
7333 {
7334 if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
7335 {
7336 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7337 if (rcStrict == VINF_SUCCESS)
7338 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7339 }
7340 else
7341 {
7342 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
7343 if (rcStrict == VINF_SUCCESS)
7344 {
7345 *pcbLimit = (uint16_t)uTmp;
7346 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7347 }
7348 }
7349 if (rcStrict == VINF_SUCCESS)
7350 *pGCPtrBase = uTmp;
7351 }
7352 else
7353 {
7354 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7355 if (rcStrict == VINF_SUCCESS)
7356 {
7357 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7358 if (rcStrict == VINF_SUCCESS)
7359 *pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
7360 }
7361 }
7362 }
7363 return rcStrict;
7364}
7365
7366
7367
7368/**
7369 * Stores a data byte.
7370 *
7371 * @returns Strict VBox status code.
7372 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7373 * @param iSegReg The index of the segment register to use for
7374 * this access. The base and limits are checked.
7375 * @param GCPtrMem The address of the guest memory.
7376 * @param u8Value The value to store.
7377 */
7378VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7379{
7380 /* The lazy approach for now... */
7381 uint8_t *pu8Dst;
7382 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7383 if (rc == VINF_SUCCESS)
7384 {
7385 *pu8Dst = u8Value;
7386 rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7387 }
7388 return rc;
7389}
7390
7391
7392#ifdef IEM_WITH_SETJMP
7393/**
7394 * Stores a data byte, longjmp on error.
7395 *
7396 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7397 * @param iSegReg The index of the segment register to use for
7398 * this access. The base and limits are checked.
7399 * @param GCPtrMem The address of the guest memory.
7400 * @param u8Value The value to store.
7401 */
7402void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7403{
7404 /* The lazy approach for now... */
7405 uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7406 *pu8Dst = u8Value;
7407 iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7408}
7409#endif
7410
7411
7412/**
7413 * Stores a data word.
7414 *
7415 * @returns Strict VBox status code.
7416 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7417 * @param iSegReg The index of the segment register to use for
7418 * this access. The base and limits are checked.
7419 * @param GCPtrMem The address of the guest memory.
7420 * @param u16Value The value to store.
7421 */
7422VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7423{
7424 /* The lazy approach for now... */
7425 uint16_t *pu16Dst;
7426 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem,
7427 IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7428 if (rc == VINF_SUCCESS)
7429 {
7430 *pu16Dst = u16Value;
7431 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7432 }
7433 return rc;
7434}
7435
7436
7437#ifdef IEM_WITH_SETJMP
7438/**
7439 * Stores a data word, longjmp on error.
7440 *
7441 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7442 * @param iSegReg The index of the segment register to use for
7443 * this access. The base and limits are checked.
7444 * @param GCPtrMem The address of the guest memory.
7445 * @param u16Value The value to store.
7446 */
7447void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7448{
7449 /* The lazy approach for now... */
7450 uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem,
7451 IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7452 *pu16Dst = u16Value;
7453 iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7454}
7455#endif
7456
7457
7458/**
7459 * Stores a data dword.
7460 *
7461 * @returns Strict VBox status code.
7462 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7463 * @param iSegReg The index of the segment register to use for
7464 * this access. The base and limits are checked.
7465 * @param GCPtrMem The address of the guest memory.
7466 * @param u32Value The value to store.
7467 */
7468VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7469{
7470 /* The lazy approach for now... */
7471 uint32_t *pu32Dst;
7472 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem,
7473 IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7474 if (rc == VINF_SUCCESS)
7475 {
7476 *pu32Dst = u32Value;
7477 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7478 }
7479 return rc;
7480}
7481
7482
7483#ifdef IEM_WITH_SETJMP
7484/**
7485 * Stores a data dword.
7486 *
7487 * @returns Strict VBox status code.
7488 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7489 * @param iSegReg The index of the segment register to use for
7490 * this access. The base and limits are checked.
7491 * @param GCPtrMem The address of the guest memory.
7492 * @param u32Value The value to store.
7493 */
7494void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7495{
7496 /* The lazy approach for now... */
7497 uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem,
7498 IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7499 *pu32Dst = u32Value;
7500 iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7501}
7502#endif
7503
7504
7505/**
7506 * Stores a data qword.
7507 *
7508 * @returns Strict VBox status code.
7509 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7510 * @param iSegReg The index of the segment register to use for
7511 * this access. The base and limits are checked.
7512 * @param GCPtrMem The address of the guest memory.
7513 * @param u64Value The value to store.
7514 */
7515VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7516{
7517 /* The lazy approach for now... */
7518 uint64_t *pu64Dst;
7519 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem,
7520 IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7521 if (rc == VINF_SUCCESS)
7522 {
7523 *pu64Dst = u64Value;
7524 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7525 }
7526 return rc;
7527}
7528
7529
7530#ifdef IEM_WITH_SETJMP
7531/**
7532 * Stores a data qword, longjmp on error.
7533 *
7534 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7535 * @param iSegReg The index of the segment register to use for
7536 * this access. The base and limits are checked.
7537 * @param GCPtrMem The address of the guest memory.
7538 * @param u64Value The value to store.
7539 */
7540void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7541{
7542 /* The lazy approach for now... */
7543 uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem,
7544 IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7545 *pu64Dst = u64Value;
7546 iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7547}
7548#endif
7549
7550
7551/**
7552 * Stores a data dqword.
7553 *
7554 * @returns Strict VBox status code.
7555 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7556 * @param iSegReg The index of the segment register to use for
7557 * this access. The base and limits are checked.
7558 * @param GCPtrMem The address of the guest memory.
7559 * @param u128Value The value to store.
7560 */
7561VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7562{
7563 /* The lazy approach for now... */
7564 PRTUINT128U pu128Dst;
7565 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem,
7566 IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7567 if (rc == VINF_SUCCESS)
7568 {
7569 pu128Dst->au64[0] = u128Value.au64[0];
7570 pu128Dst->au64[1] = u128Value.au64[1];
7571 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7572 }
7573 return rc;
7574}
7575
7576
7577#ifdef IEM_WITH_SETJMP
7578/**
7579 * Stores a data dqword, longjmp on error.
7580 *
7581 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7582 * @param iSegReg The index of the segment register to use for
7583 * this access. The base and limits are checked.
7584 * @param GCPtrMem The address of the guest memory.
7585 * @param u128Value The value to store.
7586 */
7587void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7588{
7589 /* The lazy approach for now... */
7590 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem,
7591 IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7592 pu128Dst->au64[0] = u128Value.au64[0];
7593 pu128Dst->au64[1] = u128Value.au64[1];
7594 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7595}
7596#endif
7597
7598
7599/**
7600 * Stores a data dqword, SSE aligned.
7601 *
7602 * @returns Strict VBox status code.
7603 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7604 * @param iSegReg The index of the segment register to use for
7605 * this access. The base and limits are checked.
7606 * @param GCPtrMem The address of the guest memory.
7607 * @param u128Value The value to store.
7608 */
7609VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7610{
7611 /* The lazy approach for now... */
7612 PRTUINT128U pu128Dst;
7613 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7614 (sizeof(*pu128Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
7615 if (rc == VINF_SUCCESS)
7616 {
7617 pu128Dst->au64[0] = u128Value.au64[0];
7618 pu128Dst->au64[1] = u128Value.au64[1];
7619 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7620 }
7621 return rc;
7622}
7623
7624
7625#ifdef IEM_WITH_SETJMP
7626/**
7627 * Stores a data dqword, SSE aligned.
7628 *
7629 * @returns Strict VBox status code.
7630 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7631 * @param iSegReg The index of the segment register to use for
7632 * this access. The base and limits are checked.
7633 * @param GCPtrMem The address of the guest memory.
7634 * @param u128Value The value to store.
7635 */
7636void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7637{
7638 /* The lazy approach for now... */
7639 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7640 (sizeof(*pu128Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
7641 pu128Dst->au64[0] = u128Value.au64[0];
7642 pu128Dst->au64[1] = u128Value.au64[1];
7643 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7644}
7645#endif
7646
7647
7648/**
7649 * Stores a data dqword.
7650 *
7651 * @returns Strict VBox status code.
7652 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7653 * @param iSegReg The index of the segment register to use for
7654 * this access. The base and limits are checked.
7655 * @param GCPtrMem The address of the guest memory.
7656 * @param pu256Value Pointer to the value to store.
7657 */
7658VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7659{
7660 /* The lazy approach for now... */
7661 PRTUINT256U pu256Dst;
7662 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7663 IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7664 if (rc == VINF_SUCCESS)
7665 {
7666 pu256Dst->au64[0] = pu256Value->au64[0];
7667 pu256Dst->au64[1] = pu256Value->au64[1];
7668 pu256Dst->au64[2] = pu256Value->au64[2];
7669 pu256Dst->au64[3] = pu256Value->au64[3];
7670 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7671 }
7672 return rc;
7673}
7674
7675
7676#ifdef IEM_WITH_SETJMP
7677/**
7678 * Stores a data dqword, longjmp on error.
7679 *
7680 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7681 * @param iSegReg The index of the segment register to use for
7682 * this access. The base and limits are checked.
7683 * @param GCPtrMem The address of the guest memory.
7684 * @param pu256Value Pointer to the value to store.
7685 */
7686void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7687{
7688 /* The lazy approach for now... */
7689 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7690 IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7691 pu256Dst->au64[0] = pu256Value->au64[0];
7692 pu256Dst->au64[1] = pu256Value->au64[1];
7693 pu256Dst->au64[2] = pu256Value->au64[2];
7694 pu256Dst->au64[3] = pu256Value->au64[3];
7695 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7696}
7697#endif
7698
7699
7700/**
7701 * Stores a data dqword, AVX \#GP(0) aligned.
7702 *
7703 * @returns Strict VBox status code.
7704 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7705 * @param iSegReg The index of the segment register to use for
7706 * this access. The base and limits are checked.
7707 * @param GCPtrMem The address of the guest memory.
7708 * @param pu256Value Pointer to the value to store.
7709 */
7710VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7711{
7712 /* The lazy approach for now... */
7713 PRTUINT256U pu256Dst;
7714 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7715 IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP);
7716 if (rc == VINF_SUCCESS)
7717 {
7718 pu256Dst->au64[0] = pu256Value->au64[0];
7719 pu256Dst->au64[1] = pu256Value->au64[1];
7720 pu256Dst->au64[2] = pu256Value->au64[2];
7721 pu256Dst->au64[3] = pu256Value->au64[3];
7722 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7723 }
7724 return rc;
7725}
7726
7727
7728#ifdef IEM_WITH_SETJMP
7729/**
7730 * Stores a data dqword, AVX aligned.
7731 *
7732 * @returns Strict VBox status code.
7733 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7734 * @param iSegReg The index of the segment register to use for
7735 * this access. The base and limits are checked.
7736 * @param GCPtrMem The address of the guest memory.
7737 * @param pu256Value Pointer to the value to store.
7738 */
7739void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7740{
7741 /* The lazy approach for now... */
7742 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7743 IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP);
7744 pu256Dst->au64[0] = pu256Value->au64[0];
7745 pu256Dst->au64[1] = pu256Value->au64[1];
7746 pu256Dst->au64[2] = pu256Value->au64[2];
7747 pu256Dst->au64[3] = pu256Value->au64[3];
7748 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7749}
7750#endif
7751
7752
7753/**
7754 * Stores a descriptor register (sgdt, sidt).
7755 *
7756 * @returns Strict VBox status code.
7757 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7758 * @param cbLimit The limit.
7759 * @param GCPtrBase The base address.
7760 * @param iSegReg The index of the segment register to use for
7761 * this access. The base and limits are checked.
7762 * @param GCPtrMem The address of the guest memory.
7763 */
7764VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7765{
7766 /*
7767 * The SIDT and SGDT instructions actually stores the data using two
7768 * independent writes (see bs3CpuBasic2_sidt_sgdt_One). The instructions
7769 * does not respond to opsize prefixes.
7770 */
7771 VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
7772 if (rcStrict == VINF_SUCCESS)
7773 {
7774 if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
7775 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
7776 IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
7777 ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
7778 else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
7779 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
7780 else
7781 rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
7782 }
7783 return rcStrict;
7784}
7785
7786
7787/**
7788 * Pushes a word onto the stack.
7789 *
7790 * @returns Strict VBox status code.
7791 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7792 * @param u16Value The value to push.
7793 */
7794VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT
7795{
7796 /* Increment the stack pointer. */
7797 uint64_t uNewRsp;
7798 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
7799
7800 /* Write the word the lazy way. */
7801 uint16_t *pu16Dst;
7802 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop,
7803 IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
7804 if (rc == VINF_SUCCESS)
7805 {
7806 *pu16Dst = u16Value;
7807 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
7808 }
7809
7810 /* Commit the new RSP value unless we an access handler made trouble. */
7811 if (rc == VINF_SUCCESS)
7812 pVCpu->cpum.GstCtx.rsp = uNewRsp;
7813
7814 return rc;
7815}
7816
7817
7818/**
7819 * Pushes a dword onto the stack.
7820 *
7821 * @returns Strict VBox status code.
7822 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7823 * @param u32Value The value to push.
7824 */
7825VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7826{
7827 /* Increment the stack pointer. */
7828 uint64_t uNewRsp;
7829 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7830
7831 /* Write the dword the lazy way. */
7832 uint32_t *pu32Dst;
7833 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop,
7834 IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
7835 if (rc == VINF_SUCCESS)
7836 {
7837 *pu32Dst = u32Value;
7838 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
7839 }
7840
7841 /* Commit the new RSP value unless we an access handler made trouble. */
7842 if (rc == VINF_SUCCESS)
7843 pVCpu->cpum.GstCtx.rsp = uNewRsp;
7844
7845 return rc;
7846}
7847
7848
7849/**
7850 * Pushes a dword segment register value onto the stack.
7851 *
7852 * @returns Strict VBox status code.
7853 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7854 * @param u32Value The value to push.
7855 */
7856VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7857{
7858 /* Increment the stack pointer. */
7859 uint64_t uNewRsp;
7860 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7861
7862 /* The intel docs talks about zero extending the selector register
7863 value. My actual intel CPU here might be zero extending the value
7864 but it still only writes the lower word... */
7865 /** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
7866 * happens when crossing an electric page boundrary, is the high word checked
7867 * for write accessibility or not? Probably it is. What about segment limits?
7868 * It appears this behavior is also shared with trap error codes.
7869 *
7870 * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
7871 * ancient hardware when it actually did change. */
7872 uint16_t *pu16Dst;
7873 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop,
7874 IEM_ACCESS_STACK_RW, sizeof(*pu16Dst) - 1); /** @todo 2 or 4 alignment check for PUSH SS? */
7875 if (rc == VINF_SUCCESS)
7876 {
7877 *pu16Dst = (uint16_t)u32Value;
7878 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
7879 }
7880
7881 /* Commit the new RSP value unless we an access handler made trouble. */
7882 if (rc == VINF_SUCCESS)
7883 pVCpu->cpum.GstCtx.rsp = uNewRsp;
7884
7885 return rc;
7886}
7887
7888
7889/**
7890 * Pushes a qword onto the stack.
7891 *
7892 * @returns Strict VBox status code.
7893 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7894 * @param u64Value The value to push.
7895 */
7896VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT
7897{
7898 /* Increment the stack pointer. */
7899 uint64_t uNewRsp;
7900 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
7901
7902 /* Write the word the lazy way. */
7903 uint64_t *pu64Dst;
7904 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop,
7905 IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
7906 if (rc == VINF_SUCCESS)
7907 {
7908 *pu64Dst = u64Value;
7909 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
7910 }
7911
7912 /* Commit the new RSP value unless we an access handler made trouble. */
7913 if (rc == VINF_SUCCESS)
7914 pVCpu->cpum.GstCtx.rsp = uNewRsp;
7915
7916 return rc;
7917}
7918
7919
7920/**
7921 * Pops a word from the stack.
7922 *
7923 * @returns Strict VBox status code.
7924 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7925 * @param pu16Value Where to store the popped value.
7926 */
7927VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT
7928{
7929 /* Increment the stack pointer. */
7930 uint64_t uNewRsp;
7931 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
7932
7933 /* Write the word the lazy way. */
7934 uint16_t const *pu16Src;
7935 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop,
7936 IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
7937 if (rc == VINF_SUCCESS)
7938 {
7939 *pu16Value = *pu16Src;
7940 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
7941
7942 /* Commit the new RSP value. */
7943 if (rc == VINF_SUCCESS)
7944 pVCpu->cpum.GstCtx.rsp = uNewRsp;
7945 }
7946
7947 return rc;
7948}
7949
7950
7951/**
7952 * Pops a dword from the stack.
7953 *
7954 * @returns Strict VBox status code.
7955 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7956 * @param pu32Value Where to store the popped value.
7957 */
7958VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT
7959{
7960 /* Increment the stack pointer. */
7961 uint64_t uNewRsp;
7962 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
7963
7964 /* Write the word the lazy way. */
7965 uint32_t const *pu32Src;
7966 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop,
7967 IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
7968 if (rc == VINF_SUCCESS)
7969 {
7970 *pu32Value = *pu32Src;
7971 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
7972
7973 /* Commit the new RSP value. */
7974 if (rc == VINF_SUCCESS)
7975 pVCpu->cpum.GstCtx.rsp = uNewRsp;
7976 }
7977
7978 return rc;
7979}
7980
7981
7982/**
7983 * Pops a qword from the stack.
7984 *
7985 * @returns Strict VBox status code.
7986 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7987 * @param pu64Value Where to store the popped value.
7988 */
7989VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT
7990{
7991 /* Increment the stack pointer. */
7992 uint64_t uNewRsp;
7993 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
7994
7995 /* Write the word the lazy way. */
7996 uint64_t const *pu64Src;
7997 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop,
7998 IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
7999 if (rc == VINF_SUCCESS)
8000 {
8001 *pu64Value = *pu64Src;
8002 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
8003
8004 /* Commit the new RSP value. */
8005 if (rc == VINF_SUCCESS)
8006 pVCpu->cpum.GstCtx.rsp = uNewRsp;
8007 }
8008
8009 return rc;
8010}
8011
8012
8013/**
8014 * Pushes a word onto the stack, using a temporary stack pointer.
8015 *
8016 * @returns Strict VBox status code.
8017 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8018 * @param u16Value The value to push.
8019 * @param pTmpRsp Pointer to the temporary stack pointer.
8020 */
8021VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8022{
8023 /* Increment the stack pointer. */
8024 RTUINT64U NewRsp = *pTmpRsp;
8025 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
8026
8027 /* Write the word the lazy way. */
8028 uint16_t *pu16Dst;
8029 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop,
8030 IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
8031 if (rc == VINF_SUCCESS)
8032 {
8033 *pu16Dst = u16Value;
8034 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
8035 }
8036
8037 /* Commit the new RSP value unless we an access handler made trouble. */
8038 if (rc == VINF_SUCCESS)
8039 *pTmpRsp = NewRsp;
8040
8041 return rc;
8042}
8043
8044
8045/**
8046 * Pushes a dword onto the stack, using a temporary stack pointer.
8047 *
8048 * @returns Strict VBox status code.
8049 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8050 * @param u32Value The value to push.
8051 * @param pTmpRsp Pointer to the temporary stack pointer.
8052 */
8053VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8054{
8055 /* Increment the stack pointer. */
8056 RTUINT64U NewRsp = *pTmpRsp;
8057 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
8058
8059 /* Write the word the lazy way. */
8060 uint32_t *pu32Dst;
8061 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop,
8062 IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
8063 if (rc == VINF_SUCCESS)
8064 {
8065 *pu32Dst = u32Value;
8066 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
8067 }
8068
8069 /* Commit the new RSP value unless we an access handler made trouble. */
8070 if (rc == VINF_SUCCESS)
8071 *pTmpRsp = NewRsp;
8072
8073 return rc;
8074}
8075
8076
8077/**
8078 * Pushes a dword onto the stack, using a temporary stack pointer.
8079 *
8080 * @returns Strict VBox status code.
8081 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8082 * @param u64Value The value to push.
8083 * @param pTmpRsp Pointer to the temporary stack pointer.
8084 */
8085VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8086{
8087 /* Increment the stack pointer. */
8088 RTUINT64U NewRsp = *pTmpRsp;
8089 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
8090
8091 /* Write the word the lazy way. */
8092 uint64_t *pu64Dst;
8093 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop,
8094 IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
8095 if (rc == VINF_SUCCESS)
8096 {
8097 *pu64Dst = u64Value;
8098 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
8099 }
8100
8101 /* Commit the new RSP value unless we an access handler made trouble. */
8102 if (rc == VINF_SUCCESS)
8103 *pTmpRsp = NewRsp;
8104
8105 return rc;
8106}
8107
8108
8109/**
8110 * Pops a word from the stack, using a temporary stack pointer.
8111 *
8112 * @returns Strict VBox status code.
8113 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8114 * @param pu16Value Where to store the popped value.
8115 * @param pTmpRsp Pointer to the temporary stack pointer.
8116 */
8117VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8118{
8119 /* Increment the stack pointer. */
8120 RTUINT64U NewRsp = *pTmpRsp;
8121 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
8122
8123 /* Write the word the lazy way. */
8124 uint16_t const *pu16Src;
8125 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop,
8126 IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
8127 if (rc == VINF_SUCCESS)
8128 {
8129 *pu16Value = *pu16Src;
8130 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
8131
8132 /* Commit the new RSP value. */
8133 if (rc == VINF_SUCCESS)
8134 *pTmpRsp = NewRsp;
8135 }
8136
8137 return rc;
8138}
8139
8140
8141/**
8142 * Pops a dword from the stack, using a temporary stack pointer.
8143 *
8144 * @returns Strict VBox status code.
8145 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8146 * @param pu32Value Where to store the popped value.
8147 * @param pTmpRsp Pointer to the temporary stack pointer.
8148 */
8149VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8150{
8151 /* Increment the stack pointer. */
8152 RTUINT64U NewRsp = *pTmpRsp;
8153 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
8154
8155 /* Write the word the lazy way. */
8156 uint32_t const *pu32Src;
8157 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop,
8158 IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
8159 if (rc == VINF_SUCCESS)
8160 {
8161 *pu32Value = *pu32Src;
8162 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
8163
8164 /* Commit the new RSP value. */
8165 if (rc == VINF_SUCCESS)
8166 *pTmpRsp = NewRsp;
8167 }
8168
8169 return rc;
8170}
8171
8172
8173/**
8174 * Pops a qword from the stack, using a temporary stack pointer.
8175 *
8176 * @returns Strict VBox status code.
8177 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8178 * @param pu64Value Where to store the popped value.
8179 * @param pTmpRsp Pointer to the temporary stack pointer.
8180 */
8181VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8182{
8183 /* Increment the stack pointer. */
8184 RTUINT64U NewRsp = *pTmpRsp;
8185 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
8186
8187 /* Write the word the lazy way. */
8188 uint64_t const *pu64Src;
8189 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop,
8190 IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
8191 if (rcStrict == VINF_SUCCESS)
8192 {
8193 *pu64Value = *pu64Src;
8194 rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
8195
8196 /* Commit the new RSP value. */
8197 if (rcStrict == VINF_SUCCESS)
8198 *pTmpRsp = NewRsp;
8199 }
8200
8201 return rcStrict;
8202}
8203
8204
8205/**
8206 * Begin a special stack push (used by interrupt, exceptions and such).
8207 *
8208 * This will raise \#SS or \#PF if appropriate.
8209 *
8210 * @returns Strict VBox status code.
8211 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8212 * @param cbMem The number of bytes to push onto the stack.
8213 * @param cbAlign The alignment mask (7, 3, 1).
8214 * @param ppvMem Where to return the pointer to the stack memory.
8215 * As with the other memory functions this could be
8216 * direct access or bounce buffered access, so
8217 * don't commit register until the commit call
8218 * succeeds.
8219 * @param puNewRsp Where to return the new RSP value. This must be
8220 * passed unchanged to
8221 * iemMemStackPushCommitSpecial().
8222 */
8223VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8224 void **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT
8225{
8226 Assert(cbMem < UINT8_MAX);
8227 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
8228 return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop,
8229 IEM_ACCESS_STACK_W, cbAlign);
8230}
8231
8232
8233/**
8234 * Commits a special stack push (started by iemMemStackPushBeginSpecial).
8235 *
8236 * This will update the rSP.
8237 *
8238 * @returns Strict VBox status code.
8239 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8240 * @param pvMem The pointer returned by
8241 * iemMemStackPushBeginSpecial().
8242 * @param uNewRsp The new RSP value returned by
8243 * iemMemStackPushBeginSpecial().
8244 */
8245VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT
8246{
8247 VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
8248 if (rcStrict == VINF_SUCCESS)
8249 pVCpu->cpum.GstCtx.rsp = uNewRsp;
8250 return rcStrict;
8251}
8252
8253
8254/**
8255 * Begin a special stack pop (used by iret, retf and such).
8256 *
8257 * This will raise \#SS or \#PF if appropriate.
8258 *
8259 * @returns Strict VBox status code.
8260 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8261 * @param cbMem The number of bytes to pop from the stack.
8262 * @param cbAlign The alignment mask (7, 3, 1).
8263 * @param ppvMem Where to return the pointer to the stack memory.
8264 * @param puNewRsp Where to return the new RSP value. This must be
8265 * assigned to CPUMCTX::rsp manually some time
8266 * after iemMemStackPopDoneSpecial() has been
8267 * called.
8268 */
8269VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8270 void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT
8271{
8272 Assert(cbMem < UINT8_MAX);
8273 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
8274 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign);
8275}
8276
8277
8278/**
8279 * Continue a special stack pop (used by iret and retf), for the purpose of
8280 * retrieving a new stack pointer.
8281 *
8282 * This will raise \#SS or \#PF if appropriate.
8283 *
8284 * @returns Strict VBox status code.
8285 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8286 * @param off Offset from the top of the stack. This is zero
8287 * except in the retf case.
8288 * @param cbMem The number of bytes to pop from the stack.
8289 * @param ppvMem Where to return the pointer to the stack memory.
8290 * @param uCurNewRsp The current uncommitted RSP value. (No need to
8291 * return this because all use of this function is
8292 * to retrieve a new value and anything we return
8293 * here would be discarded.)
8294 */
8295VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t off, size_t cbMem,
8296 void const **ppvMem, uint64_t uCurNewRsp) RT_NOEXCEPT
8297{
8298 Assert(cbMem < UINT8_MAX);
8299
8300 /* The essense of iemRegGetRspForPopEx and friends: */ /** @todo put this into a inlined function? */
8301 RTGCPTR GCPtrTop;
8302 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
8303 GCPtrTop = uCurNewRsp;
8304 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
8305 GCPtrTop = (uint32_t)uCurNewRsp;
8306 else
8307 GCPtrTop = (uint16_t)uCurNewRsp;
8308
8309 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop + off, IEM_ACCESS_STACK_R,
8310 0 /* checked in iemMemStackPopBeginSpecial */);
8311}
8312
8313
8314/**
8315 * Done with a special stack pop (started by iemMemStackPopBeginSpecial or
8316 * iemMemStackPopContinueSpecial).
8317 *
8318 * The caller will manually commit the rSP.
8319 *
8320 * @returns Strict VBox status code.
8321 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8322 * @param pvMem The pointer returned by
8323 * iemMemStackPopBeginSpecial() or
8324 * iemMemStackPopContinueSpecial().
8325 */
8326VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT
8327{
8328 return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
8329}
8330
8331
8332/**
8333 * Fetches a system table byte.
8334 *
8335 * @returns Strict VBox status code.
8336 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8337 * @param pbDst Where to return the byte.
8338 * @param iSegReg The index of the segment register to use for
8339 * this access. The base and limits are checked.
8340 * @param GCPtrMem The address of the guest memory.
8341 */
8342VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8343{
8344 /* The lazy approach for now... */
8345 uint8_t const *pbSrc;
8346 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8347 if (rc == VINF_SUCCESS)
8348 {
8349 *pbDst = *pbSrc;
8350 rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
8351 }
8352 return rc;
8353}
8354
8355
8356/**
8357 * Fetches a system table word.
8358 *
8359 * @returns Strict VBox status code.
8360 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8361 * @param pu16Dst Where to return the word.
8362 * @param iSegReg The index of the segment register to use for
8363 * this access. The base and limits are checked.
8364 * @param GCPtrMem The address of the guest memory.
8365 */
8366VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8367{
8368 /* The lazy approach for now... */
8369 uint16_t const *pu16Src;
8370 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8371 if (rc == VINF_SUCCESS)
8372 {
8373 *pu16Dst = *pu16Src;
8374 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
8375 }
8376 return rc;
8377}
8378
8379
8380/**
8381 * Fetches a system table dword.
8382 *
8383 * @returns Strict VBox status code.
8384 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8385 * @param pu32Dst Where to return the dword.
8386 * @param iSegReg The index of the segment register to use for
8387 * this access. The base and limits are checked.
8388 * @param GCPtrMem The address of the guest memory.
8389 */
8390VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8391{
8392 /* The lazy approach for now... */
8393 uint32_t const *pu32Src;
8394 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8395 if (rc == VINF_SUCCESS)
8396 {
8397 *pu32Dst = *pu32Src;
8398 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
8399 }
8400 return rc;
8401}
8402
8403
8404/**
8405 * Fetches a system table qword.
8406 *
8407 * @returns Strict VBox status code.
8408 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8409 * @param pu64Dst Where to return the qword.
8410 * @param iSegReg The index of the segment register to use for
8411 * this access. The base and limits are checked.
8412 * @param GCPtrMem The address of the guest memory.
8413 */
8414VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8415{
8416 /* The lazy approach for now... */
8417 uint64_t const *pu64Src;
8418 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8419 if (rc == VINF_SUCCESS)
8420 {
8421 *pu64Dst = *pu64Src;
8422 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
8423 }
8424 return rc;
8425}
8426
8427
8428/**
8429 * Fetches a descriptor table entry with caller specified error code.
8430 *
8431 * @returns Strict VBox status code.
8432 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8433 * @param pDesc Where to return the descriptor table entry.
8434 * @param uSel The selector which table entry to fetch.
8435 * @param uXcpt The exception to raise on table lookup error.
8436 * @param uErrorCode The error code associated with the exception.
8437 */
8438static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
8439 uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT
8440{
8441 AssertPtr(pDesc);
8442 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
8443
8444 /** @todo did the 286 require all 8 bytes to be accessible? */
8445 /*
8446 * Get the selector table base and check bounds.
8447 */
8448 RTGCPTR GCPtrBase;
8449 if (uSel & X86_SEL_LDT)
8450 {
8451 if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
8452 || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
8453 {
8454 Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
8455 uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
8456 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
8457 uErrorCode, 0);
8458 }
8459
8460 Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
8461 GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
8462 }
8463 else
8464 {
8465 if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
8466 {
8467 Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
8468 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
8469 uErrorCode, 0);
8470 }
8471 GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
8472 }
8473
8474 /*
8475 * Read the legacy descriptor and maybe the long mode extensions if
8476 * required.
8477 */
8478 VBOXSTRICTRC rcStrict;
8479 if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
8480 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
8481 else
8482 {
8483 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
8484 if (rcStrict == VINF_SUCCESS)
8485 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
8486 if (rcStrict == VINF_SUCCESS)
8487 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
8488 if (rcStrict == VINF_SUCCESS)
8489 pDesc->Legacy.au16[3] = 0;
8490 else
8491 return rcStrict;
8492 }
8493
8494 if (rcStrict == VINF_SUCCESS)
8495 {
8496 if ( !IEM_IS_LONG_MODE(pVCpu)
8497 || pDesc->Legacy.Gen.u1DescType)
8498 pDesc->Long.au64[1] = 0;
8499 else if ((uint32_t)(uSel | X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
8500 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1);
8501 else
8502 {
8503 Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
8504 /** @todo is this the right exception? */
8505 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
8506 }
8507 }
8508 return rcStrict;
8509}
8510
8511
8512/**
8513 * Fetches a descriptor table entry.
8514 *
8515 * @returns Strict VBox status code.
8516 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8517 * @param pDesc Where to return the descriptor table entry.
8518 * @param uSel The selector which table entry to fetch.
8519 * @param uXcpt The exception to raise on table lookup error.
8520 */
8521VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT
8522{
8523 return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
8524}
8525
8526
8527/**
8528 * Marks the selector descriptor as accessed (only non-system descriptors).
8529 *
8530 * This function ASSUMES that iemMemFetchSelDesc has be called previously and
8531 * will therefore skip the limit checks.
8532 *
8533 * @returns Strict VBox status code.
8534 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8535 * @param uSel The selector.
8536 */
8537VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
8538{
8539 /*
8540 * Get the selector table base and calculate the entry address.
8541 */
8542 RTGCPTR GCPtr = uSel & X86_SEL_LDT
8543 ? pVCpu->cpum.GstCtx.ldtr.u64Base
8544 : pVCpu->cpum.GstCtx.gdtr.pGdt;
8545 GCPtr += uSel & X86_SEL_MASK;
8546
8547 /*
8548 * ASMAtomicBitSet will assert if the address is misaligned, so do some
8549 * ugly stuff to avoid this. This will make sure it's an atomic access
8550 * as well more or less remove any question about 8-bit or 32-bit accesss.
8551 */
8552 VBOXSTRICTRC rcStrict;
8553 uint32_t volatile *pu32;
8554 if ((GCPtr & 3) == 0)
8555 {
8556 /* The normal case, map the 32-bit bits around the accessed bit (40). */
8557 GCPtr += 2 + 2;
8558 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8559 if (rcStrict != VINF_SUCCESS)
8560 return rcStrict;
8561 ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
8562 }
8563 else
8564 {
8565 /* The misaligned GDT/LDT case, map the whole thing. */
8566 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8567 if (rcStrict != VINF_SUCCESS)
8568 return rcStrict;
8569 switch ((uintptr_t)pu32 & 3)
8570 {
8571 case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
8572 case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
8573 case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
8574 case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
8575 }
8576 }
8577
8578 return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
8579}
8580
8581/** @} */
8582
8583/** @name Opcode Helpers.
8584 * @{
8585 */
8586
8587/**
8588 * Calculates the effective address of a ModR/M memory operand.
8589 *
8590 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8591 *
8592 * @return Strict VBox status code.
8593 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8594 * @param bRm The ModRM byte.
8595 * @param cbImm The size of any immediate following the
8596 * effective address opcode bytes. Important for
8597 * RIP relative addressing.
8598 * @param pGCPtrEff Where to return the effective address.
8599 */
8600VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff) RT_NOEXCEPT
8601{
8602 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8603# define SET_SS_DEF() \
8604 do \
8605 { \
8606 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8607 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8608 } while (0)
8609
8610 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8611 {
8612/** @todo Check the effective address size crap! */
8613 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8614 {
8615 uint16_t u16EffAddr;
8616
8617 /* Handle the disp16 form with no registers first. */
8618 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
8619 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8620 else
8621 {
8622 /* Get the displacment. */
8623 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8624 {
8625 case 0: u16EffAddr = 0; break;
8626 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8627 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8628 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8629 }
8630
8631 /* Add the base and index registers to the disp. */
8632 switch (bRm & X86_MODRM_RM_MASK)
8633 {
8634 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8635 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8636 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8637 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8638 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8639 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8640 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8641 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8642 }
8643 }
8644
8645 *pGCPtrEff = u16EffAddr;
8646 }
8647 else
8648 {
8649 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8650 uint32_t u32EffAddr;
8651
8652 /* Handle the disp32 form with no registers first. */
8653 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
8654 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8655 else
8656 {
8657 /* Get the register (or SIB) value. */
8658 switch ((bRm & X86_MODRM_RM_MASK))
8659 {
8660 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8661 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8662 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8663 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8664 case 4: /* SIB */
8665 {
8666 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8667
8668 /* Get the index and scale it. */
8669 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8670 {
8671 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8672 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8673 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8674 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8675 case 4: u32EffAddr = 0; /*none */ break;
8676 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8677 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8678 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8679 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8680 }
8681 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8682
8683 /* add base */
8684 switch (bSib & X86_SIB_BASE_MASK)
8685 {
8686 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8687 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8688 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8689 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8690 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
8691 case 5:
8692 if ((bRm & X86_MODRM_MOD_MASK) != 0)
8693 {
8694 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8695 SET_SS_DEF();
8696 }
8697 else
8698 {
8699 uint32_t u32Disp;
8700 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8701 u32EffAddr += u32Disp;
8702 }
8703 break;
8704 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8705 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8706 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8707 }
8708 break;
8709 }
8710 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8711 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8712 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8713 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8714 }
8715
8716 /* Get and add the displacement. */
8717 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8718 {
8719 case 0:
8720 break;
8721 case 1:
8722 {
8723 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8724 u32EffAddr += i8Disp;
8725 break;
8726 }
8727 case 2:
8728 {
8729 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8730 u32EffAddr += u32Disp;
8731 break;
8732 }
8733 default:
8734 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
8735 }
8736
8737 }
8738 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
8739 *pGCPtrEff = u32EffAddr;
8740 else
8741 {
8742 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
8743 *pGCPtrEff = u32EffAddr & UINT16_MAX;
8744 }
8745 }
8746 }
8747 else
8748 {
8749 uint64_t u64EffAddr;
8750
8751 /* Handle the rip+disp32 form with no registers first. */
8752 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
8753 {
8754 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8755 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
8756 }
8757 else
8758 {
8759 /* Get the register (or SIB) value. */
8760 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
8761 {
8762 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8763 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8764 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8765 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8766 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8767 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8768 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8769 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8770 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8771 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8772 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8773 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8774 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8775 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8776 /* SIB */
8777 case 4:
8778 case 12:
8779 {
8780 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8781
8782 /* Get the index and scale it. */
8783 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
8784 {
8785 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8786 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8787 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8788 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8789 case 4: u64EffAddr = 0; /*none */ break;
8790 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8791 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8792 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8793 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8794 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8795 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8796 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8797 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8798 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8799 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8800 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8801 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8802 }
8803 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8804
8805 /* add base */
8806 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
8807 {
8808 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8809 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8810 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8811 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8812 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
8813 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8814 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8815 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8816 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8817 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8818 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8819 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8820 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8821 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8822 /* complicated encodings */
8823 case 5:
8824 case 13:
8825 if ((bRm & X86_MODRM_MOD_MASK) != 0)
8826 {
8827 if (!pVCpu->iem.s.uRexB)
8828 {
8829 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8830 SET_SS_DEF();
8831 }
8832 else
8833 u64EffAddr += pVCpu->cpum.GstCtx.r13;
8834 }
8835 else
8836 {
8837 uint32_t u32Disp;
8838 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8839 u64EffAddr += (int32_t)u32Disp;
8840 }
8841 break;
8842 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8843 }
8844 break;
8845 }
8846 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8847 }
8848
8849 /* Get and add the displacement. */
8850 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8851 {
8852 case 0:
8853 break;
8854 case 1:
8855 {
8856 int8_t i8Disp;
8857 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8858 u64EffAddr += i8Disp;
8859 break;
8860 }
8861 case 2:
8862 {
8863 uint32_t u32Disp;
8864 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8865 u64EffAddr += (int32_t)u32Disp;
8866 break;
8867 }
8868 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
8869 }
8870
8871 }
8872
8873 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8874 *pGCPtrEff = u64EffAddr;
8875 else
8876 {
8877 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8878 *pGCPtrEff = u64EffAddr & UINT32_MAX;
8879 }
8880 }
8881
8882 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
8883 return VINF_SUCCESS;
8884}
8885
8886
8887/**
8888 * Calculates the effective address of a ModR/M memory operand.
8889 *
8890 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8891 *
8892 * @return Strict VBox status code.
8893 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8894 * @param bRm The ModRM byte.
8895 * @param cbImm The size of any immediate following the
8896 * effective address opcode bytes. Important for
8897 * RIP relative addressing.
8898 * @param pGCPtrEff Where to return the effective address.
8899 * @param offRsp RSP displacement.
8900 */
8901VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp) RT_NOEXCEPT
8902{
8903 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8904# define SET_SS_DEF() \
8905 do \
8906 { \
8907 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8908 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8909 } while (0)
8910
8911 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8912 {
8913/** @todo Check the effective address size crap! */
8914 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8915 {
8916 uint16_t u16EffAddr;
8917
8918 /* Handle the disp16 form with no registers first. */
8919 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
8920 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8921 else
8922 {
8923 /* Get the displacment. */
8924 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8925 {
8926 case 0: u16EffAddr = 0; break;
8927 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8928 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8929 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8930 }
8931
8932 /* Add the base and index registers to the disp. */
8933 switch (bRm & X86_MODRM_RM_MASK)
8934 {
8935 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8936 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8937 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8938 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8939 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8940 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8941 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8942 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8943 }
8944 }
8945
8946 *pGCPtrEff = u16EffAddr;
8947 }
8948 else
8949 {
8950 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8951 uint32_t u32EffAddr;
8952
8953 /* Handle the disp32 form with no registers first. */
8954 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
8955 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8956 else
8957 {
8958 /* Get the register (or SIB) value. */
8959 switch ((bRm & X86_MODRM_RM_MASK))
8960 {
8961 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8962 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8963 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8964 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8965 case 4: /* SIB */
8966 {
8967 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8968
8969 /* Get the index and scale it. */
8970 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8971 {
8972 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8973 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8974 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8975 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8976 case 4: u32EffAddr = 0; /*none */ break;
8977 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8978 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8979 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8980 IEM_NOT_REACHED_DEFAULT_CASE_RET();
8981 }
8982 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8983
8984 /* add base */
8985 switch (bSib & X86_SIB_BASE_MASK)
8986 {
8987 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8988 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8989 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8990 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8991 case 4:
8992 u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
8993 SET_SS_DEF();
8994 break;
8995 case 5:
8996 if ((bRm & X86_MODRM_MOD_MASK) != 0)
8997 {
8998 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8999 SET_SS_DEF();
9000 }
9001 else
9002 {
9003 uint32_t u32Disp;
9004 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9005 u32EffAddr += u32Disp;
9006 }
9007 break;
9008 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
9009 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
9010 IEM_NOT_REACHED_DEFAULT_CASE_RET();
9011 }
9012 break;
9013 }
9014 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
9015 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9016 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9017 IEM_NOT_REACHED_DEFAULT_CASE_RET();
9018 }
9019
9020 /* Get and add the displacement. */
9021 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9022 {
9023 case 0:
9024 break;
9025 case 1:
9026 {
9027 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9028 u32EffAddr += i8Disp;
9029 break;
9030 }
9031 case 2:
9032 {
9033 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9034 u32EffAddr += u32Disp;
9035 break;
9036 }
9037 default:
9038 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
9039 }
9040
9041 }
9042 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9043 *pGCPtrEff = u32EffAddr;
9044 else
9045 {
9046 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9047 *pGCPtrEff = u32EffAddr & UINT16_MAX;
9048 }
9049 }
9050 }
9051 else
9052 {
9053 uint64_t u64EffAddr;
9054
9055 /* Handle the rip+disp32 form with no registers first. */
9056 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
9057 {
9058 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9059 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9060 }
9061 else
9062 {
9063 /* Get the register (or SIB) value. */
9064 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
9065 {
9066 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9067 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9068 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9069 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9070 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9071 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9072 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9073 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9074 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9075 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9076 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9077 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9078 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9079 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9080 /* SIB */
9081 case 4:
9082 case 12:
9083 {
9084 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9085
9086 /* Get the index and scale it. */
9087 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
9088 {
9089 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9090 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9091 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9092 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9093 case 4: u64EffAddr = 0; /*none */ break;
9094 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9095 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9096 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9097 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9098 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9099 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9100 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9101 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9102 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9103 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9104 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9105 IEM_NOT_REACHED_DEFAULT_CASE_RET();
9106 }
9107 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9108
9109 /* add base */
9110 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
9111 {
9112 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9113 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9114 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9115 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9116 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
9117 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9118 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9119 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9120 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9121 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9122 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9123 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9124 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9125 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9126 /* complicated encodings */
9127 case 5:
9128 case 13:
9129 if ((bRm & X86_MODRM_MOD_MASK) != 0)
9130 {
9131 if (!pVCpu->iem.s.uRexB)
9132 {
9133 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9134 SET_SS_DEF();
9135 }
9136 else
9137 u64EffAddr += pVCpu->cpum.GstCtx.r13;
9138 }
9139 else
9140 {
9141 uint32_t u32Disp;
9142 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9143 u64EffAddr += (int32_t)u32Disp;
9144 }
9145 break;
9146 IEM_NOT_REACHED_DEFAULT_CASE_RET();
9147 }
9148 break;
9149 }
9150 IEM_NOT_REACHED_DEFAULT_CASE_RET();
9151 }
9152
9153 /* Get and add the displacement. */
9154 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9155 {
9156 case 0:
9157 break;
9158 case 1:
9159 {
9160 int8_t i8Disp;
9161 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9162 u64EffAddr += i8Disp;
9163 break;
9164 }
9165 case 2:
9166 {
9167 uint32_t u32Disp;
9168 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9169 u64EffAddr += (int32_t)u32Disp;
9170 break;
9171 }
9172 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
9173 }
9174
9175 }
9176
9177 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9178 *pGCPtrEff = u64EffAddr;
9179 else
9180 {
9181 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9182 *pGCPtrEff = u64EffAddr & UINT32_MAX;
9183 }
9184 }
9185
9186 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
9187 return VINF_SUCCESS;
9188}
9189
9190
9191#ifdef IEM_WITH_SETJMP
9192/**
9193 * Calculates the effective address of a ModR/M memory operand.
9194 *
9195 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
9196 *
9197 * May longjmp on internal error.
9198 *
9199 * @return The effective address.
9200 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9201 * @param bRm The ModRM byte.
9202 * @param cbImm The size of any immediate following the
9203 * effective address opcode bytes. Important for
9204 * RIP relative addressing.
9205 */
9206RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm) RT_NOEXCEPT
9207{
9208 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
9209# define SET_SS_DEF() \
9210 do \
9211 { \
9212 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
9213 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
9214 } while (0)
9215
9216 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
9217 {
9218/** @todo Check the effective address size crap! */
9219 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
9220 {
9221 uint16_t u16EffAddr;
9222
9223 /* Handle the disp16 form with no registers first. */
9224 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
9225 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
9226 else
9227 {
9228 /* Get the displacment. */
9229 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9230 {
9231 case 0: u16EffAddr = 0; break;
9232 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
9233 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
9234 default: AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); /* (caller checked for these) */
9235 }
9236
9237 /* Add the base and index registers to the disp. */
9238 switch (bRm & X86_MODRM_RM_MASK)
9239 {
9240 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
9241 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
9242 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
9243 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
9244 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
9245 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
9246 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
9247 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
9248 }
9249 }
9250
9251 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
9252 return u16EffAddr;
9253 }
9254
9255 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9256 uint32_t u32EffAddr;
9257
9258 /* Handle the disp32 form with no registers first. */
9259 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
9260 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
9261 else
9262 {
9263 /* Get the register (or SIB) value. */
9264 switch ((bRm & X86_MODRM_RM_MASK))
9265 {
9266 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9267 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9268 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9269 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9270 case 4: /* SIB */
9271 {
9272 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9273
9274 /* Get the index and scale it. */
9275 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
9276 {
9277 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9278 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9279 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9280 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9281 case 4: u32EffAddr = 0; /*none */ break;
9282 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
9283 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9284 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9285 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9286 }
9287 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9288
9289 /* add base */
9290 switch (bSib & X86_SIB_BASE_MASK)
9291 {
9292 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
9293 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
9294 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
9295 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
9296 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
9297 case 5:
9298 if ((bRm & X86_MODRM_MOD_MASK) != 0)
9299 {
9300 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
9301 SET_SS_DEF();
9302 }
9303 else
9304 {
9305 uint32_t u32Disp;
9306 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9307 u32EffAddr += u32Disp;
9308 }
9309 break;
9310 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
9311 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
9312 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9313 }
9314 break;
9315 }
9316 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
9317 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9318 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9319 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9320 }
9321
9322 /* Get and add the displacement. */
9323 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9324 {
9325 case 0:
9326 break;
9327 case 1:
9328 {
9329 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9330 u32EffAddr += i8Disp;
9331 break;
9332 }
9333 case 2:
9334 {
9335 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9336 u32EffAddr += u32Disp;
9337 break;
9338 }
9339 default:
9340 AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); /* (caller checked for these) */
9341 }
9342 }
9343
9344 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9345 {
9346 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
9347 return u32EffAddr;
9348 }
9349 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9350 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
9351 return u32EffAddr & UINT16_MAX;
9352 }
9353
9354 uint64_t u64EffAddr;
9355
9356 /* Handle the rip+disp32 form with no registers first. */
9357 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
9358 {
9359 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9360 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9361 }
9362 else
9363 {
9364 /* Get the register (or SIB) value. */
9365 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
9366 {
9367 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9368 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9369 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9370 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9371 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9372 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9373 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9374 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9375 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9376 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9377 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9378 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9379 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9380 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9381 /* SIB */
9382 case 4:
9383 case 12:
9384 {
9385 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9386
9387 /* Get the index and scale it. */
9388 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
9389 {
9390 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9391 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9392 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9393 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9394 case 4: u64EffAddr = 0; /*none */ break;
9395 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9396 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9397 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9398 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9399 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9400 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9401 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9402 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9403 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9404 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9405 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9406 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9407 }
9408 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9409
9410 /* add base */
9411 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
9412 {
9413 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9414 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9415 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9416 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9417 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
9418 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9419 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9420 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9421 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9422 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9423 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9424 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9425 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9426 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9427 /* complicated encodings */
9428 case 5:
9429 case 13:
9430 if ((bRm & X86_MODRM_MOD_MASK) != 0)
9431 {
9432 if (!pVCpu->iem.s.uRexB)
9433 {
9434 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9435 SET_SS_DEF();
9436 }
9437 else
9438 u64EffAddr += pVCpu->cpum.GstCtx.r13;
9439 }
9440 else
9441 {
9442 uint32_t u32Disp;
9443 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9444 u64EffAddr += (int32_t)u32Disp;
9445 }
9446 break;
9447 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9448 }
9449 break;
9450 }
9451 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9452 }
9453
9454 /* Get and add the displacement. */
9455 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9456 {
9457 case 0:
9458 break;
9459 case 1:
9460 {
9461 int8_t i8Disp;
9462 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9463 u64EffAddr += i8Disp;
9464 break;
9465 }
9466 case 2:
9467 {
9468 uint32_t u32Disp;
9469 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9470 u64EffAddr += (int32_t)u32Disp;
9471 break;
9472 }
9473 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
9474 }
9475
9476 }
9477
9478 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9479 {
9480 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
9481 return u64EffAddr;
9482 }
9483 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9484 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
9485 return u64EffAddr & UINT32_MAX;
9486}
9487#endif /* IEM_WITH_SETJMP */
9488
9489/** @} */
9490
9491
9492#ifdef LOG_ENABLED
9493/**
9494 * Logs the current instruction.
9495 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9496 * @param fSameCtx Set if we have the same context information as the VMM,
9497 * clear if we may have already executed an instruction in
9498 * our debug context. When clear, we assume IEMCPU holds
9499 * valid CPU mode info.
9500 *
9501 * The @a fSameCtx parameter is now misleading and obsolete.
9502 * @param pszFunction The IEM function doing the execution.
9503 */
9504static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT
9505{
9506# ifdef IN_RING3
9507 if (LogIs2Enabled())
9508 {
9509 char szInstr[256];
9510 uint32_t cbInstr = 0;
9511 if (fSameCtx)
9512 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
9513 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
9514 szInstr, sizeof(szInstr), &cbInstr);
9515 else
9516 {
9517 uint32_t fFlags = 0;
9518 switch (pVCpu->iem.s.enmCpuMode)
9519 {
9520 case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;
9521 case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;
9522 case IEMMODE_16BIT:
9523 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
9524 fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
9525 else
9526 fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;
9527 break;
9528 }
9529 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
9530 szInstr, sizeof(szInstr), &cbInstr);
9531 }
9532
9533 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9534 Log2(("**** %s\n"
9535 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
9536 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
9537 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
9538 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
9539 " %s\n"
9540 , pszFunction,
9541 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
9542 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
9543 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
9544 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
9545 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
9546 szInstr));
9547
9548 if (LogIs3Enabled())
9549 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
9550 }
9551 else
9552# endif
9553 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
9554 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
9555 RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
9556}
9557#endif /* LOG_ENABLED */
9558
9559
9560#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9561/**
9562 * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
9563 * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
9564 *
9565 * @returns Modified rcStrict.
9566 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9567 * @param rcStrict The instruction execution status.
9568 */
9569static VBOXSTRICTRC iemHandleNestedInstructionBoundaryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
9570{
9571 Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
9572 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF))
9573 {
9574 /* VMX preemption timer takes priority over NMI-window exits. */
9575 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
9576 {
9577 rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
9578 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
9579 }
9580 /*
9581 * Check remaining intercepts.
9582 *
9583 * NMI-window and Interrupt-window VM-exits.
9584 * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
9585 * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
9586 *
9587 * See Intel spec. 26.7.6 "NMI-Window Exiting".
9588 * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9589 */
9590 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW)
9591 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9592 && !TRPMHasTrap(pVCpu))
9593 {
9594 Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
9595 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
9596 && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
9597 {
9598 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
9599 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
9600 }
9601 else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
9602 && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
9603 {
9604 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
9605 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
9606 }
9607 }
9608 }
9609 /* TPR-below threshold/APIC write has the highest priority. */
9610 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
9611 {
9612 rcStrict = iemVmxApicWriteEmulation(pVCpu);
9613 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9614 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
9615 }
9616 /* MTF takes priority over VMX-preemption timer. */
9617 else
9618 {
9619 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
9620 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9621 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
9622 }
9623 return rcStrict;
9624}
9625#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9626
9627
9628/**
9629 * The actual code execution bits of IEMExecOne, IEMExecOneEx, and
9630 * IEMExecOneWithPrefetchedByPC.
9631 *
9632 * Similar code is found in IEMExecLots.
9633 *
9634 * @return Strict VBox status code.
9635 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9636 * @param fExecuteInhibit If set, execute the instruction following CLI,
9637 * POP SS and MOV SS,GR.
9638 * @param pszFunction The calling function name.
9639 */
9640DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
9641{
9642 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9643 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9644 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9645 RT_NOREF_PV(pszFunction);
9646
9647#ifdef IEM_WITH_SETJMP
9648 VBOXSTRICTRC rcStrict;
9649 jmp_buf JmpBuf;
9650 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
9651 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9652 if ((rcStrict = setjmp(JmpBuf)) == 0)
9653 {
9654 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9655 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9656 }
9657 else
9658 pVCpu->iem.s.cLongJumps++;
9659 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9660#else
9661 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9662 VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9663#endif
9664 if (rcStrict == VINF_SUCCESS)
9665 pVCpu->iem.s.cInstructions++;
9666 if (pVCpu->iem.s.cActiveMappings > 0)
9667 {
9668 Assert(rcStrict != VINF_SUCCESS);
9669 iemMemRollback(pVCpu);
9670 }
9671 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9672 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9673 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9674
9675//#ifdef DEBUG
9676// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
9677//#endif
9678
9679#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9680 /*
9681 * Perform any VMX nested-guest instruction boundary actions.
9682 *
9683 * If any of these causes a VM-exit, we must skip executing the next
9684 * instruction (would run into stale page tables). A VM-exit makes sure
9685 * there is no interrupt-inhibition, so that should ensure we don't go
9686 * to try execute the next instruction. Clearing fExecuteInhibit is
9687 * problematic because of the setjmp/longjmp clobbering above.
9688 */
9689 if ( !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
9690 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)
9691 || rcStrict != VINF_SUCCESS)
9692 { /* likely */ }
9693 else
9694 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9695#endif
9696
9697 /* Execute the next instruction as well if a cli, pop ss or
9698 mov ss, Gr has just completed successfully. */
9699 if ( fExecuteInhibit
9700 && rcStrict == VINF_SUCCESS
9701 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9702 && EMIsInhibitInterruptsActive(pVCpu))
9703 {
9704 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock);
9705 if (rcStrict == VINF_SUCCESS)
9706 {
9707#ifdef LOG_ENABLED
9708 iemLogCurInstr(pVCpu, false, pszFunction);
9709#endif
9710#ifdef IEM_WITH_SETJMP
9711 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9712 if ((rcStrict = setjmp(JmpBuf)) == 0)
9713 {
9714 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9715 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9716 }
9717 else
9718 pVCpu->iem.s.cLongJumps++;
9719 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9720#else
9721 IEM_OPCODE_GET_NEXT_U8(&b);
9722 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9723#endif
9724 if (rcStrict == VINF_SUCCESS)
9725 {
9726 pVCpu->iem.s.cInstructions++;
9727#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9728 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
9729 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))
9730 { /* likely */ }
9731 else
9732 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9733#endif
9734 }
9735 if (pVCpu->iem.s.cActiveMappings > 0)
9736 {
9737 Assert(rcStrict != VINF_SUCCESS);
9738 iemMemRollback(pVCpu);
9739 }
9740 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9741 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9742 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9743 }
9744 else if (pVCpu->iem.s.cActiveMappings > 0)
9745 iemMemRollback(pVCpu);
9746 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
9747 }
9748
9749 /*
9750 * Return value fiddling, statistics and sanity assertions.
9751 */
9752 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9753
9754 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9755 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9756 return rcStrict;
9757}
9758
9759
9760/**
9761 * Execute one instruction.
9762 *
9763 * @return Strict VBox status code.
9764 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9765 */
9766VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
9767{
9768 AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
9769#ifdef LOG_ENABLED
9770 iemLogCurInstr(pVCpu, true, "IEMExecOne");
9771#endif
9772
9773 /*
9774 * Do the decoding and emulation.
9775 */
9776 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9777 if (rcStrict == VINF_SUCCESS)
9778 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
9779 else if (pVCpu->iem.s.cActiveMappings > 0)
9780 iemMemRollback(pVCpu);
9781
9782 if (rcStrict != VINF_SUCCESS)
9783 LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9784 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9785 return rcStrict;
9786}
9787
9788
9789VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9790{
9791 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9792
9793 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9794 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9795 if (rcStrict == VINF_SUCCESS)
9796 {
9797 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
9798 if (pcbWritten)
9799 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9800 }
9801 else if (pVCpu->iem.s.cActiveMappings > 0)
9802 iemMemRollback(pVCpu);
9803
9804 return rcStrict;
9805}
9806
9807
9808VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9809 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9810{
9811 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9812
9813 VBOXSTRICTRC rcStrict;
9814 if ( cbOpcodeBytes
9815 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9816 {
9817 iemInitDecoder(pVCpu, false, false);
9818#ifdef IEM_WITH_CODE_TLB
9819 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9820 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9821 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9822 pVCpu->iem.s.offCurInstrStart = 0;
9823 pVCpu->iem.s.offInstrNextByte = 0;
9824#else
9825 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9826 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9827#endif
9828 rcStrict = VINF_SUCCESS;
9829 }
9830 else
9831 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9832 if (rcStrict == VINF_SUCCESS)
9833 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
9834 else if (pVCpu->iem.s.cActiveMappings > 0)
9835 iemMemRollback(pVCpu);
9836
9837 return rcStrict;
9838}
9839
9840
9841VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9842{
9843 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9844
9845 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9846 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9847 if (rcStrict == VINF_SUCCESS)
9848 {
9849 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
9850 if (pcbWritten)
9851 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9852 }
9853 else if (pVCpu->iem.s.cActiveMappings > 0)
9854 iemMemRollback(pVCpu);
9855
9856 return rcStrict;
9857}
9858
9859
9860VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9861 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9862{
9863 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9864
9865 VBOXSTRICTRC rcStrict;
9866 if ( cbOpcodeBytes
9867 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9868 {
9869 iemInitDecoder(pVCpu, true, false);
9870#ifdef IEM_WITH_CODE_TLB
9871 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9872 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9873 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9874 pVCpu->iem.s.offCurInstrStart = 0;
9875 pVCpu->iem.s.offInstrNextByte = 0;
9876#else
9877 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9878 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9879#endif
9880 rcStrict = VINF_SUCCESS;
9881 }
9882 else
9883 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9884 if (rcStrict == VINF_SUCCESS)
9885 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
9886 else if (pVCpu->iem.s.cActiveMappings > 0)
9887 iemMemRollback(pVCpu);
9888
9889 return rcStrict;
9890}
9891
9892
9893/**
9894 * For debugging DISGetParamSize, may come in handy.
9895 *
9896 * @returns Strict VBox status code.
9897 * @param pVCpu The cross context virtual CPU structure of the
9898 * calling EMT.
9899 * @param pCtxCore The context core structure.
9900 * @param OpcodeBytesPC The PC of the opcode bytes.
9901 * @param pvOpcodeBytes Prefeched opcode bytes.
9902 * @param cbOpcodeBytes Number of prefetched bytes.
9903 * @param pcbWritten Where to return the number of bytes written.
9904 * Optional.
9905 */
9906VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9907 const void *pvOpcodeBytes, size_t cbOpcodeBytes,
9908 uint32_t *pcbWritten)
9909{
9910 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9911
9912 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9913 VBOXSTRICTRC rcStrict;
9914 if ( cbOpcodeBytes
9915 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9916 {
9917 iemInitDecoder(pVCpu, true, false);
9918#ifdef IEM_WITH_CODE_TLB
9919 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9920 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9921 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9922 pVCpu->iem.s.offCurInstrStart = 0;
9923 pVCpu->iem.s.offInstrNextByte = 0;
9924#else
9925 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9926 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9927#endif
9928 rcStrict = VINF_SUCCESS;
9929 }
9930 else
9931 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9932 if (rcStrict == VINF_SUCCESS)
9933 {
9934 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
9935 if (pcbWritten)
9936 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9937 }
9938 else if (pVCpu->iem.s.cActiveMappings > 0)
9939 iemMemRollback(pVCpu);
9940
9941 return rcStrict;
9942}
9943
9944
9945/**
9946 * For handling split cacheline lock operations when the host has split-lock
9947 * detection enabled.
9948 *
9949 * This will cause the interpreter to disregard the lock prefix and implicit
9950 * locking (xchg).
9951 *
9952 * @returns Strict VBox status code.
9953 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9954 */
9955VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
9956{
9957 /*
9958 * Do the decoding and emulation.
9959 */
9960 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /*fDisregardLock*/);
9961 if (rcStrict == VINF_SUCCESS)
9962 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
9963 else if (pVCpu->iem.s.cActiveMappings > 0)
9964 iemMemRollback(pVCpu);
9965
9966 if (rcStrict != VINF_SUCCESS)
9967 LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9968 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9969 return rcStrict;
9970}
9971
9972
9973VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
9974{
9975 uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
9976 AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
9977
9978 /*
9979 * See if there is an interrupt pending in TRPM, inject it if we can.
9980 */
9981 /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
9982#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9983 bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
9984 if (fIntrEnabled)
9985 {
9986 if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
9987 fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9988 else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
9989 fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
9990 else
9991 {
9992 Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
9993 fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
9994 }
9995 }
9996#else
9997 bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9998#endif
9999
10000 /** @todo What if we are injecting an exception and not an interrupt? Is that
10001 * possible here? For now we assert it is indeed only an interrupt. */
10002 if ( fIntrEnabled
10003 && TRPMHasTrap(pVCpu)
10004 && EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
10005 {
10006 uint8_t u8TrapNo;
10007 TRPMEVENT enmType;
10008 uint32_t uErrCode;
10009 RTGCPTR uCr2;
10010 int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */, NULL /* fIcebp */);
10011 AssertRC(rc2);
10012 Assert(enmType == TRPM_HARDWARE_INT);
10013 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
10014 TRPMResetTrap(pVCpu);
10015#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10016 /* Injecting an event may cause a VM-exit. */
10017 if ( rcStrict != VINF_SUCCESS
10018 && rcStrict != VINF_IEM_RAISED_XCPT)
10019 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
10020#else
10021 NOREF(rcStrict);
10022#endif
10023 }
10024
10025 /*
10026 * Initial decoder init w/ prefetch, then setup setjmp.
10027 */
10028 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10029 if (rcStrict == VINF_SUCCESS)
10030 {
10031#ifdef IEM_WITH_SETJMP
10032 jmp_buf JmpBuf;
10033 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10034 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10035 pVCpu->iem.s.cActiveMappings = 0;
10036 if ((rcStrict = setjmp(JmpBuf)) == 0)
10037#endif
10038 {
10039 /*
10040 * The run loop. We limit ourselves to 4096 instructions right now.
10041 */
10042 uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
10043 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10044 for (;;)
10045 {
10046 /*
10047 * Log the state.
10048 */
10049#ifdef LOG_ENABLED
10050 iemLogCurInstr(pVCpu, true, "IEMExecLots");
10051#endif
10052
10053 /*
10054 * Do the decoding and emulation.
10055 */
10056 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10057 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10058 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10059 {
10060 Assert(pVCpu->iem.s.cActiveMappings == 0);
10061 pVCpu->iem.s.cInstructions++;
10062
10063#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10064 /* Perform any VMX nested-guest instruction boundary actions. */
10065 uint64_t fCpu = pVCpu->fLocalForcedActions;
10066 if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
10067 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)))
10068 { /* likely */ }
10069 else
10070 {
10071 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
10072 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10073 fCpu = pVCpu->fLocalForcedActions;
10074 else
10075 {
10076 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10077 break;
10078 }
10079 }
10080#endif
10081 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10082 {
10083#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
10084 uint64_t fCpu = pVCpu->fLocalForcedActions;
10085#endif
10086 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10087 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10088 | VMCPU_FF_TLB_FLUSH
10089 | VMCPU_FF_INHIBIT_INTERRUPTS
10090 | VMCPU_FF_BLOCK_NMIS
10091 | VMCPU_FF_UNHALT );
10092
10093 if (RT_LIKELY( ( !fCpu
10094 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
10095 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
10096 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
10097 {
10098 if (cMaxInstructionsGccStupidity-- > 0)
10099 {
10100 /* Poll timers every now an then according to the caller's specs. */
10101 if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
10102 || !TMTimerPollBool(pVM, pVCpu))
10103 {
10104 Assert(pVCpu->iem.s.cActiveMappings == 0);
10105 iemReInitDecoder(pVCpu);
10106 continue;
10107 }
10108 }
10109 }
10110 }
10111 Assert(pVCpu->iem.s.cActiveMappings == 0);
10112 }
10113 else if (pVCpu->iem.s.cActiveMappings > 0)
10114 iemMemRollback(pVCpu);
10115 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10116 break;
10117 }
10118 }
10119#ifdef IEM_WITH_SETJMP
10120 else
10121 {
10122 if (pVCpu->iem.s.cActiveMappings > 0)
10123 iemMemRollback(pVCpu);
10124# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10125 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10126# endif
10127 pVCpu->iem.s.cLongJumps++;
10128 }
10129 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10130#endif
10131
10132 /*
10133 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10134 */
10135 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10136 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10137 }
10138 else
10139 {
10140 if (pVCpu->iem.s.cActiveMappings > 0)
10141 iemMemRollback(pVCpu);
10142
10143#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10144 /*
10145 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10146 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10147 */
10148 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10149#endif
10150 }
10151
10152 /*
10153 * Maybe re-enter raw-mode and log.
10154 */
10155 if (rcStrict != VINF_SUCCESS)
10156 LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10157 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10158 if (pcInstructions)
10159 *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
10160 return rcStrict;
10161}
10162
10163
10164/**
10165 * Interface used by EMExecuteExec, does exit statistics and limits.
10166 *
10167 * @returns Strict VBox status code.
10168 * @param pVCpu The cross context virtual CPU structure.
10169 * @param fWillExit To be defined.
10170 * @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
10171 * @param cMaxInstructions Maximum number of instructions to execute.
10172 * @param cMaxInstructionsWithoutExits
10173 * The max number of instructions without exits.
10174 * @param pStats Where to return statistics.
10175 */
10176VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
10177 uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
10178{
10179 NOREF(fWillExit); /** @todo define flexible exit crits */
10180
10181 /*
10182 * Initialize return stats.
10183 */
10184 pStats->cInstructions = 0;
10185 pStats->cExits = 0;
10186 pStats->cMaxExitDistance = 0;
10187 pStats->cReserved = 0;
10188
10189 /*
10190 * Initial decoder init w/ prefetch, then setup setjmp.
10191 */
10192 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10193 if (rcStrict == VINF_SUCCESS)
10194 {
10195#ifdef IEM_WITH_SETJMP
10196 jmp_buf JmpBuf;
10197 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10198 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10199 pVCpu->iem.s.cActiveMappings = 0;
10200 if ((rcStrict = setjmp(JmpBuf)) == 0)
10201#endif
10202 {
10203#ifdef IN_RING0
10204 bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
10205#endif
10206 uint32_t cInstructionSinceLastExit = 0;
10207
10208 /*
10209 * The run loop. We limit ourselves to 4096 instructions right now.
10210 */
10211 PVM pVM = pVCpu->CTX_SUFF(pVM);
10212 for (;;)
10213 {
10214 /*
10215 * Log the state.
10216 */
10217#ifdef LOG_ENABLED
10218 iemLogCurInstr(pVCpu, true, "IEMExecForExits");
10219#endif
10220
10221 /*
10222 * Do the decoding and emulation.
10223 */
10224 uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
10225
10226 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10227 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10228
10229 if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
10230 && cInstructionSinceLastExit > 0 /* don't count the first */ )
10231 {
10232 pStats->cExits += 1;
10233 if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
10234 pStats->cMaxExitDistance = cInstructionSinceLastExit;
10235 cInstructionSinceLastExit = 0;
10236 }
10237
10238 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10239 {
10240 Assert(pVCpu->iem.s.cActiveMappings == 0);
10241 pVCpu->iem.s.cInstructions++;
10242 pStats->cInstructions++;
10243 cInstructionSinceLastExit++;
10244
10245#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10246 /* Perform any VMX nested-guest instruction boundary actions. */
10247 uint64_t fCpu = pVCpu->fLocalForcedActions;
10248 if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
10249 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)))
10250 { /* likely */ }
10251 else
10252 {
10253 rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
10254 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10255 fCpu = pVCpu->fLocalForcedActions;
10256 else
10257 {
10258 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10259 break;
10260 }
10261 }
10262#endif
10263 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10264 {
10265#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
10266 uint64_t fCpu = pVCpu->fLocalForcedActions;
10267#endif
10268 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10269 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10270 | VMCPU_FF_TLB_FLUSH
10271 | VMCPU_FF_INHIBIT_INTERRUPTS
10272 | VMCPU_FF_BLOCK_NMIS
10273 | VMCPU_FF_UNHALT );
10274 if (RT_LIKELY( ( ( !fCpu
10275 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
10276 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
10277 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
10278 || pStats->cInstructions < cMinInstructions))
10279 {
10280 if (pStats->cInstructions < cMaxInstructions)
10281 {
10282 if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
10283 {
10284#ifdef IN_RING0
10285 if ( !fCheckPreemptionPending
10286 || !RTThreadPreemptIsPending(NIL_RTTHREAD))
10287#endif
10288 {
10289 Assert(pVCpu->iem.s.cActiveMappings == 0);
10290 iemReInitDecoder(pVCpu);
10291 continue;
10292 }
10293#ifdef IN_RING0
10294 rcStrict = VINF_EM_RAW_INTERRUPT;
10295 break;
10296#endif
10297 }
10298 }
10299 }
10300 Assert(!(fCpu & VMCPU_FF_IEM));
10301 }
10302 Assert(pVCpu->iem.s.cActiveMappings == 0);
10303 }
10304 else if (pVCpu->iem.s.cActiveMappings > 0)
10305 iemMemRollback(pVCpu);
10306 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10307 break;
10308 }
10309 }
10310#ifdef IEM_WITH_SETJMP
10311 else
10312 {
10313 if (pVCpu->iem.s.cActiveMappings > 0)
10314 iemMemRollback(pVCpu);
10315 pVCpu->iem.s.cLongJumps++;
10316 }
10317 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10318#endif
10319
10320 /*
10321 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10322 */
10323 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10324 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10325 }
10326 else
10327 {
10328 if (pVCpu->iem.s.cActiveMappings > 0)
10329 iemMemRollback(pVCpu);
10330
10331#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10332 /*
10333 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10334 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10335 */
10336 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10337#endif
10338 }
10339
10340 /*
10341 * Maybe re-enter raw-mode and log.
10342 */
10343 if (rcStrict != VINF_SUCCESS)
10344 LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
10345 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
10346 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
10347 return rcStrict;
10348}
10349
10350
10351/**
10352 * Injects a trap, fault, abort, software interrupt or external interrupt.
10353 *
10354 * The parameter list matches TRPMQueryTrapAll pretty closely.
10355 *
10356 * @returns Strict VBox status code.
10357 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10358 * @param u8TrapNo The trap number.
10359 * @param enmType What type is it (trap/fault/abort), software
10360 * interrupt or hardware interrupt.
10361 * @param uErrCode The error code if applicable.
10362 * @param uCr2 The CR2 value if applicable.
10363 * @param cbInstr The instruction length (only relevant for
10364 * software interrupts).
10365 */
10366VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
10367 uint8_t cbInstr)
10368{
10369 iemInitDecoder(pVCpu, false, false);
10370#ifdef DBGFTRACE_ENABLED
10371 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
10372 u8TrapNo, enmType, uErrCode, uCr2);
10373#endif
10374
10375 uint32_t fFlags;
10376 switch (enmType)
10377 {
10378 case TRPM_HARDWARE_INT:
10379 Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
10380 fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
10381 uErrCode = uCr2 = 0;
10382 break;
10383
10384 case TRPM_SOFTWARE_INT:
10385 Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
10386 fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
10387 uErrCode = uCr2 = 0;
10388 break;
10389
10390 case TRPM_TRAP:
10391 Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
10392 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
10393 if (u8TrapNo == X86_XCPT_PF)
10394 fFlags |= IEM_XCPT_FLAGS_CR2;
10395 switch (u8TrapNo)
10396 {
10397 case X86_XCPT_DF:
10398 case X86_XCPT_TS:
10399 case X86_XCPT_NP:
10400 case X86_XCPT_SS:
10401 case X86_XCPT_PF:
10402 case X86_XCPT_AC:
10403 case X86_XCPT_GP:
10404 fFlags |= IEM_XCPT_FLAGS_ERR;
10405 break;
10406 }
10407 break;
10408
10409 IEM_NOT_REACHED_DEFAULT_CASE_RET();
10410 }
10411
10412 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
10413
10414 if (pVCpu->iem.s.cActiveMappings > 0)
10415 iemMemRollback(pVCpu);
10416
10417 return rcStrict;
10418}
10419
10420
10421/**
10422 * Injects the active TRPM event.
10423 *
10424 * @returns Strict VBox status code.
10425 * @param pVCpu The cross context virtual CPU structure.
10426 */
10427VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
10428{
10429#ifndef IEM_IMPLEMENTS_TASKSWITCH
10430 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
10431#else
10432 uint8_t u8TrapNo;
10433 TRPMEVENT enmType;
10434 uint32_t uErrCode;
10435 RTGCUINTPTR uCr2;
10436 uint8_t cbInstr;
10437 int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
10438 if (RT_FAILURE(rc))
10439 return rc;
10440
10441 /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
10442 * ICEBP \#DB injection as a special case. */
10443 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
10444#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
10445 if (rcStrict == VINF_SVM_VMEXIT)
10446 rcStrict = VINF_SUCCESS;
10447#endif
10448#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10449 if (rcStrict == VINF_VMX_VMEXIT)
10450 rcStrict = VINF_SUCCESS;
10451#endif
10452 /** @todo Are there any other codes that imply the event was successfully
10453 * delivered to the guest? See @bugref{6607}. */
10454 if ( rcStrict == VINF_SUCCESS
10455 || rcStrict == VINF_IEM_RAISED_XCPT)
10456 TRPMResetTrap(pVCpu);
10457
10458 return rcStrict;
10459#endif
10460}
10461
10462
10463VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
10464{
10465 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10466 return VERR_NOT_IMPLEMENTED;
10467}
10468
10469
10470VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
10471{
10472 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10473 return VERR_NOT_IMPLEMENTED;
10474}
10475
10476
10477#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
10478/**
10479 * Executes a IRET instruction with default operand size.
10480 *
10481 * This is for PATM.
10482 *
10483 * @returns VBox status code.
10484 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10485 * @param pCtxCore The register frame.
10486 */
10487VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
10488{
10489 PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
10490
10491 iemCtxCoreToCtx(pCtx, pCtxCore);
10492 iemInitDecoder(pVCpu);
10493 VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
10494 if (rcStrict == VINF_SUCCESS)
10495 iemCtxToCtxCore(pCtxCore, pCtx);
10496 else
10497 LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10498 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10499 return rcStrict;
10500}
10501#endif
10502
10503
10504/**
10505 * Interface for HM and EM for executing string I/O OUT (write) instructions.
10506 *
10507 * This API ASSUMES that the caller has already verified that the guest code is
10508 * allowed to access the I/O port. (The I/O port is in the DX register in the
10509 * guest state.)
10510 *
10511 * @returns Strict VBox status code.
10512 * @param pVCpu The cross context virtual CPU structure.
10513 * @param cbValue The size of the I/O port access (1, 2, or 4).
10514 * @param enmAddrMode The addressing mode.
10515 * @param fRepPrefix Indicates whether a repeat prefix is used
10516 * (doesn't matter which for this instruction).
10517 * @param cbInstr The instruction length in bytes.
10518 * @param iEffSeg The effective segment address.
10519 * @param fIoChecked Whether the access to the I/O port has been
10520 * checked or not. It's typically checked in the
10521 * HM scenario.
10522 */
10523VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10524 bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
10525{
10526 AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
10527 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10528
10529 /*
10530 * State init.
10531 */
10532 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10533
10534 /*
10535 * Switch orgy for getting to the right handler.
10536 */
10537 VBOXSTRICTRC rcStrict;
10538 if (fRepPrefix)
10539 {
10540 switch (enmAddrMode)
10541 {
10542 case IEMMODE_16BIT:
10543 switch (cbValue)
10544 {
10545 case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10546 case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10547 case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10548 default:
10549 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10550 }
10551 break;
10552
10553 case IEMMODE_32BIT:
10554 switch (cbValue)
10555 {
10556 case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10557 case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10558 case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10559 default:
10560 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10561 }
10562 break;
10563
10564 case IEMMODE_64BIT:
10565 switch (cbValue)
10566 {
10567 case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10568 case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10569 case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10570 default:
10571 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10572 }
10573 break;
10574
10575 default:
10576 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10577 }
10578 }
10579 else
10580 {
10581 switch (enmAddrMode)
10582 {
10583 case IEMMODE_16BIT:
10584 switch (cbValue)
10585 {
10586 case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10587 case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10588 case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10589 default:
10590 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10591 }
10592 break;
10593
10594 case IEMMODE_32BIT:
10595 switch (cbValue)
10596 {
10597 case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10598 case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10599 case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10600 default:
10601 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10602 }
10603 break;
10604
10605 case IEMMODE_64BIT:
10606 switch (cbValue)
10607 {
10608 case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10609 case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10610 case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10611 default:
10612 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10613 }
10614 break;
10615
10616 default:
10617 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10618 }
10619 }
10620
10621 if (pVCpu->iem.s.cActiveMappings)
10622 iemMemRollback(pVCpu);
10623
10624 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10625}
10626
10627
10628/**
10629 * Interface for HM and EM for executing string I/O IN (read) instructions.
10630 *
10631 * This API ASSUMES that the caller has already verified that the guest code is
10632 * allowed to access the I/O port. (The I/O port is in the DX register in the
10633 * guest state.)
10634 *
10635 * @returns Strict VBox status code.
10636 * @param pVCpu The cross context virtual CPU structure.
10637 * @param cbValue The size of the I/O port access (1, 2, or 4).
10638 * @param enmAddrMode The addressing mode.
10639 * @param fRepPrefix Indicates whether a repeat prefix is used
10640 * (doesn't matter which for this instruction).
10641 * @param cbInstr The instruction length in bytes.
10642 * @param fIoChecked Whether the access to the I/O port has been
10643 * checked or not. It's typically checked in the
10644 * HM scenario.
10645 */
10646VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10647 bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
10648{
10649 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10650
10651 /*
10652 * State init.
10653 */
10654 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10655
10656 /*
10657 * Switch orgy for getting to the right handler.
10658 */
10659 VBOXSTRICTRC rcStrict;
10660 if (fRepPrefix)
10661 {
10662 switch (enmAddrMode)
10663 {
10664 case IEMMODE_16BIT:
10665 switch (cbValue)
10666 {
10667 case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10668 case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10669 case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10670 default:
10671 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10672 }
10673 break;
10674
10675 case IEMMODE_32BIT:
10676 switch (cbValue)
10677 {
10678 case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10679 case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10680 case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10681 default:
10682 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10683 }
10684 break;
10685
10686 case IEMMODE_64BIT:
10687 switch (cbValue)
10688 {
10689 case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10690 case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10691 case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10692 default:
10693 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10694 }
10695 break;
10696
10697 default:
10698 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10699 }
10700 }
10701 else
10702 {
10703 switch (enmAddrMode)
10704 {
10705 case IEMMODE_16BIT:
10706 switch (cbValue)
10707 {
10708 case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10709 case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10710 case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10711 default:
10712 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10713 }
10714 break;
10715
10716 case IEMMODE_32BIT:
10717 switch (cbValue)
10718 {
10719 case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10720 case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10721 case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10722 default:
10723 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10724 }
10725 break;
10726
10727 case IEMMODE_64BIT:
10728 switch (cbValue)
10729 {
10730 case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10731 case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10732 case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10733 default:
10734 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10735 }
10736 break;
10737
10738 default:
10739 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10740 }
10741 }
10742
10743 if ( pVCpu->iem.s.cActiveMappings == 0
10744 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
10745 { /* likely */ }
10746 else
10747 {
10748 AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
10749 iemMemRollback(pVCpu);
10750 }
10751 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10752}
10753
10754
10755/**
10756 * Interface for rawmode to write execute an OUT instruction.
10757 *
10758 * @returns Strict VBox status code.
10759 * @param pVCpu The cross context virtual CPU structure.
10760 * @param cbInstr The instruction length in bytes.
10761 * @param u16Port The port to read.
10762 * @param fImm Whether the port is specified using an immediate operand or
10763 * using the implicit DX register.
10764 * @param cbReg The register size.
10765 *
10766 * @remarks In ring-0 not all of the state needs to be synced in.
10767 */
10768VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10769{
10770 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10771 Assert(cbReg <= 4 && cbReg != 3);
10772
10773 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10774 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
10775 Assert(!pVCpu->iem.s.cActiveMappings);
10776 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10777}
10778
10779
10780/**
10781 * Interface for rawmode to write execute an IN instruction.
10782 *
10783 * @returns Strict VBox status code.
10784 * @param pVCpu The cross context virtual CPU structure.
10785 * @param cbInstr The instruction length in bytes.
10786 * @param u16Port The port to read.
10787 * @param fImm Whether the port is specified using an immediate operand or
10788 * using the implicit DX.
10789 * @param cbReg The register size.
10790 */
10791VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10792{
10793 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10794 Assert(cbReg <= 4 && cbReg != 3);
10795
10796 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10797 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
10798 Assert(!pVCpu->iem.s.cActiveMappings);
10799 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10800}
10801
10802
10803/**
10804 * Interface for HM and EM to write to a CRx register.
10805 *
10806 * @returns Strict VBox status code.
10807 * @param pVCpu The cross context virtual CPU structure.
10808 * @param cbInstr The instruction length in bytes.
10809 * @param iCrReg The control register number (destination).
10810 * @param iGReg The general purpose register number (source).
10811 *
10812 * @remarks In ring-0 not all of the state needs to be synced in.
10813 */
10814VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
10815{
10816 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10817 Assert(iCrReg < 16);
10818 Assert(iGReg < 16);
10819
10820 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10821 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
10822 Assert(!pVCpu->iem.s.cActiveMappings);
10823 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10824}
10825
10826
10827/**
10828 * Interface for HM and EM to read from a CRx register.
10829 *
10830 * @returns Strict VBox status code.
10831 * @param pVCpu The cross context virtual CPU structure.
10832 * @param cbInstr The instruction length in bytes.
10833 * @param iGReg The general purpose register number (destination).
10834 * @param iCrReg The control register number (source).
10835 *
10836 * @remarks In ring-0 not all of the state needs to be synced in.
10837 */
10838VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10839{
10840 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10841 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
10842 | CPUMCTX_EXTRN_APIC_TPR);
10843 Assert(iCrReg < 16);
10844 Assert(iGReg < 16);
10845
10846 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10847 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
10848 Assert(!pVCpu->iem.s.cActiveMappings);
10849 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10850}
10851
10852
10853/**
10854 * Interface for HM and EM to clear the CR0[TS] bit.
10855 *
10856 * @returns Strict VBox status code.
10857 * @param pVCpu The cross context virtual CPU structure.
10858 * @param cbInstr The instruction length in bytes.
10859 *
10860 * @remarks In ring-0 not all of the state needs to be synced in.
10861 */
10862VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
10863{
10864 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10865
10866 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10867 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
10868 Assert(!pVCpu->iem.s.cActiveMappings);
10869 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10870}
10871
10872
10873/**
10874 * Interface for HM and EM to emulate the LMSW instruction (loads CR0).
10875 *
10876 * @returns Strict VBox status code.
10877 * @param pVCpu The cross context virtual CPU structure.
10878 * @param cbInstr The instruction length in bytes.
10879 * @param uValue The value to load into CR0.
10880 * @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
10881 * memory operand. Otherwise pass NIL_RTGCPTR.
10882 *
10883 * @remarks In ring-0 not all of the state needs to be synced in.
10884 */
10885VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
10886{
10887 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10888
10889 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10890 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
10891 Assert(!pVCpu->iem.s.cActiveMappings);
10892 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10893}
10894
10895
10896/**
10897 * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
10898 *
10899 * Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
10900 *
10901 * @returns Strict VBox status code.
10902 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10903 * @param cbInstr The instruction length in bytes.
10904 * @remarks In ring-0 not all of the state needs to be synced in.
10905 * @thread EMT(pVCpu)
10906 */
10907VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
10908{
10909 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10910
10911 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10912 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
10913 Assert(!pVCpu->iem.s.cActiveMappings);
10914 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10915}
10916
10917
10918/**
10919 * Interface for HM and EM to emulate the WBINVD instruction.
10920 *
10921 * @returns Strict VBox status code.
10922 * @param pVCpu The cross context virtual CPU structure.
10923 * @param cbInstr The instruction length in bytes.
10924 *
10925 * @remarks In ring-0 not all of the state needs to be synced in.
10926 */
10927VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10928{
10929 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10930
10931 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10932 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
10933 Assert(!pVCpu->iem.s.cActiveMappings);
10934 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10935}
10936
10937
10938/**
10939 * Interface for HM and EM to emulate the INVD instruction.
10940 *
10941 * @returns Strict VBox status code.
10942 * @param pVCpu The cross context virtual CPU structure.
10943 * @param cbInstr The instruction length in bytes.
10944 *
10945 * @remarks In ring-0 not all of the state needs to be synced in.
10946 */
10947VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10948{
10949 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10950
10951 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10952 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
10953 Assert(!pVCpu->iem.s.cActiveMappings);
10954 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10955}
10956
10957
10958/**
10959 * Interface for HM and EM to emulate the INVLPG instruction.
10960 *
10961 * @returns Strict VBox status code.
10962 * @retval VINF_PGM_SYNC_CR3
10963 *
10964 * @param pVCpu The cross context virtual CPU structure.
10965 * @param cbInstr The instruction length in bytes.
10966 * @param GCPtrPage The effective address of the page to invalidate.
10967 *
10968 * @remarks In ring-0 not all of the state needs to be synced in.
10969 */
10970VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
10971{
10972 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10973
10974 iemInitExec(pVCpu, false /*fBypassHandlers*/);
10975 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
10976 Assert(!pVCpu->iem.s.cActiveMappings);
10977 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10978}
10979
10980
10981/**
10982 * Interface for HM and EM to emulate the INVPCID instruction.
10983 *
10984 * @returns Strict VBox status code.
10985 * @retval VINF_PGM_SYNC_CR3
10986 *
10987 * @param pVCpu The cross context virtual CPU structure.
10988 * @param cbInstr The instruction length in bytes.
10989 * @param iEffSeg The effective segment register.
10990 * @param GCPtrDesc The effective address of the INVPCID descriptor.
10991 * @param uType The invalidation type.
10992 *
10993 * @remarks In ring-0 not all of the state needs to be synced in.
10994 */
10995VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
10996 uint64_t uType)
10997{
10998 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
10999
11000 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11001 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
11002 Assert(!pVCpu->iem.s.cActiveMappings);
11003 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11004}
11005
11006
11007/**
11008 * Interface for HM and EM to emulate the CPUID instruction.
11009 *
11010 * @returns Strict VBox status code.
11011 *
11012 * @param pVCpu The cross context virtual CPU structure.
11013 * @param cbInstr The instruction length in bytes.
11014 *
11015 * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
11016 */
11017VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
11018{
11019 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11020 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11021
11022 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11023 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
11024 Assert(!pVCpu->iem.s.cActiveMappings);
11025 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11026}
11027
11028
11029/**
11030 * Interface for HM and EM to emulate the RDPMC instruction.
11031 *
11032 * @returns Strict VBox status code.
11033 *
11034 * @param pVCpu The cross context virtual CPU structure.
11035 * @param cbInstr The instruction length in bytes.
11036 *
11037 * @remarks Not all of the state needs to be synced in.
11038 */
11039VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
11040{
11041 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11042 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
11043
11044 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11045 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
11046 Assert(!pVCpu->iem.s.cActiveMappings);
11047 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11048}
11049
11050
11051/**
11052 * Interface for HM and EM to emulate the RDTSC instruction.
11053 *
11054 * @returns Strict VBox status code.
11055 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11056 *
11057 * @param pVCpu The cross context virtual CPU structure.
11058 * @param cbInstr The instruction length in bytes.
11059 *
11060 * @remarks Not all of the state needs to be synced in.
11061 */
11062VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
11063{
11064 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11065 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
11066
11067 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11068 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
11069 Assert(!pVCpu->iem.s.cActiveMappings);
11070 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11071}
11072
11073
11074/**
11075 * Interface for HM and EM to emulate the RDTSCP instruction.
11076 *
11077 * @returns Strict VBox status code.
11078 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11079 *
11080 * @param pVCpu The cross context virtual CPU structure.
11081 * @param cbInstr The instruction length in bytes.
11082 *
11083 * @remarks Not all of the state needs to be synced in. Recommended
11084 * to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
11085 */
11086VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
11087{
11088 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11089 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
11090
11091 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11092 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
11093 Assert(!pVCpu->iem.s.cActiveMappings);
11094 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11095}
11096
11097
11098/**
11099 * Interface for HM and EM to emulate the RDMSR instruction.
11100 *
11101 * @returns Strict VBox status code.
11102 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11103 *
11104 * @param pVCpu The cross context virtual CPU structure.
11105 * @param cbInstr The instruction length in bytes.
11106 *
11107 * @remarks Not all of the state needs to be synced in. Requires RCX and
11108 * (currently) all MSRs.
11109 */
11110VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11111{
11112 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11113 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS);
11114
11115 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11116 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
11117 Assert(!pVCpu->iem.s.cActiveMappings);
11118 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11119}
11120
11121
11122/**
11123 * Interface for HM and EM to emulate the WRMSR instruction.
11124 *
11125 * @returns Strict VBox status code.
11126 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11127 *
11128 * @param pVCpu The cross context virtual CPU structure.
11129 * @param cbInstr The instruction length in bytes.
11130 *
11131 * @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
11132 * and (currently) all MSRs.
11133 */
11134VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11135{
11136 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11137 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
11138 | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS);
11139
11140 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11141 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
11142 Assert(!pVCpu->iem.s.cActiveMappings);
11143 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11144}
11145
11146
11147/**
11148 * Interface for HM and EM to emulate the MONITOR instruction.
11149 *
11150 * @returns Strict VBox status code.
11151 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11152 *
11153 * @param pVCpu The cross context virtual CPU structure.
11154 * @param cbInstr The instruction length in bytes.
11155 *
11156 * @remarks Not all of the state needs to be synced in.
11157 * @remarks ASSUMES the default segment of DS and no segment override prefixes
11158 * are used.
11159 */
11160VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
11161{
11162 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11163 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11164
11165 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11166 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
11167 Assert(!pVCpu->iem.s.cActiveMappings);
11168 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11169}
11170
11171
11172/**
11173 * Interface for HM and EM to emulate the MWAIT instruction.
11174 *
11175 * @returns Strict VBox status code.
11176 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11177 *
11178 * @param pVCpu The cross context virtual CPU structure.
11179 * @param cbInstr The instruction length in bytes.
11180 *
11181 * @remarks Not all of the state needs to be synced in.
11182 */
11183VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
11184{
11185 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11186 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX);
11187
11188 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11189 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
11190 Assert(!pVCpu->iem.s.cActiveMappings);
11191 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11192}
11193
11194
11195/**
11196 * Interface for HM and EM to emulate the HLT instruction.
11197 *
11198 * @returns Strict VBox status code.
11199 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11200 *
11201 * @param pVCpu The cross context virtual CPU structure.
11202 * @param cbInstr The instruction length in bytes.
11203 *
11204 * @remarks Not all of the state needs to be synced in.
11205 */
11206VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
11207{
11208 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
11209
11210 iemInitExec(pVCpu, false /*fBypassHandlers*/);
11211 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
11212 Assert(!pVCpu->iem.s.cActiveMappings);
11213 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11214}
11215
11216
11217/**
11218 * Checks if IEM is in the process of delivering an event (interrupt or
11219 * exception).
11220 *
11221 * @returns true if we're in the process of raising an interrupt or exception,
11222 * false otherwise.
11223 * @param pVCpu The cross context virtual CPU structure.
11224 * @param puVector Where to store the vector associated with the
11225 * currently delivered event, optional.
11226 * @param pfFlags Where to store th event delivery flags (see
11227 * IEM_XCPT_FLAGS_XXX), optional.
11228 * @param puErr Where to store the error code associated with the
11229 * event, optional.
11230 * @param puCr2 Where to store the CR2 associated with the event,
11231 * optional.
11232 * @remarks The caller should check the flags to determine if the error code and
11233 * CR2 are valid for the event.
11234 */
11235VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2)
11236{
11237 bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
11238 if (fRaisingXcpt)
11239 {
11240 if (puVector)
11241 *puVector = pVCpu->iem.s.uCurXcpt;
11242 if (pfFlags)
11243 *pfFlags = pVCpu->iem.s.fCurXcpt;
11244 if (puErr)
11245 *puErr = pVCpu->iem.s.uCurXcptErr;
11246 if (puCr2)
11247 *puCr2 = pVCpu->iem.s.uCurXcptCr2;
11248 }
11249 return fRaisingXcpt;
11250}
11251
11252#ifdef IN_RING3
11253
11254/**
11255 * Handles the unlikely and probably fatal merge cases.
11256 *
11257 * @returns Merged status code.
11258 * @param rcStrict Current EM status code.
11259 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11260 * with @a rcStrict.
11261 * @param iMemMap The memory mapping index. For error reporting only.
11262 * @param pVCpu The cross context virtual CPU structure of the calling
11263 * thread, for error reporting only.
11264 */
11265DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
11266 unsigned iMemMap, PVMCPUCC pVCpu)
11267{
11268 if (RT_FAILURE_NP(rcStrict))
11269 return rcStrict;
11270
11271 if (RT_FAILURE_NP(rcStrictCommit))
11272 return rcStrictCommit;
11273
11274 if (rcStrict == rcStrictCommit)
11275 return rcStrictCommit;
11276
11277 AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
11278 VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
11279 pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
11280 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
11281 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
11282 return VERR_IOM_FF_STATUS_IPE;
11283}
11284
11285
11286/**
11287 * Helper for IOMR3ProcessForceFlag.
11288 *
11289 * @returns Merged status code.
11290 * @param rcStrict Current EM status code.
11291 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11292 * with @a rcStrict.
11293 * @param iMemMap The memory mapping index. For error reporting only.
11294 * @param pVCpu The cross context virtual CPU structure of the calling
11295 * thread, for error reporting only.
11296 */
11297DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
11298{
11299 /* Simple. */
11300 if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))
11301 return rcStrictCommit;
11302
11303 if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
11304 return rcStrict;
11305
11306 /* EM scheduling status codes. */
11307 if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
11308 && rcStrict <= VINF_EM_LAST))
11309 {
11310 if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
11311 && rcStrictCommit <= VINF_EM_LAST))
11312 return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
11313 }
11314
11315 /* Unlikely */
11316 return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
11317}
11318
11319
11320/**
11321 * Called by force-flag handling code when VMCPU_FF_IEM is set.
11322 *
11323 * @returns Merge between @a rcStrict and what the commit operation returned.
11324 * @param pVM The cross context VM structure.
11325 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
11326 * @param rcStrict The status code returned by ring-0 or raw-mode.
11327 */
11328VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
11329{
11330 /*
11331 * Reset the pending commit.
11332 */
11333 AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)
11334 & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),
11335 ("%#x %#x %#x\n",
11336 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11337 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
11338
11339 /*
11340 * Commit the pending bounce buffers (usually just one).
11341 */
11342 unsigned cBufs = 0;
11343 unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
11344 while (iMemMap-- > 0)
11345 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))
11346 {
11347 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
11348 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
11349 Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
11350
11351 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
11352 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
11353 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
11354
11355 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
11356 {
11357 VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
11358 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
11359 pbBuf,
11360 cbFirst,
11361 PGMACCESSORIGIN_IEM);
11362 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
11363 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
11364 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
11365 VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
11366 }
11367
11368 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
11369 {
11370 VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
11371 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
11372 pbBuf + cbFirst,
11373 cbSecond,
11374 PGMACCESSORIGIN_IEM);
11375 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
11376 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
11377 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
11378 VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
11379 }
11380 cBufs++;
11381 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
11382 }
11383
11384 AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
11385 ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
11386 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11387 pVCpu->iem.s.cActiveMappings = 0;
11388 return rcStrict;
11389}
11390
11391#endif /* IN_RING3 */
11392
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use