VirtualBox

About
Screenshots
Downloads
Documentation
End-user docs
Technical docs
Contribute
Community

Browse Source

Context Navigation

source: vbox/trunk/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h@ 84044

Last change on this file since 84044 was 83772, checked in by vboxsync, 4 years ago
VMM: VC++ 14.1 warning fixes. bugref:8489
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 227.2 KB

Line
1	/* $Id: NEMAllNativeTemplate-win.cpp.h 83772 2020-04-17 16:28:54Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, Windows code template ring-0/3.
4	*/
5
6	/*
7	* Copyright (C) 2018-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Defined Constants And Macros *
21	*********************************************************************************************************************************/
22	/** Copy back a segment from hyper-V. */
23	#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \
24	do { \
25	(a_Dst).u64Base = (a_Src).Base; \
26	(a_Dst).u32Limit = (a_Src).Limit; \
27	(a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \
28	(a_Dst).Attr.u = (a_Src).Attributes; \
29	(a_Dst).fFlags = CPUMSELREG_FLAGS_VALID; \
30	} while (0)
31
32	/** @def NEMWIN_ASSERT_MSG_REG_VAL
33	* Asserts the correctness of a register value in a message/context.
34	*/
35	#if 0
36	# define NEMWIN_NEED_GET_REGISTER
37	# if defined(IN_RING0) \|\| defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
38	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
39	do { \
40	HV_REGISTER_VALUE TmpVal; \
41	nemHCWinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
42	AssertMsg(a_Expr, a_Msg); \
43	} while (0)
44	# else
45	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
46	do { \
47	WHV_REGISTER_VALUE TmpVal; \
48	nemR3WinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
49	AssertMsg(a_Expr, a_Msg); \
50	} while (0)
51	# endif
52	#else
53	# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) do { } while (0)
54	#endif
55
56	/** @def NEMWIN_ASSERT_MSG_REG_VAL
57	* Asserts the correctness of a 64-bit register value in a message/context.
58	*/
59	#define NEMWIN_ASSERT_MSG_REG_VAL64(a_pVCpu, a_enmReg, a_u64Val) \
60	NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, (a_u64Val) == TmpVal.Reg64, \
61	(#a_u64Val "=%#RX64, expected %#RX64\n", (a_u64Val), TmpVal.Reg64))
62	/** @def NEMWIN_ASSERT_MSG_REG_VAL
63	* Asserts the correctness of a segment register value in a message/context.
64	*/
65	#define NEMWIN_ASSERT_MSG_REG_SEG(a_pVCpu, a_enmReg, a_SReg) \
66	NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, \
67	(a_SReg).Base == TmpVal.Segment.Base \
68	&& (a_SReg).Limit == TmpVal.Segment.Limit \
69	&& (a_SReg).Selector == TmpVal.Segment.Selector \
70	&& (a_SReg).Attributes == TmpVal.Segment.Attributes, \
71	( #a_SReg "=%#RX16 {%#RX64 LB %#RX32,%#RX16} expected %#RX16 {%#RX64 LB %#RX32,%#RX16}\n", \
72	(a_SReg).Selector, (a_SReg).Base, (a_SReg).Limit, (a_SReg).Attributes, \
73	TmpVal.Segment.Selector, TmpVal.Segment.Base, TmpVal.Segment.Limit, TmpVal.Segment.Attributes))
74
75
76	/*********************************************************************************************************************************
77	* Global Variables *
78	*********************************************************************************************************************************/
79	/** NEM_WIN_PAGE_STATE_XXX names. */
80	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
81
82	/** HV_INTERCEPT_ACCESS_TYPE names. */
83	static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
84
85
86	/*********************************************************************************************************************************
87	* Internal Functions *
88	*********************************************************************************************************************************/
89	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
90	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged);
91
92
93
94	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
95
96	/**
97	* Wrapper around VMMR0_DO_NEM_MAP_PAGES for a single page.
98	*
99	* @returns VBox status code.
100	* @param pVM The cross context VM structure.
101	* @param pVCpu The cross context virtual CPU structure of the caller.
102	* @param GCPhysSrc The source page. Does not need to be page aligned.
103	* @param GCPhysDst The destination page. Same as @a GCPhysSrc except for
104	* when A20 is disabled.
105	* @param fFlags HV_MAP_GPA_XXX.
106	*/
107	DECLINLINE(int) nemHCWinHypercallMapPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, uint32_t fFlags)
108	{
109	#ifdef IN_RING0
110	/** @todo optimize further, caller generally has the physical address. */
111	return nemR0WinMapPages(pVM, pVCpu,
112	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
113	GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
114	1, fFlags);
115	#else
116	pVCpu->nem.s.Hypercall.MapPages.GCPhysSrc = GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
117	pVCpu->nem.s.Hypercall.MapPages.GCPhysDst = GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
118	pVCpu->nem.s.Hypercall.MapPages.cPages = 1;
119	pVCpu->nem.s.Hypercall.MapPages.fFlags = fFlags;
120	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_MAP_PAGES, 0, NULL);
121	#endif
122	}
123
124
125	/**
126	* Wrapper around VMMR0_DO_NEM_UNMAP_PAGES for a single page.
127	*
128	* @returns VBox status code.
129	* @param pVM The cross context VM structure.
130	* @param pVCpu The cross context virtual CPU structure of the caller.
131	* @param GCPhys The page to unmap. Does not need to be page aligned.
132	*/
133	DECLINLINE(int) nemHCWinHypercallUnmapPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
134	{
135	# ifdef IN_RING0
136	return nemR0WinUnmapPages(pVM, pVCpu, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, 1);
137	# else
138	pVCpu->nem.s.Hypercall.UnmapPages.GCPhys = GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK;
139	pVCpu->nem.s.Hypercall.UnmapPages.cPages = 1;
140	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_UNMAP_PAGES, 0, NULL);
141	# endif
142	}
143
144	#endif /* NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
145	#ifndef IN_RING0
146
147	NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVMCC pVM, PVMCPUCC pVCpu)
148	{
149	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
150	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
151	if (pVM->nem.s.fUseRing0Runloop)
152	# endif
153	{
154	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_EXPORT_STATE, 0, NULL);
155	AssertLogRelRCReturn(rc, rc);
156	return rc;
157	}
158	# endif
159	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
160
161	/*
162	* The following is very similar to what nemR0WinExportState() does.
163	*/
164	WHV_REGISTER_NAME aenmNames[128];
165	WHV_REGISTER_VALUE aValues[128];
166
167	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK);
168	if ( !fWhat
169	&& pVCpu->nem.s.fCurrentInterruptWindows == pVCpu->nem.s.fDesiredInterruptWindows)
170	return VINF_SUCCESS;
171	uintptr_t iReg = 0;
172
173	# define ADD_REG64(a_enmName, a_uValue) do { \
174	aenmNames[iReg] = (a_enmName); \
175	aValues[iReg].Reg128.High64 = 0; \
176	aValues[iReg].Reg64 = (a_uValue); \
177	iReg++; \
178	} while (0)
179	# define ADD_REG128(a_enmName, a_uValueLo, a_uValueHi) do { \
180	aenmNames[iReg] = (a_enmName); \
181	aValues[iReg].Reg128.Low64 = (a_uValueLo); \
182	aValues[iReg].Reg128.High64 = (a_uValueHi); \
183	iReg++; \
184	} while (0)
185
186	/* GPRs */
187	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
188	{
189	if (fWhat & CPUMCTX_EXTRN_RAX)
190	ADD_REG64(WHvX64RegisterRax, pVCpu->cpum.GstCtx.rax);
191	if (fWhat & CPUMCTX_EXTRN_RCX)
192	ADD_REG64(WHvX64RegisterRcx, pVCpu->cpum.GstCtx.rcx);
193	if (fWhat & CPUMCTX_EXTRN_RDX)
194	ADD_REG64(WHvX64RegisterRdx, pVCpu->cpum.GstCtx.rdx);
195	if (fWhat & CPUMCTX_EXTRN_RBX)
196	ADD_REG64(WHvX64RegisterRbx, pVCpu->cpum.GstCtx.rbx);
197	if (fWhat & CPUMCTX_EXTRN_RSP)
198	ADD_REG64(WHvX64RegisterRsp, pVCpu->cpum.GstCtx.rsp);
199	if (fWhat & CPUMCTX_EXTRN_RBP)
200	ADD_REG64(WHvX64RegisterRbp, pVCpu->cpum.GstCtx.rbp);
201	if (fWhat & CPUMCTX_EXTRN_RSI)
202	ADD_REG64(WHvX64RegisterRsi, pVCpu->cpum.GstCtx.rsi);
203	if (fWhat & CPUMCTX_EXTRN_RDI)
204	ADD_REG64(WHvX64RegisterRdi, pVCpu->cpum.GstCtx.rdi);
205	if (fWhat & CPUMCTX_EXTRN_R8_R15)
206	{
207	ADD_REG64(WHvX64RegisterR8, pVCpu->cpum.GstCtx.r8);
208	ADD_REG64(WHvX64RegisterR9, pVCpu->cpum.GstCtx.r9);
209	ADD_REG64(WHvX64RegisterR10, pVCpu->cpum.GstCtx.r10);
210	ADD_REG64(WHvX64RegisterR11, pVCpu->cpum.GstCtx.r11);
211	ADD_REG64(WHvX64RegisterR12, pVCpu->cpum.GstCtx.r12);
212	ADD_REG64(WHvX64RegisterR13, pVCpu->cpum.GstCtx.r13);
213	ADD_REG64(WHvX64RegisterR14, pVCpu->cpum.GstCtx.r14);
214	ADD_REG64(WHvX64RegisterR15, pVCpu->cpum.GstCtx.r15);
215	}
216	}
217
218	/* RIP & Flags */
219	if (fWhat & CPUMCTX_EXTRN_RIP)
220	ADD_REG64(WHvX64RegisterRip, pVCpu->cpum.GstCtx.rip);
221	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
222	ADD_REG64(WHvX64RegisterRflags, pVCpu->cpum.GstCtx.rflags.u);
223
224	/* Segments */
225	# define ADD_SEG(a_enmName, a_SReg) \
226	do { \
227	aenmNames[iReg] = a_enmName; \
228	aValues[iReg].Segment.Base = (a_SReg).u64Base; \
229	aValues[iReg].Segment.Limit = (a_SReg).u32Limit; \
230	aValues[iReg].Segment.Selector = (a_SReg).Sel; \
231	aValues[iReg].Segment.Attributes = (a_SReg).Attr.u; \
232	iReg++; \
233	} while (0)
234	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
235	{
236	if (fWhat & CPUMCTX_EXTRN_ES)
237	ADD_SEG(WHvX64RegisterEs, pVCpu->cpum.GstCtx.es);
238	if (fWhat & CPUMCTX_EXTRN_CS)
239	ADD_SEG(WHvX64RegisterCs, pVCpu->cpum.GstCtx.cs);
240	if (fWhat & CPUMCTX_EXTRN_SS)
241	ADD_SEG(WHvX64RegisterSs, pVCpu->cpum.GstCtx.ss);
242	if (fWhat & CPUMCTX_EXTRN_DS)
243	ADD_SEG(WHvX64RegisterDs, pVCpu->cpum.GstCtx.ds);
244	if (fWhat & CPUMCTX_EXTRN_FS)
245	ADD_SEG(WHvX64RegisterFs, pVCpu->cpum.GstCtx.fs);
246	if (fWhat & CPUMCTX_EXTRN_GS)
247	ADD_SEG(WHvX64RegisterGs, pVCpu->cpum.GstCtx.gs);
248	}
249
250	/* Descriptor tables & task segment. */
251	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
252	{
253	if (fWhat & CPUMCTX_EXTRN_LDTR)
254	ADD_SEG(WHvX64RegisterLdtr, pVCpu->cpum.GstCtx.ldtr);
255	if (fWhat & CPUMCTX_EXTRN_TR)
256	ADD_SEG(WHvX64RegisterTr, pVCpu->cpum.GstCtx.tr);
257	if (fWhat & CPUMCTX_EXTRN_IDTR)
258	{
259	aenmNames[iReg] = WHvX64RegisterIdtr;
260	aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.idtr.cbIdt;
261	aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.idtr.pIdt;
262	iReg++;
263	}
264	if (fWhat & CPUMCTX_EXTRN_GDTR)
265	{
266	aenmNames[iReg] = WHvX64RegisterGdtr;
267	aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt;
268	aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.gdtr.pGdt;
269	iReg++;
270	}
271	}
272
273	/* Control registers. */
274	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
275	{
276	if (fWhat & CPUMCTX_EXTRN_CR0)
277	ADD_REG64(WHvX64RegisterCr0, pVCpu->cpum.GstCtx.cr0);
278	if (fWhat & CPUMCTX_EXTRN_CR2)
279	ADD_REG64(WHvX64RegisterCr2, pVCpu->cpum.GstCtx.cr2);
280	if (fWhat & CPUMCTX_EXTRN_CR3)
281	ADD_REG64(WHvX64RegisterCr3, pVCpu->cpum.GstCtx.cr3);
282	if (fWhat & CPUMCTX_EXTRN_CR4)
283	ADD_REG64(WHvX64RegisterCr4, pVCpu->cpum.GstCtx.cr4);
284	}
285	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
286	ADD_REG64(WHvX64RegisterCr8, CPUMGetGuestCR8(pVCpu));
287
288	/* Debug registers. */
289	/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
290	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
291	{
292	ADD_REG64(WHvX64RegisterDr0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
293	ADD_REG64(WHvX64RegisterDr1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
294	ADD_REG64(WHvX64RegisterDr2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
295	ADD_REG64(WHvX64RegisterDr3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
296	}
297	if (fWhat & CPUMCTX_EXTRN_DR6)
298	ADD_REG64(WHvX64RegisterDr6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
299	if (fWhat & CPUMCTX_EXTRN_DR7)
300	ADD_REG64(WHvX64RegisterDr7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
301
302	/* Floating point state. */
303	if (fWhat & CPUMCTX_EXTRN_X87)
304	{
305	ADD_REG128(WHvX64RegisterFpMmx0, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[1]);
306	ADD_REG128(WHvX64RegisterFpMmx1, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[1]);
307	ADD_REG128(WHvX64RegisterFpMmx2, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[1]);
308	ADD_REG128(WHvX64RegisterFpMmx3, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[1]);
309	ADD_REG128(WHvX64RegisterFpMmx4, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[1]);
310	ADD_REG128(WHvX64RegisterFpMmx5, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[1]);
311	ADD_REG128(WHvX64RegisterFpMmx6, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[1]);
312	ADD_REG128(WHvX64RegisterFpMmx7, pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[1]);
313
314	aenmNames[iReg] = WHvX64RegisterFpControlStatus;
315	aValues[iReg].FpControlStatus.FpControl = pVCpu->cpum.GstCtx.pXStateR3->x87.FCW;
316	aValues[iReg].FpControlStatus.FpStatus = pVCpu->cpum.GstCtx.pXStateR3->x87.FSW;
317	aValues[iReg].FpControlStatus.FpTag = pVCpu->cpum.GstCtx.pXStateR3->x87.FTW;
318	aValues[iReg].FpControlStatus.Reserved = pVCpu->cpum.GstCtx.pXStateR3->x87.FTW >> 8;
319	aValues[iReg].FpControlStatus.LastFpOp = pVCpu->cpum.GstCtx.pXStateR3->x87.FOP;
320	aValues[iReg].FpControlStatus.LastFpRip = (pVCpu->cpum.GstCtx.pXStateR3->x87.FPUIP)
321	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.CS << 32)
322	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd1 << 48);
323	iReg++;
324
325	aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
326	aValues[iReg].XmmControlStatus.LastFpRdp = (pVCpu->cpum.GstCtx.pXStateR3->x87.FPUDP)
327	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.DS << 32)
328	\| ((uint64_t)pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd2 << 48);
329	aValues[iReg].XmmControlStatus.XmmStatusControl = pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR;
330	aValues[iReg].XmmControlStatus.XmmStatusControlMask = pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
331	iReg++;
332	}
333
334	/* Vector state. */
335	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
336	{
337	ADD_REG128(WHvX64RegisterXmm0, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Hi);
338	ADD_REG128(WHvX64RegisterXmm1, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Hi);
339	ADD_REG128(WHvX64RegisterXmm2, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Hi);
340	ADD_REG128(WHvX64RegisterXmm3, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Hi);
341	ADD_REG128(WHvX64RegisterXmm4, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Hi);
342	ADD_REG128(WHvX64RegisterXmm5, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Hi);
343	ADD_REG128(WHvX64RegisterXmm6, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Hi);
344	ADD_REG128(WHvX64RegisterXmm7, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Hi);
345	ADD_REG128(WHvX64RegisterXmm8, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Hi);
346	ADD_REG128(WHvX64RegisterXmm9, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Hi);
347	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Hi);
348	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Hi);
349	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Hi);
350	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Hi);
351	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Hi);
352	ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Hi);
353	}
354
355	/* MSRs */
356	// WHvX64RegisterTsc - don't touch
357	if (fWhat & CPUMCTX_EXTRN_EFER)
358	ADD_REG64(WHvX64RegisterEfer, pVCpu->cpum.GstCtx.msrEFER);
359	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
360	ADD_REG64(WHvX64RegisterKernelGsBase, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
361	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
362	{
363	ADD_REG64(WHvX64RegisterSysenterCs, pVCpu->cpum.GstCtx.SysEnter.cs);
364	ADD_REG64(WHvX64RegisterSysenterEip, pVCpu->cpum.GstCtx.SysEnter.eip);
365	ADD_REG64(WHvX64RegisterSysenterEsp, pVCpu->cpum.GstCtx.SysEnter.esp);
366	}
367	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
368	{
369	ADD_REG64(WHvX64RegisterStar, pVCpu->cpum.GstCtx.msrSTAR);
370	ADD_REG64(WHvX64RegisterLstar, pVCpu->cpum.GstCtx.msrLSTAR);
371	ADD_REG64(WHvX64RegisterCstar, pVCpu->cpum.GstCtx.msrCSTAR);
372	ADD_REG64(WHvX64RegisterSfmask, pVCpu->cpum.GstCtx.msrSFMASK);
373	}
374	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
375	{
376	ADD_REG64(WHvX64RegisterApicBase, APICGetBaseMsrNoCheck(pVCpu));
377	ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
378	#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
379	ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
380	#endif
381	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
382	ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
383	ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
384	ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
385	ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
386	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000);
387	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000);
388	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000);
389	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000);
390	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000);
391	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000);
392	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000);
393	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000);
394	ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
395	#if 0 /** @todo these registers aren't available? Might explain something.. .*/
396	const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
397	if (enmCpuVendor != CPUMCPUVENDOR_AMD)
398	{
399	ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
400	ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
401	}
402	#endif
403	}
404
405	/* event injection (clear it). */
406	if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)
407	ADD_REG64(WHvRegisterPendingInterruption, 0);
408
409	/* Interruptibility state. This can get a little complicated since we get
410	half of the state via HV_X64_VP_EXECUTION_STATE. */
411	if ( (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
412	== (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI) )
413	{
414	ADD_REG64(WHvRegisterInterruptState, 0);
415	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
416	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
417	aValues[iReg - 1].InterruptState.InterruptShadow = 1;
418	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
419	aValues[iReg - 1].InterruptState.NmiMasked = 1;
420	}
421	else if (fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT)
422	{
423	if ( pVCpu->nem.s.fLastInterruptShadow
424	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
425	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip))
426	{
427	ADD_REG64(WHvRegisterInterruptState, 0);
428	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
429	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip)
430	aValues[iReg - 1].InterruptState.InterruptShadow = 1;
431	/** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */
432	//if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
433	// aValues[iReg - 1].InterruptState.NmiMasked = 1;
434	}
435	}
436	else
437	Assert(!(fWhat & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI));
438
439	/* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */
440	uint8_t const fDesiredIntWin = pVCpu->nem.s.fDesiredInterruptWindows;
441	if ( fDesiredIntWin
442	\|\| pVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin)
443	{
444	pVCpu->nem.s.fCurrentInterruptWindows = pVCpu->nem.s.fDesiredInterruptWindows;
445	ADD_REG64(WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin);
446	Assert(aValues[iReg - 1].DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI));
447	Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR));
448	Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptPriority == (unsigned)((fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT));
449	}
450
451	/// @todo WHvRegisterPendingEvent
452
453	/*
454	* Set the registers.
455	*/
456	Assert(iReg < RT_ELEMENTS(aValues));
457	Assert(iReg < RT_ELEMENTS(aenmNames));
458	# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
459	Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
460	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
461	# endif
462	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
463	if (SUCCEEDED(hrc))
464	{
465	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK \| CPUMCTX_EXTRN_KEEPER_NEM;
466	return VINF_SUCCESS;
467	}
468	AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
469	pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
470	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
471	return VERR_INTERNAL_ERROR;
472
473	# undef ADD_REG64
474	# undef ADD_REG128
475	# undef ADD_SEG
476
477	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
478	}
479
480
481	NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
482	{
483	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
484	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
485	if (pVM->nem.s.fUseRing0Runloop)
486	# endif
487	{
488	/* See NEMR0ImportState */
489	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_IMPORT_STATE, fWhat, NULL);
490	if (RT_SUCCESS(rc))
491	return rc;
492	if (rc == VERR_NEM_FLUSH_TLB)
493	return PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /fGlobal/);
494	AssertLogRelRCReturn(rc, rc);
495	return rc;
496	}
497	# endif
498	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
499	WHV_REGISTER_NAME aenmNames[128];
500
501	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
502	uintptr_t iReg = 0;
503
504	/* GPRs */
505	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
506	{
507	if (fWhat & CPUMCTX_EXTRN_RAX)
508	aenmNames[iReg++] = WHvX64RegisterRax;
509	if (fWhat & CPUMCTX_EXTRN_RCX)
510	aenmNames[iReg++] = WHvX64RegisterRcx;
511	if (fWhat & CPUMCTX_EXTRN_RDX)
512	aenmNames[iReg++] = WHvX64RegisterRdx;
513	if (fWhat & CPUMCTX_EXTRN_RBX)
514	aenmNames[iReg++] = WHvX64RegisterRbx;
515	if (fWhat & CPUMCTX_EXTRN_RSP)
516	aenmNames[iReg++] = WHvX64RegisterRsp;
517	if (fWhat & CPUMCTX_EXTRN_RBP)
518	aenmNames[iReg++] = WHvX64RegisterRbp;
519	if (fWhat & CPUMCTX_EXTRN_RSI)
520	aenmNames[iReg++] = WHvX64RegisterRsi;
521	if (fWhat & CPUMCTX_EXTRN_RDI)
522	aenmNames[iReg++] = WHvX64RegisterRdi;
523	if (fWhat & CPUMCTX_EXTRN_R8_R15)
524	{
525	aenmNames[iReg++] = WHvX64RegisterR8;
526	aenmNames[iReg++] = WHvX64RegisterR9;
527	aenmNames[iReg++] = WHvX64RegisterR10;
528	aenmNames[iReg++] = WHvX64RegisterR11;
529	aenmNames[iReg++] = WHvX64RegisterR12;
530	aenmNames[iReg++] = WHvX64RegisterR13;
531	aenmNames[iReg++] = WHvX64RegisterR14;
532	aenmNames[iReg++] = WHvX64RegisterR15;
533	}
534	}
535
536	/* RIP & Flags */
537	if (fWhat & CPUMCTX_EXTRN_RIP)
538	aenmNames[iReg++] = WHvX64RegisterRip;
539	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
540	aenmNames[iReg++] = WHvX64RegisterRflags;
541
542	/* Segments */
543	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
544	{
545	if (fWhat & CPUMCTX_EXTRN_ES)
546	aenmNames[iReg++] = WHvX64RegisterEs;
547	if (fWhat & CPUMCTX_EXTRN_CS)
548	aenmNames[iReg++] = WHvX64RegisterCs;
549	if (fWhat & CPUMCTX_EXTRN_SS)
550	aenmNames[iReg++] = WHvX64RegisterSs;
551	if (fWhat & CPUMCTX_EXTRN_DS)
552	aenmNames[iReg++] = WHvX64RegisterDs;
553	if (fWhat & CPUMCTX_EXTRN_FS)
554	aenmNames[iReg++] = WHvX64RegisterFs;
555	if (fWhat & CPUMCTX_EXTRN_GS)
556	aenmNames[iReg++] = WHvX64RegisterGs;
557	}
558
559	/* Descriptor tables. */
560	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
561	{
562	if (fWhat & CPUMCTX_EXTRN_LDTR)
563	aenmNames[iReg++] = WHvX64RegisterLdtr;
564	if (fWhat & CPUMCTX_EXTRN_TR)
565	aenmNames[iReg++] = WHvX64RegisterTr;
566	if (fWhat & CPUMCTX_EXTRN_IDTR)
567	aenmNames[iReg++] = WHvX64RegisterIdtr;
568	if (fWhat & CPUMCTX_EXTRN_GDTR)
569	aenmNames[iReg++] = WHvX64RegisterGdtr;
570	}
571
572	/* Control registers. */
573	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
574	{
575	if (fWhat & CPUMCTX_EXTRN_CR0)
576	aenmNames[iReg++] = WHvX64RegisterCr0;
577	if (fWhat & CPUMCTX_EXTRN_CR2)
578	aenmNames[iReg++] = WHvX64RegisterCr2;
579	if (fWhat & CPUMCTX_EXTRN_CR3)
580	aenmNames[iReg++] = WHvX64RegisterCr3;
581	if (fWhat & CPUMCTX_EXTRN_CR4)
582	aenmNames[iReg++] = WHvX64RegisterCr4;
583	}
584	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
585	aenmNames[iReg++] = WHvX64RegisterCr8;
586
587	/* Debug registers. */
588	if (fWhat & CPUMCTX_EXTRN_DR7)
589	aenmNames[iReg++] = WHvX64RegisterDr7;
590	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
591	{
592	if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_DR7))
593	{
594	fWhat \|= CPUMCTX_EXTRN_DR7;
595	aenmNames[iReg++] = WHvX64RegisterDr7;
596	}
597	aenmNames[iReg++] = WHvX64RegisterDr0;
598	aenmNames[iReg++] = WHvX64RegisterDr1;
599	aenmNames[iReg++] = WHvX64RegisterDr2;
600	aenmNames[iReg++] = WHvX64RegisterDr3;
601	}
602	if (fWhat & CPUMCTX_EXTRN_DR6)
603	aenmNames[iReg++] = WHvX64RegisterDr6;
604
605	/* Floating point state. */
606	if (fWhat & CPUMCTX_EXTRN_X87)
607	{
608	aenmNames[iReg++] = WHvX64RegisterFpMmx0;
609	aenmNames[iReg++] = WHvX64RegisterFpMmx1;
610	aenmNames[iReg++] = WHvX64RegisterFpMmx2;
611	aenmNames[iReg++] = WHvX64RegisterFpMmx3;
612	aenmNames[iReg++] = WHvX64RegisterFpMmx4;
613	aenmNames[iReg++] = WHvX64RegisterFpMmx5;
614	aenmNames[iReg++] = WHvX64RegisterFpMmx6;
615	aenmNames[iReg++] = WHvX64RegisterFpMmx7;
616	aenmNames[iReg++] = WHvX64RegisterFpControlStatus;
617	}
618	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
619	aenmNames[iReg++] = WHvX64RegisterXmmControlStatus;
620
621	/* Vector state. */
622	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
623	{
624	aenmNames[iReg++] = WHvX64RegisterXmm0;
625	aenmNames[iReg++] = WHvX64RegisterXmm1;
626	aenmNames[iReg++] = WHvX64RegisterXmm2;
627	aenmNames[iReg++] = WHvX64RegisterXmm3;
628	aenmNames[iReg++] = WHvX64RegisterXmm4;
629	aenmNames[iReg++] = WHvX64RegisterXmm5;
630	aenmNames[iReg++] = WHvX64RegisterXmm6;
631	aenmNames[iReg++] = WHvX64RegisterXmm7;
632	aenmNames[iReg++] = WHvX64RegisterXmm8;
633	aenmNames[iReg++] = WHvX64RegisterXmm9;
634	aenmNames[iReg++] = WHvX64RegisterXmm10;
635	aenmNames[iReg++] = WHvX64RegisterXmm11;
636	aenmNames[iReg++] = WHvX64RegisterXmm12;
637	aenmNames[iReg++] = WHvX64RegisterXmm13;
638	aenmNames[iReg++] = WHvX64RegisterXmm14;
639	aenmNames[iReg++] = WHvX64RegisterXmm15;
640	}
641
642	/* MSRs */
643	// WHvX64RegisterTsc - don't touch
644	if (fWhat & CPUMCTX_EXTRN_EFER)
645	aenmNames[iReg++] = WHvX64RegisterEfer;
646	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
647	aenmNames[iReg++] = WHvX64RegisterKernelGsBase;
648	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
649	{
650	aenmNames[iReg++] = WHvX64RegisterSysenterCs;
651	aenmNames[iReg++] = WHvX64RegisterSysenterEip;
652	aenmNames[iReg++] = WHvX64RegisterSysenterEsp;
653	}
654	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
655	{
656	aenmNames[iReg++] = WHvX64RegisterStar;
657	aenmNames[iReg++] = WHvX64RegisterLstar;
658	aenmNames[iReg++] = WHvX64RegisterCstar;
659	aenmNames[iReg++] = WHvX64RegisterSfmask;
660	}
661
662	//#ifdef LOG_ENABLED
663	// const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
664	//#endif
665	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
666	{
667	aenmNames[iReg++] = WHvX64RegisterApicBase; /// @todo APIC BASE
668	aenmNames[iReg++] = WHvX64RegisterPat;
669	#if 0 /def LOG_ENABLED/ /** @todo Check if WHvX64RegisterMsrMtrrCap works... */
670	aenmNames[iReg++] = WHvX64RegisterMsrMtrrCap;
671	#endif
672	aenmNames[iReg++] = WHvX64RegisterMsrMtrrDefType;
673	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix64k00000;
674	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16k80000;
675	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16kA0000;
676	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC0000;
677	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC8000;
678	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD0000;
679	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD8000;
680	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE0000;
681	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE8000;
682	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF0000;
683	aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF8000;
684	aenmNames[iReg++] = WHvX64RegisterTscAux;
685	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
686	//#ifdef LOG_ENABLED
687	// if (enmCpuVendor != CPUMCPUVENDOR_AMD)
688	// aenmNames[iReg++] = HvX64RegisterIa32FeatureControl;
689	//#endif
690	}
691
692	/* Interruptibility. */
693	if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
694	{
695	aenmNames[iReg++] = WHvRegisterInterruptState;
696	aenmNames[iReg++] = WHvX64RegisterRip;
697	}
698
699	/* event injection */
700	aenmNames[iReg++] = WHvRegisterPendingInterruption;
701	aenmNames[iReg++] = WHvRegisterPendingEvent0; /** @todo renamed to WHvRegisterPendingEvent */
702
703	size_t const cRegs = iReg;
704	Assert(cRegs < RT_ELEMENTS(aenmNames));
705
706	/*
707	* Get the registers.
708	*/
709	WHV_REGISTER_VALUE aValues[128];
710	RT_ZERO(aValues);
711	Assert(RT_ELEMENTS(aValues) >= cRegs);
712	Assert(RT_ELEMENTS(aenmNames) >= cRegs);
713	# ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
714	Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
715	pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
716	# endif
717	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues);
718	AssertLogRelMsgReturn(SUCCEEDED(hrc),
719	("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
720	pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
721	, VERR_NEM_GET_REGISTERS_FAILED);
722
723	iReg = 0;
724	# define GET_REG64(a_DstVar, a_enmName) do { \
725	Assert(aenmNames[iReg] == (a_enmName)); \
726	(a_DstVar) = aValues[iReg].Reg64; \
727	iReg++; \
728	} while (0)
729	# define GET_REG64_LOG7(a_DstVar, a_enmName, a_szLogName) do { \
730	Assert(aenmNames[iReg] == (a_enmName)); \
731	if ((a_DstVar) != aValues[iReg].Reg64) \
732	Log7(("NEM/%u: " a_szLogName " changed %RX64 -> %RX64\n", pVCpu->idCpu, (a_DstVar), aValues[iReg].Reg64)); \
733	(a_DstVar) = aValues[iReg].Reg64; \
734	iReg++; \
735	} while (0)
736	# define GET_REG128(a_DstVarLo, a_DstVarHi, a_enmName) do { \
737	Assert(aenmNames[iReg] == a_enmName); \
738	(a_DstVarLo) = aValues[iReg].Reg128.Low64; \
739	(a_DstVarHi) = aValues[iReg].Reg128.High64; \
740	iReg++; \
741	} while (0)
742	# define GET_SEG(a_SReg, a_enmName) do { \
743	Assert(aenmNames[iReg] == (a_enmName)); \
744	NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[iReg].Segment); \
745	iReg++; \
746	} while (0)
747
748	/* GPRs */
749	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
750	{
751	if (fWhat & CPUMCTX_EXTRN_RAX)
752	GET_REG64(pVCpu->cpum.GstCtx.rax, WHvX64RegisterRax);
753	if (fWhat & CPUMCTX_EXTRN_RCX)
754	GET_REG64(pVCpu->cpum.GstCtx.rcx, WHvX64RegisterRcx);
755	if (fWhat & CPUMCTX_EXTRN_RDX)
756	GET_REG64(pVCpu->cpum.GstCtx.rdx, WHvX64RegisterRdx);
757	if (fWhat & CPUMCTX_EXTRN_RBX)
758	GET_REG64(pVCpu->cpum.GstCtx.rbx, WHvX64RegisterRbx);
759	if (fWhat & CPUMCTX_EXTRN_RSP)
760	GET_REG64(pVCpu->cpum.GstCtx.rsp, WHvX64RegisterRsp);
761	if (fWhat & CPUMCTX_EXTRN_RBP)
762	GET_REG64(pVCpu->cpum.GstCtx.rbp, WHvX64RegisterRbp);
763	if (fWhat & CPUMCTX_EXTRN_RSI)
764	GET_REG64(pVCpu->cpum.GstCtx.rsi, WHvX64RegisterRsi);
765	if (fWhat & CPUMCTX_EXTRN_RDI)
766	GET_REG64(pVCpu->cpum.GstCtx.rdi, WHvX64RegisterRdi);
767	if (fWhat & CPUMCTX_EXTRN_R8_R15)
768	{
769	GET_REG64(pVCpu->cpum.GstCtx.r8, WHvX64RegisterR8);
770	GET_REG64(pVCpu->cpum.GstCtx.r9, WHvX64RegisterR9);
771	GET_REG64(pVCpu->cpum.GstCtx.r10, WHvX64RegisterR10);
772	GET_REG64(pVCpu->cpum.GstCtx.r11, WHvX64RegisterR11);
773	GET_REG64(pVCpu->cpum.GstCtx.r12, WHvX64RegisterR12);
774	GET_REG64(pVCpu->cpum.GstCtx.r13, WHvX64RegisterR13);
775	GET_REG64(pVCpu->cpum.GstCtx.r14, WHvX64RegisterR14);
776	GET_REG64(pVCpu->cpum.GstCtx.r15, WHvX64RegisterR15);
777	}
778	}
779
780	/* RIP & Flags */
781	if (fWhat & CPUMCTX_EXTRN_RIP)
782	GET_REG64(pVCpu->cpum.GstCtx.rip, WHvX64RegisterRip);
783	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
784	GET_REG64(pVCpu->cpum.GstCtx.rflags.u, WHvX64RegisterRflags);
785
786	/* Segments */
787	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
788	{
789	if (fWhat & CPUMCTX_EXTRN_ES)
790	GET_SEG(pVCpu->cpum.GstCtx.es, WHvX64RegisterEs);
791	if (fWhat & CPUMCTX_EXTRN_CS)
792	GET_SEG(pVCpu->cpum.GstCtx.cs, WHvX64RegisterCs);
793	if (fWhat & CPUMCTX_EXTRN_SS)
794	GET_SEG(pVCpu->cpum.GstCtx.ss, WHvX64RegisterSs);
795	if (fWhat & CPUMCTX_EXTRN_DS)
796	GET_SEG(pVCpu->cpum.GstCtx.ds, WHvX64RegisterDs);
797	if (fWhat & CPUMCTX_EXTRN_FS)
798	GET_SEG(pVCpu->cpum.GstCtx.fs, WHvX64RegisterFs);
799	if (fWhat & CPUMCTX_EXTRN_GS)
800	GET_SEG(pVCpu->cpum.GstCtx.gs, WHvX64RegisterGs);
801	}
802
803	/* Descriptor tables and the task segment. */
804	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
805	{
806	if (fWhat & CPUMCTX_EXTRN_LDTR)
807	GET_SEG(pVCpu->cpum.GstCtx.ldtr, WHvX64RegisterLdtr);
808
809	if (fWhat & CPUMCTX_EXTRN_TR)
810	{
811	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
812	avoid to trigger sanity assertions around the code, always fix this. */
813	GET_SEG(pVCpu->cpum.GstCtx.tr, WHvX64RegisterTr);
814	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
815	{
816	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
817	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
818	break;
819	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
820	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
821	break;
822	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
823	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
824	break;
825	}
826	}
827	if (fWhat & CPUMCTX_EXTRN_IDTR)
828	{
829	Assert(aenmNames[iReg] == WHvX64RegisterIdtr);
830	pVCpu->cpum.GstCtx.idtr.cbIdt = aValues[iReg].Table.Limit;
831	pVCpu->cpum.GstCtx.idtr.pIdt = aValues[iReg].Table.Base;
832	iReg++;
833	}
834	if (fWhat & CPUMCTX_EXTRN_GDTR)
835	{
836	Assert(aenmNames[iReg] == WHvX64RegisterGdtr);
837	pVCpu->cpum.GstCtx.gdtr.cbGdt = aValues[iReg].Table.Limit;
838	pVCpu->cpum.GstCtx.gdtr.pGdt = aValues[iReg].Table.Base;
839	iReg++;
840	}
841	}
842
843	/* Control registers. */
844	bool fMaybeChangedMode = false;
845	bool fUpdateCr3 = false;
846	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
847	{
848	if (fWhat & CPUMCTX_EXTRN_CR0)
849	{
850	Assert(aenmNames[iReg] == WHvX64RegisterCr0);
851	if (pVCpu->cpum.GstCtx.cr0 != aValues[iReg].Reg64)
852	{
853	CPUMSetGuestCR0(pVCpu, aValues[iReg].Reg64);
854	fMaybeChangedMode = true;
855	}
856	iReg++;
857	}
858	if (fWhat & CPUMCTX_EXTRN_CR2)
859	GET_REG64(pVCpu->cpum.GstCtx.cr2, WHvX64RegisterCr2);
860	if (fWhat & CPUMCTX_EXTRN_CR3)
861	{
862	if (pVCpu->cpum.GstCtx.cr3 != aValues[iReg].Reg64)
863	{
864	CPUMSetGuestCR3(pVCpu, aValues[iReg].Reg64);
865	fUpdateCr3 = true;
866	}
867	iReg++;
868	}
869	if (fWhat & CPUMCTX_EXTRN_CR4)
870	{
871	if (pVCpu->cpum.GstCtx.cr4 != aValues[iReg].Reg64)
872	{
873	CPUMSetGuestCR4(pVCpu, aValues[iReg].Reg64);
874	fMaybeChangedMode = true;
875	}
876	iReg++;
877	}
878	}
879	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
880	{
881	Assert(aenmNames[iReg] == WHvX64RegisterCr8);
882	APICSetTpr(pVCpu, (uint8_t)aValues[iReg].Reg64 << 4);
883	iReg++;
884	}
885
886	/* Debug registers. */
887	if (fWhat & CPUMCTX_EXTRN_DR7)
888	{
889	Assert(aenmNames[iReg] == WHvX64RegisterDr7);
890	if (pVCpu->cpum.GstCtx.dr[7] != aValues[iReg].Reg64)
891	CPUMSetGuestDR7(pVCpu, aValues[iReg].Reg64);
892	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
893	iReg++;
894	}
895	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
896	{
897	Assert(aenmNames[iReg] == WHvX64RegisterDr0);
898	Assert(aenmNames[iReg+3] == WHvX64RegisterDr3);
899	if (pVCpu->cpum.GstCtx.dr[0] != aValues[iReg].Reg64)
900	CPUMSetGuestDR0(pVCpu, aValues[iReg].Reg64);
901	iReg++;
902	if (pVCpu->cpum.GstCtx.dr[1] != aValues[iReg].Reg64)
903	CPUMSetGuestDR1(pVCpu, aValues[iReg].Reg64);
904	iReg++;
905	if (pVCpu->cpum.GstCtx.dr[2] != aValues[iReg].Reg64)
906	CPUMSetGuestDR2(pVCpu, aValues[iReg].Reg64);
907	iReg++;
908	if (pVCpu->cpum.GstCtx.dr[3] != aValues[iReg].Reg64)
909	CPUMSetGuestDR3(pVCpu, aValues[iReg].Reg64);
910	iReg++;
911	}
912	if (fWhat & CPUMCTX_EXTRN_DR6)
913	{
914	Assert(aenmNames[iReg] == WHvX64RegisterDr6);
915	if (pVCpu->cpum.GstCtx.dr[6] != aValues[iReg].Reg64)
916	CPUMSetGuestDR6(pVCpu, aValues[iReg].Reg64);
917	iReg++;
918	}
919
920	/* Floating point state. */
921	if (fWhat & CPUMCTX_EXTRN_X87)
922	{
923	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[0].au64[1], WHvX64RegisterFpMmx0);
924	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[1].au64[1], WHvX64RegisterFpMmx1);
925	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[2].au64[1], WHvX64RegisterFpMmx2);
926	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[3].au64[1], WHvX64RegisterFpMmx3);
927	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[4].au64[1], WHvX64RegisterFpMmx4);
928	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[5].au64[1], WHvX64RegisterFpMmx5);
929	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[6].au64[1], WHvX64RegisterFpMmx6);
930	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.pXStateR3->x87.aRegs[7].au64[1], WHvX64RegisterFpMmx7);
931
932	Assert(aenmNames[iReg] == WHvX64RegisterFpControlStatus);
933	pVCpu->cpum.GstCtx.pXStateR3->x87.FCW = aValues[iReg].FpControlStatus.FpControl;
934	pVCpu->cpum.GstCtx.pXStateR3->x87.FSW = aValues[iReg].FpControlStatus.FpStatus;
935	pVCpu->cpum.GstCtx.pXStateR3->x87.FTW = aValues[iReg].FpControlStatus.FpTag
936	/\| (aValues[iReg].FpControlStatus.Reserved << 8)/;
937	pVCpu->cpum.GstCtx.pXStateR3->x87.FOP = aValues[iReg].FpControlStatus.LastFpOp;
938	pVCpu->cpum.GstCtx.pXStateR3->x87.FPUIP = (uint32_t)aValues[iReg].FpControlStatus.LastFpRip;
939	pVCpu->cpum.GstCtx.pXStateR3->x87.CS = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 32);
940	pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd1 = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 48);
941	iReg++;
942	}
943
944	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
945	{
946	Assert(aenmNames[iReg] == WHvX64RegisterXmmControlStatus);
947	if (fWhat & CPUMCTX_EXTRN_X87)
948	{
949	pVCpu->cpum.GstCtx.pXStateR3->x87.FPUDP = (uint32_t)aValues[iReg].XmmControlStatus.LastFpRdp;
950	pVCpu->cpum.GstCtx.pXStateR3->x87.DS = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 32);
951	pVCpu->cpum.GstCtx.pXStateR3->x87.Rsrvd2 = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 48);
952	}
953	pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR = aValues[iReg].XmmControlStatus.XmmStatusControl;
954	pVCpu->cpum.GstCtx.pXStateR3->x87.MXCSR_MASK = aValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
955	iReg++;
956	}
957
958	/* Vector state. */
959	if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
960	{
961	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 0].uXmm.s.Hi, WHvX64RegisterXmm0);
962	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 1].uXmm.s.Hi, WHvX64RegisterXmm1);
963	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 2].uXmm.s.Hi, WHvX64RegisterXmm2);
964	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 3].uXmm.s.Hi, WHvX64RegisterXmm3);
965	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 4].uXmm.s.Hi, WHvX64RegisterXmm4);
966	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 5].uXmm.s.Hi, WHvX64RegisterXmm5);
967	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 6].uXmm.s.Hi, WHvX64RegisterXmm6);
968	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 7].uXmm.s.Hi, WHvX64RegisterXmm7);
969	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 8].uXmm.s.Hi, WHvX64RegisterXmm8);
970	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[ 9].uXmm.s.Hi, WHvX64RegisterXmm9);
971	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[10].uXmm.s.Hi, WHvX64RegisterXmm10);
972	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[11].uXmm.s.Hi, WHvX64RegisterXmm11);
973	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[12].uXmm.s.Hi, WHvX64RegisterXmm12);
974	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[13].uXmm.s.Hi, WHvX64RegisterXmm13);
975	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[14].uXmm.s.Hi, WHvX64RegisterXmm14);
976	GET_REG128(pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.pXStateR3->x87.aXMM[15].uXmm.s.Hi, WHvX64RegisterXmm15);
977	}
978
979	/* MSRs */
980	// WHvX64RegisterTsc - don't touch
981	if (fWhat & CPUMCTX_EXTRN_EFER)
982	{
983	Assert(aenmNames[iReg] == WHvX64RegisterEfer);
984	if (aValues[iReg].Reg64 != pVCpu->cpum.GstCtx.msrEFER)
985	{
986	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, aValues[iReg].Reg64));
987	if ((aValues[iReg].Reg64 ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
988	PGMNotifyNxeChanged(pVCpu, RT_BOOL(aValues[iReg].Reg64 & MSR_K6_EFER_NXE));
989	pVCpu->cpum.GstCtx.msrEFER = aValues[iReg].Reg64;
990	fMaybeChangedMode = true;
991	}
992	iReg++;
993	}
994	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
995	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrKERNELGSBASE, WHvX64RegisterKernelGsBase, "MSR KERNEL_GS_BASE");
996	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
997	{
998	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.cs, WHvX64RegisterSysenterCs, "MSR SYSENTER.CS");
999	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.eip, WHvX64RegisterSysenterEip, "MSR SYSENTER.EIP");
1000	GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.esp, WHvX64RegisterSysenterEsp, "MSR SYSENTER.ESP");
1001	}
1002	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1003	{
1004	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSTAR, WHvX64RegisterStar, "MSR STAR");
1005	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrLSTAR, WHvX64RegisterLstar, "MSR LSTAR");
1006	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrCSTAR, WHvX64RegisterCstar, "MSR CSTAR");
1007	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSFMASK, WHvX64RegisterSfmask, "MSR SFMASK");
1008	}
1009	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1010	{
1011	Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
1012	const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
1013	if (aValues[iReg].Reg64 != uOldBase)
1014	{
1015	Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
1016	pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
1017	int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
1018	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
1019	}
1020	iReg++;
1021
1022	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
1023	#if 0 /def LOG_ENABLED/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
1024	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
1025	#endif
1026	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1027	GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE");
1028	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
1029	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
1030	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
1031	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000");
1032	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000");
1033	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000");
1034	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000");
1035	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000");
1036	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000");
1037	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000");
1038	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000");
1039	GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
1040	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
1041	}
1042
1043	/* Interruptibility. */
1044	if (fWhat & (CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
1045	{
1046	Assert(aenmNames[iReg] == WHvRegisterInterruptState);
1047	Assert(aenmNames[iReg + 1] == WHvX64RegisterRip);
1048
1049	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
1050	{
1051	pVCpu->nem.s.fLastInterruptShadow = aValues[iReg].InterruptState.InterruptShadow;
1052	if (aValues[iReg].InterruptState.InterruptShadow)
1053	EMSetInhibitInterruptsPC(pVCpu, aValues[iReg + 1].Reg64);
1054	else
1055	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1056	}
1057
1058	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI))
1059	{
1060	if (aValues[iReg].InterruptState.NmiMasked)
1061	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
1062	else
1063	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
1064	}
1065
1066	fWhat \|= CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
1067	iReg += 2;
1068	}
1069
1070	/* Event injection. */
1071	/// @todo WHvRegisterPendingInterruption
1072	Assert(aenmNames[iReg] == WHvRegisterPendingInterruption);
1073	if (aValues[iReg].PendingInterruption.InterruptionPending)
1074	{
1075	Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
1076	aValues[iReg].PendingInterruption.InterruptionType, aValues[iReg].PendingInterruption.InterruptionVector,
1077	aValues[iReg].PendingInterruption.DeliverErrorCode, aValues[iReg].PendingInterruption.ErrorCode,
1078	aValues[iReg].PendingInterruption.InstructionLength, aValues[iReg].PendingInterruption.NestedEvent));
1079	AssertMsg((aValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0,
1080	("%#RX64\n", aValues[iReg].PendingInterruption.AsUINT64));
1081	}
1082
1083	/// @todo WHvRegisterPendingEvent0 (renamed to WHvRegisterPendingEvent).
1084
1085	/* Almost done, just update extrn flags and maybe change PGM mode. */
1086	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1087	if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
1088	pVCpu->cpum.GstCtx.fExtrn = 0;
1089
1090	/* Typical. */
1091	if (!fMaybeChangedMode && !fUpdateCr3)
1092	return VINF_SUCCESS;
1093
1094	/*
1095	* Slow.
1096	*/
1097	if (fMaybeChangedMode)
1098	{
1099	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
1100	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1101	}
1102
1103	if (fUpdateCr3)
1104	{
1105	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
1106	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1107	}
1108
1109	return VINF_SUCCESS;
1110	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1111	}
1112
1113	#endif /* !IN_RING0 */
1114
1115
1116	/**
1117	* Interface for importing state on demand (used by IEM).
1118	*
1119	* @returns VBox status code.
1120	* @param pVCpu The cross context CPU structure.
1121	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1122	*/
1123	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1124	{
1125	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1126
1127	#ifdef IN_RING0
1128	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1129	return nemR0WinImportState(pVCpu->pGVM, pVCpu, &pVCpu->cpum.GstCtx, fWhat, true /fCanUpdateCr3/);
1130	# else
1131	RT_NOREF(pVCpu, fWhat);
1132	return VERR_NOT_IMPLEMENTED;
1133	# endif
1134	#else
1135	return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1136	#endif
1137	}
1138
1139
1140	/**
1141	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1142	*
1143	* @returns VBox status code.
1144	* @param pVCpu The cross context CPU structure.
1145	* @param pcTicks Where to return the CPU tick count.
1146	* @param puAux Where to return the TSC_AUX register value.
1147	*/
1148	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
1149	{
1150	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1151
1152	#ifdef IN_RING3
1153	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1154	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1155	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1156
1157	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1158	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
1159	if (pVM->nem.s.fUseRing0Runloop)
1160	# endif
1161	{
1162	/* Call ring-0 and get the values. */
1163	int rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_QUERY_CPU_TICK, 0, NULL);
1164	AssertLogRelRCReturn(rc, rc);
1165	*pcTicks = pVCpu->nem.s.Hypercall.QueryCpuTick.cTicks;
1166	if (puAux)
1167	*puAux = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX
1168	? pVCpu->nem.s.Hypercall.QueryCpuTick.uAux : CPUMGetGuestTscAux(pVCpu);
1169	return VINF_SUCCESS;
1170	}
1171	# endif
1172	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1173	/* Call the offical API. */
1174	WHV_REGISTER_NAME aenmNames[2] = { WHvX64RegisterTsc, WHvX64RegisterTscAux };
1175	WHV_REGISTER_VALUE aValues[2] = { {0, 0}, {0, 0} };
1176	Assert(RT_ELEMENTS(aenmNames) == RT_ELEMENTS(aValues));
1177	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, 2, aValues);
1178	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1179	("WHvGetVirtualProcessorRegisters(%p, %u,{tsc,tsc_aux},2,) -> %Rhrc (Last=%#x/%u)\n",
1180	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1181	, VERR_NEM_GET_REGISTERS_FAILED);
1182	*pcTicks = aValues[0].Reg64;
1183	if (puAux)
1184	*pcTicks = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX ? aValues[0].Reg64 : CPUMGetGuestTscAux(pVCpu);
1185	return VINF_SUCCESS;
1186	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1187	#else /* IN_RING0 */
1188	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1189	int rc = nemR0WinQueryCpuTick(pVCpu->pGVM, pVCpu, pcTicks, puAux);
1190	if (RT_SUCCESS(rc) && puAux && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX))
1191	*puAux = CPUMGetGuestTscAux(pVCpu);
1192	return rc;
1193	# else
1194	RT_NOREF(pVCpu, pcTicks, puAux);
1195	return VERR_NOT_IMPLEMENTED;
1196	# endif
1197	#endif /* IN_RING0 */
1198	}
1199
1200
1201	/**
1202	* Resumes CPU clock (TSC) on all virtual CPUs.
1203	*
1204	* This is called by TM when the VM is started, restored, resumed or similar.
1205	*
1206	* @returns VBox status code.
1207	* @param pVM The cross context VM structure.
1208	* @param pVCpu The cross context CPU structure of the calling EMT.
1209	* @param uPausedTscValue The TSC value at the time of pausing.
1210	*/
1211	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
1212	{
1213	#ifdef IN_RING0
1214	# ifdef NEM_WIN_WITH_RING0_RUNLOOP
1215	return nemR0WinResumeCpuTickOnAll(pVM, pVCpu, uPausedTscValue);
1216	# else
1217	RT_NOREF(pVM, pVCpu, uPausedTscValue);
1218	return VERR_NOT_IMPLEMENTED;
1219	# endif
1220	#else /* IN_RING3 */
1221	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1222	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1223
1224	# if defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1225	# if !defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS) && defined(NEM_WIN_WITH_RING0_RUNLOOP)
1226	if (pVM->nem.s.fUseRing0Runloop)
1227	# endif
1228	{
1229	/* Call ring-0 and do it all there. */
1230	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL, uPausedTscValue, NULL);
1231	}
1232	# endif
1233	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS
1234	/*
1235	* Call the offical API to do the job.
1236	*/
1237	if (pVM->cCpus > 1)
1238	RTThreadYield(); /* Try decrease the chance that we get rescheduled in the middle. */
1239
1240	/* Start with the first CPU. */
1241	WHV_REGISTER_NAME enmName = WHvX64RegisterTsc;
1242	WHV_REGISTER_VALUE Value = {0, 0};
1243	Value.Reg64 = uPausedTscValue;
1244	uint64_t const uFirstTsc = ASMReadTSC();
1245	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, 0 /iCpu/, &enmName, 1, &Value);
1246	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1247	("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64) -> %Rhrc (Last=%#x/%u)\n",
1248	pVM->nem.s.hPartition, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1249	, VERR_NEM_SET_TSC);
1250
1251	/* Do the other CPUs, adjusting for elapsed TSC and keeping finger crossed
1252	that we don't introduce too much drift here. */
1253	for (VMCPUID iCpu = 1; iCpu < pVM->cCpus; iCpu++)
1254	{
1255	Assert(enmName == WHvX64RegisterTsc);
1256	const uint64_t offDelta = (ASMReadTSC() - uFirstTsc);
1257	Value.Reg64 = uPausedTscValue + offDelta;
1258	hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, iCpu, &enmName, 1, &Value);
1259	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1260	("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64 + %#RX64) -> %Rhrc (Last=%#x/%u)\n",
1261	pVM->nem.s.hPartition, iCpu, uPausedTscValue, offDelta, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1262	, VERR_NEM_SET_TSC);
1263	}
1264
1265	return VINF_SUCCESS;
1266	# endif /* !NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS */
1267	#endif /* IN_RING3 */
1268	}
1269
1270	#ifdef NEMWIN_NEED_GET_REGISTER
1271	# if defined(IN_RING0) \|\| defined(NEM_WIN_USE_HYPERCALLS_FOR_REGISTERS)
1272	/** Worker for assertion macro. */
1273	NEM_TMPL_STATIC int nemHCWinGetRegister(PVMCPUCC pVCpu, PGVMCPU pGVCpu, uint32_t enmReg, HV_REGISTER_VALUE *pRetValue)
1274	{
1275	RT_ZERO(*pRetValue);
1276	# ifdef IN_RING3
1277	RT_NOREF(pVCpu, pGVCpu, enmReg);
1278	return VERR_NOT_IMPLEMENTED;
1279	# else
1280	NOREF(pVCpu);
1281
1282	/*
1283	* Hypercall parameters.
1284	*/
1285	HV_INPUT_GET_VP_REGISTERS pInput = (HV_INPUT_GET_VP_REGISTERS )pGVCpu->nem.s.HypercallData.pbPage;
1286	AssertPtrReturn(pInput, VERR_INTERNAL_ERROR_3);
1287	AssertReturn(g_pfnHvlInvokeHypercall, VERR_NEM_MISSING_KERNEL_API);
1288
1289	pInput->PartitionId = pVCpu->pGVM->nemr0.s.idHvPartition;
1290	pInput->VpIndex = pVCpu->idCpu;
1291	pInput->fFlags = 0;
1292	pInput->Names[0] = (HV_REGISTER_NAME)enmReg;
1293
1294	size_t const cbInput = RT_ALIGN_Z(RT_UOFFSETOF(HV_INPUT_GET_VP_REGISTERS, Names[1]), 32);
1295	HV_REGISTER_VALUE paValues = (HV_REGISTER_VALUE )((uint8_t *)pInput + cbInput);
1296	RT_BZERO(paValues, sizeof(paValues[0]) * 1);
1297
1298	/*
1299	* Make the hypercall and copy out the value.
1300	*/
1301	uint64_t uResult = g_pfnHvlInvokeHypercall(HV_MAKE_CALL_INFO(HvCallGetVpRegisters, 1),
1302	pGVCpu->nem.s.HypercallData.HCPhysPage,
1303	pGVCpu->nem.s.HypercallData.HCPhysPage + cbInput);
1304	AssertLogRelMsgReturn(uResult == HV_MAKE_CALL_REP_RET(1), ("uResult=%RX64 cRegs=%#x\n", uResult, 1),
1305	VERR_NEM_GET_REGISTERS_FAILED);
1306
1307	*pRetValue = paValues[0];
1308	return VINF_SUCCESS;
1309	# endif
1310	}
1311	# else
1312	/** Worker for assertion macro. */
1313	NEM_TMPL_STATIC int nemR3WinGetRegister(PVMCPUCC a_pVCpu, uint32_t a_enmReg, WHV_REGISTER_VALUE pValue)
1314	{
1315	RT_ZERO(*pRetValue);
1316	RT_NOREF(pVCpu, pGVCpu, enmReg);
1317	return VERR_NOT_IMPLEMENTED;
1318	}
1319	# endif
1320	#endif
1321
1322
1323	#ifdef LOG_ENABLED
1324	/**
1325	* Get the virtual processor running status.
1326	*/
1327	DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPUCC pVCpu)
1328	{
1329	# ifdef IN_RING0
1330	NOREF(pVCpu);
1331	return VidProcessorStatusUndefined;
1332	# else
1333	RTERRVARS Saved;
1334	RTErrVarsSave(&Saved);
1335
1336	/*
1337	* This API is disabled in release builds, it seems. On build 17101 it requires
1338	* the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
1339	*/
1340	VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
1341	NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
1342	AssertRC(rcNt);
1343
1344	RTErrVarsRestore(&Saved);
1345	return enmCpuStatus;
1346	# endif
1347	}
1348	#endif /* LOG_ENABLED */
1349
1350
1351	#if defined(NEM_WIN_USE_OUR_OWN_RUN_API) \|\| defined(NEM_WIN_WITH_RING0_RUNLOOP)
1352	# ifdef IN_RING3 /* hopefully not needed in ring-0, as we'd need KTHREADs and KeAlertThread. */
1353	/**
1354	* Our own WHvCancelRunVirtualProcessor that can later be moved to ring-0.
1355	*
1356	* This is an experiment only.
1357	*
1358	* @returns VBox status code.
1359	* @param pVM The cross context VM structure.
1360	* @param pVCpu The cross context virtual CPU structure of the
1361	* calling EMT.
1362	*/
1363	NEM_TMPL_STATIC int nemHCWinCancelRunVirtualProcessor(PVMCC pVM, PVMCPUCC pVCpu)
1364	{
1365	/*
1366	* Work the state.
1367	*
1368	* From the looks of things, we should let the EMT call VidStopVirtualProcessor.
1369	* So, we just need to modify the state and kick the EMT if it's waiting on
1370	* messages. For the latter we use QueueUserAPC / KeAlterThread.
1371	*/
1372	for (;;)
1373	{
1374	VMCPUSTATE enmState = VMCPU_GET_STATE(pVCpu);
1375	switch (enmState)
1376	{
1377	case VMCPUSTATE_STARTED_EXEC_NEM:
1378	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM))
1379	{
1380	DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM -> CANCELED");
1381	Log8(("nemHCWinCancelRunVirtualProcessor: Switched %u to canceled state\n", pVCpu->idCpu));
1382	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelChangedState);
1383	return VINF_SUCCESS;
1384	}
1385	break;
1386
1387	case VMCPUSTATE_STARTED_EXEC_NEM_WAIT:
1388	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED, VMCPUSTATE_STARTED_EXEC_NEM_WAIT))
1389	{
1390	DBGFTRACE_CUSTOM(pVM, "VMCPUSTATE_STARTED_EXEC_NEM_WAIT -> CANCELED");
1391	# ifdef IN_RING0
1392	NTSTATUS rcNt = KeAlertThread(??);
1393	DBGFTRACE_CUSTOM(pVM, "KeAlertThread -> %#x", rcNt);
1394	# else
1395	NTSTATUS rcNt = NtAlertThread(pVCpu->nem.s.hNativeThreadHandle);
1396	DBGFTRACE_CUSTOM(pVM, "NtAlertThread -> %#x", rcNt);
1397	# endif
1398	Log8(("nemHCWinCancelRunVirtualProcessor: Alerted %u: %#x\n", pVCpu->idCpu, rcNt));
1399	Assert(rcNt == STATUS_SUCCESS);
1400	if (NT_SUCCESS(rcNt))
1401	{
1402	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatCancelAlertedThread);
1403	return VINF_SUCCESS;
1404	}
1405	AssertLogRelMsgFailedReturn(("NtAlertThread failed: %#x\n", rcNt), RTErrConvertFromNtStatus(rcNt));
1406	}
1407	break;
1408
1409	default:
1410	return VINF_SUCCESS;
1411	}
1412
1413	ASMNopPause();
1414	RT_NOREF(pVM);
1415	}
1416	}
1417	# endif /* IN_RING3 */
1418	#endif /* NEM_WIN_USE_OUR_OWN_RUN_API \|\| NEM_WIN_WITH_RING0_RUNLOOP */
1419
1420
1421	#ifdef LOG_ENABLED
1422	/**
1423	* Logs the current CPU state.
1424	*/
1425	NEM_TMPL_STATIC void nemHCWinLogState(PVMCC pVM, PVMCPUCC pVCpu)
1426	{
1427	if (LogIs3Enabled())
1428	{
1429	# if 0 // def IN_RING3 - causes lazy state import assertions all over CPUM.
1430	char szRegs[4096];
1431	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
1432	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
1433	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
1434	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
1435	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
1436	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
1437	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
1438	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
1439	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
1440	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
1441	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
1442	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
1443	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
1444	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
1445	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
1446	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
1447	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
1448	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
1449	" efer=%016VR{efer}\n"
1450	" pat=%016VR{pat}\n"
1451	" sf_mask=%016VR{sf_mask}\n"
1452	"krnl_gs_base=%016VR{krnl_gs_base}\n"
1453	" lstar=%016VR{lstar}\n"
1454	" star=%016VR{star} cstar=%016VR{cstar}\n"
1455	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
1456	);
1457
1458	char szInstr[256];
1459	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
1460	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
1461	szInstr, sizeof(szInstr), NULL);
1462	Log3(("%s%s\n", szRegs, szInstr));
1463	# else
1464	/** @todo stat logging in ring-0 */
1465	RT_NOREF(pVM, pVCpu);
1466	# endif
1467	}
1468	}
1469	#endif /* LOG_ENABLED */
1470
1471
1472	/** Macro used by nemHCWinExecStateToLogStr and nemR3WinExecStateToLogStr. */
1473	#define SWITCH_IT(a_szPrefix) \
1474	do \
1475	switch (u)\
1476	{ \
1477	case 0x00: return a_szPrefix ""; \
1478	case 0x01: return a_szPrefix ",Pnd"; \
1479	case 0x02: return a_szPrefix ",Dbg"; \
1480	case 0x03: return a_szPrefix ",Pnd,Dbg"; \
1481	case 0x04: return a_szPrefix ",Shw"; \
1482	case 0x05: return a_szPrefix ",Pnd,Shw"; \
1483	case 0x06: return a_szPrefix ",Shw,Dbg"; \
1484	case 0x07: return a_szPrefix ",Pnd,Shw,Dbg"; \
1485	default: AssertFailedReturn("WTF?"); \
1486	} \
1487	while (0)
1488
1489	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1490	/**
1491	* Translates the execution stat bitfield into a short log string, VID version.
1492	*
1493	* @returns Read-only log string.
1494	* @param pMsgHdr The header which state to summarize.
1495	*/
1496	static const char nemHCWinExecStateToLogStr(HV_X64_INTERCEPT_MESSAGE_HEADER const pMsgHdr)
1497	{
1498	unsigned u = (unsigned)pMsgHdr->ExecutionState.InterruptionPending
1499	\| ((unsigned)pMsgHdr->ExecutionState.DebugActive << 1)
1500	\| ((unsigned)pMsgHdr->ExecutionState.InterruptShadow << 2);
1501	if (pMsgHdr->ExecutionState.EferLma)
1502	SWITCH_IT("LM");
1503	else if (pMsgHdr->ExecutionState.Cr0Pe)
1504	SWITCH_IT("PM");
1505	else
1506	SWITCH_IT("RM");
1507	}
1508	#elif defined(IN_RING3)
1509	/**
1510	* Translates the execution stat bitfield into a short log string, WinHv version.
1511	*
1512	* @returns Read-only log string.
1513	* @param pExitCtx The exit context which state to summarize.
1514	*/
1515	static const char nemR3WinExecStateToLogStr(WHV_VP_EXIT_CONTEXT const pExitCtx)
1516	{
1517	unsigned u = (unsigned)pExitCtx->ExecutionState.InterruptionPending
1518	\| ((unsigned)pExitCtx->ExecutionState.DebugActive << 1)
1519	\| ((unsigned)pExitCtx->ExecutionState.InterruptShadow << 2);
1520	if (pExitCtx->ExecutionState.EferLma)
1521	SWITCH_IT("LM");
1522	else if (pExitCtx->ExecutionState.Cr0Pe)
1523	SWITCH_IT("PM");
1524	else
1525	SWITCH_IT("RM");
1526	}
1527	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1528	#undef SWITCH_IT
1529
1530
1531	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1532	/**
1533	* Advances the guest RIP and clear EFLAGS.RF, VID version.
1534	*
1535	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
1536	*
1537	* @param pVCpu The cross context virtual CPU structure.
1538	* @param pExitCtx The exit context.
1539	* @param cbMinInstr The minimum instruction length, or 1 if not unknown.
1540	*/
1541	DECLINLINE(void)
1542	nemHCWinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr, uint8_t cbMinInstr)
1543	{
1544	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
1545
1546	/* Advance the RIP. */
1547	Assert(pMsgHdr->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
1548	pVCpu->cpum.GstCtx.rip += pMsgHdr->InstructionLength;
1549	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
1550
1551	/* Update interrupt inhibition. */
1552	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1553	{ /* likely */ }
1554	else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
1555	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1556	}
1557	#elif defined(IN_RING3)
1558	/**
1559	* Advances the guest RIP and clear EFLAGS.RF, WinHv version.
1560	*
1561	* This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
1562	*
1563	* @param pVCpu The cross context virtual CPU structure.
1564	* @param pExitCtx The exit context.
1565	* @param cbMinInstr The minimum instruction length, or 1 if not unknown.
1566	*/
1567	DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx, uint8_t cbMinInstr)
1568	{
1569	Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)));
1570
1571	/* Advance the RIP. */
1572	Assert(pExitCtx->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
1573	pVCpu->cpum.GstCtx.rip += pExitCtx->InstructionLength;
1574	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
1575
1576	/* Update interrupt inhibition. */
1577	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1578	{ /* likely */ }
1579	else if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
1580	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1581	}
1582	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1583
1584
1585
1586	NEM_TMPL_STATIC DECLCALLBACK(int)
1587	nemHCWinUnmapOnePageCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint8_t pu2NemState, void pvUser)
1588	{
1589	RT_NOREF_PV(pvUser);
1590	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1591	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1592	AssertRC(rc);
1593	if (RT_SUCCESS(rc))
1594	#else
1595	RT_NOREF_PV(pVCpu);
1596	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1597	if (SUCCEEDED(hrc))
1598	#endif
1599	{
1600	Log5(("NEM GPA unmap all: %RGp (cMappedPages=%u)\n", GCPhys, pVM->nem.s.cMappedPages - 1));
1601	*pu2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1602	}
1603	else
1604	{
1605	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1606	LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
1607	#else
1608	LogRel(("nemR3WinUnmapOnePageCallback: GCPhys=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1609	GCPhys, g_apszPageStates[*pu2NemState], hrc, hrc, RTNtLastStatusValue(),
1610	RTNtLastErrorValue(), pVM->nem.s.cMappedPages));
1611	#endif
1612	*pu2NemState = NEM_WIN_PAGE_STATE_NOT_SET;
1613	}
1614	if (pVM->nem.s.cMappedPages > 0)
1615	ASMAtomicDecU32(&pVM->nem.s.cMappedPages);
1616	return VINF_SUCCESS;
1617	}
1618
1619
1620	/**
1621	* State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1622	* and nemHCWinHandleMemoryAccessPageCheckerCallback.
1623	*/
1624	typedef struct NEMHCWINHMACPCCSTATE
1625	{
1626	/** Input: Write access. */
1627	bool fWriteAccess;
1628	/** Output: Set if we did something. */
1629	bool fDidSomething;
1630	/** Output: Set it we should resume. */
1631	bool fCanResume;
1632	} NEMHCWINHMACPCCSTATE;
1633
1634	/**
1635	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1636	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1637	* NEMHCWINHMACPCCSTATE structure. }
1638	*/
1639	NEM_TMPL_STATIC DECLCALLBACK(int)
1640	nemHCWinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1641	{
1642	NEMHCWINHMACPCCSTATE pState = (NEMHCWINHMACPCCSTATE )pvUser;
1643	pState->fDidSomething = false;
1644	pState->fCanResume = false;
1645
1646	/* If A20 is disabled, we may need to make another query on the masked
1647	page to get the correct protection information. */
1648	uint8_t u2State = pInfo->u2NemState;
1649	RTGCPHYS GCPhysSrc;
1650	if ( pVM->nem.s.fA20Enabled
1651	\|\| !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
1652	GCPhysSrc = GCPhys;
1653	else
1654	{
1655	GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
1656	PGMPHYSNEMPAGEINFO Info2;
1657	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
1658	AssertRCReturn(rc, rc);
1659
1660	*pInfo = Info2;
1661	pInfo->u2NemState = u2State;
1662	}
1663
1664	/*
1665	* Consolidate current page state with actual page protection and access type.
1666	* We don't really consider downgrades here, as they shouldn't happen.
1667	*/
1668	#ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1669	/** @todo Someone at microsoft please explain:
1670	* I'm not sure WTF was going on, but I ended up in a loop if I remapped a
1671	* readonly page as writable (unmap, then map again). Specifically, this was an
1672	* issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1. So, in
1673	* a hope to work around that we no longer pre-map anything, just unmap stuff
1674	* and do it lazily here. And here we will first unmap, restart, and then remap
1675	* with new protection or backing.
1676	*/
1677	#endif
1678	int rc;
1679	switch (u2State)
1680	{
1681	case NEM_WIN_PAGE_STATE_UNMAPPED:
1682	case NEM_WIN_PAGE_STATE_NOT_SET:
1683	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1684	{
1685	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1686	return VINF_SUCCESS;
1687	}
1688
1689	/* Don't bother remapping it if it's a write request to a non-writable page. */
1690	if ( pState->fWriteAccess
1691	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1692	{
1693	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1694	return VINF_SUCCESS;
1695	}
1696
1697	/* Map the page. */
1698	rc = nemHCNativeSetPhysPage(pVM,
1699	pVCpu,
1700	GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1701	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1702	pInfo->fNemProt,
1703	&u2State,
1704	true /fBackingState/);
1705	pInfo->u2NemState = u2State;
1706	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1707	GCPhys, g_apszPageStates[u2State], rc));
1708	pState->fDidSomething = true;
1709	pState->fCanResume = true;
1710	return rc;
1711
1712	case NEM_WIN_PAGE_STATE_READABLE:
1713	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1714	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1715	{
1716	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1717	return VINF_SUCCESS;
1718	}
1719
1720	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1721	/* Upgrade page to writable. */
1722	/** @todo test this*/
1723	if ( (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1724	&& pState->fWriteAccess)
1725	{
1726	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhys,
1727	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
1728	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
1729	AssertRC(rc);
1730	if (RT_SUCCESS(rc))
1731	{
1732	pInfo->u2NemState = NEM_WIN_PAGE_STATE_WRITABLE;
1733	pState->fDidSomething = true;
1734	pState->fCanResume = true;
1735	Log5(("NEM GPA write-upgrade/exit: %RGp (was %s, cMappedPages=%u)\n",
1736	GCPhys, g_apszPageStates[u2State], pVM->nem.s.cMappedPages));
1737	}
1738	}
1739	else
1740	{
1741	/* Need to emulate the acces. */
1742	AssertBreak(pInfo->fNemProt != NEM_PAGE_PROT_NONE); /* There should be no downgrades. */
1743	rc = VINF_SUCCESS;
1744	}
1745	return rc;
1746	#else
1747	break;
1748	#endif
1749
1750	case NEM_WIN_PAGE_STATE_WRITABLE:
1751	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1752	{
1753	if (pInfo->u2OldNemState == NEM_WIN_PAGE_STATE_WRITABLE)
1754	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
1755	else
1756	{
1757	pState->fCanResume = true;
1758	Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
1759	GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
1760	}
1761	return VINF_SUCCESS;
1762	}
1763	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1764	AssertFailed(); /* There should be no downgrades. */
1765	#endif
1766	break;
1767
1768	default:
1769	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1770	}
1771
1772	/*
1773	* Unmap and restart the instruction.
1774	* If this fails, which it does every so often, just unmap everything for now.
1775	*/
1776	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1777	rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
1778	AssertRC(rc);
1779	if (RT_SUCCESS(rc))
1780	#else
1781	/** @todo figure out whether we mess up the state or if it's WHv. */
1782	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
1783	if (SUCCEEDED(hrc))
1784	#endif
1785	{
1786	pState->fDidSomething = true;
1787	pState->fCanResume = true;
1788	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1789	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
1790	Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
1791	return VINF_SUCCESS;
1792	}
1793	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
1794	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhys, rc));
1795	return rc;
1796	#else
1797	LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x) Last=%#x/%u (cMappedPages=%u)\n",
1798	GCPhys, g_apszPageStates[u2State], hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue(),
1799	pVM->nem.s.cMappedPages));
1800
1801	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinUnmapOnePageCallback, NULL);
1802	Log(("nemHCWinHandleMemoryAccessPageCheckerCallback: Unmapped all (cMappedPages=%u)\n", pVM->nem.s.cMappedPages));
1803
1804	pState->fDidSomething = true;
1805	pState->fCanResume = true;
1806	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
1807	return VINF_SUCCESS;
1808	#endif
1809	}
1810
1811
1812
1813	#if defined(IN_RING0) && defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API)
1814	/**
1815	* Wrapper around nemR0WinImportState that converts VERR_NEM_FLUSH_TLB
1816	* into informational status codes and logs+asserts statuses.
1817	*
1818	* @returns VBox strict status code.
1819	* @param pGVM The global (ring-0) VM structure.
1820	* @param pGVCpu The global (ring-0) per CPU structure.
1821	* @param fWhat What to import.
1822	* @param pszCaller Who is doing the importing.
1823	*/
1824	DECLINLINE(VBOXSTRICTRC) nemR0WinImportStateStrict(PGVM pGVM, PGVMCPU pGVCpu, uint64_t fWhat, const char *pszCaller)
1825	{
1826	int rc = nemR0WinImportState(pGVM, pGVCpu, &pGVCpu->cpum.GstCtx, fWhat, true /fCanUpdateCr3/);
1827	if (RT_SUCCESS(rc))
1828	{
1829	Assert(rc == VINF_SUCCESS);
1830	return VINF_SUCCESS;
1831	}
1832
1833	if (rc == VERR_NEM_FLUSH_TLB)
1834	{
1835	Log4(("%s/%u: nemR0WinImportState -> %Rrc\n", pszCaller, pGVCpu->idCpu, -rc));
1836	return -rc;
1837	}
1838	RT_NOREF(pszCaller);
1839	AssertMsgFailedReturn(("%s/%u: nemR0WinImportState failed: %Rrc\n", pszCaller, pGVCpu->idCpu, rc), rc);
1840	}
1841	#endif /* IN_RING0 && NEM_WIN_TEMPLATE_MODE_OWN_RUN_API*/
1842
1843	#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3)
1844	/**
1845	* Wrapper around nemR0WinImportStateStrict and nemHCWinCopyStateFromHyperV.
1846	*
1847	* Unlike the wrapped APIs, this checks whether it's necessary.
1848	*
1849	* @returns VBox strict status code.
1850	* @param pVCpu The cross context per CPU structure.
1851	* @param fWhat What to import.
1852	* @param pszCaller Who is doing the importing.
1853	*/
1854	DECLINLINE(VBOXSTRICTRC) nemHCWinImportStateIfNeededStrict(PVMCPUCC pVCpu, uint64_t fWhat, const char *pszCaller)
1855	{
1856	if (pVCpu->cpum.GstCtx.fExtrn & fWhat)
1857	{
1858	# ifdef IN_RING0
1859	return nemR0WinImportStateStrict(pVCpu->pGVM, pVCpu, fWhat, pszCaller);
1860	# else
1861	RT_NOREF(pszCaller);
1862	int rc = nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1863	AssertRCReturn(rc, rc);
1864	# endif
1865	}
1866	return VINF_SUCCESS;
1867	}
1868	#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API \|\| IN_RING3 */
1869
1870	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1871	/**
1872	* Copies register state from the X64 intercept message header.
1873	*
1874	* ASSUMES no state copied yet.
1875	*
1876	* @param pVCpu The cross context per CPU structure.
1877	* @param pHdr The X64 intercept message header.
1878	* @sa nemR3WinCopyStateFromX64Header
1879	*/
1880	DECLINLINE(void) nemHCWinCopyStateFromX64Header(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pHdr)
1881	{
1882	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
1883	== (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT));
1884	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pHdr->CsSegment);
1885	pVCpu->cpum.GstCtx.rip = pHdr->Rip;
1886	pVCpu->cpum.GstCtx.rflags.u = pHdr->Rflags;
1887
1888	pVCpu->nem.s.fLastInterruptShadow = pHdr->ExecutionState.InterruptShadow;
1889	if (!pHdr->ExecutionState.InterruptShadow)
1890	{
1891	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1892	{ /* likely */ }
1893	else
1894	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1895	}
1896	else
1897	EMSetInhibitInterruptsPC(pVCpu, pHdr->Rip);
1898
1899	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT);
1900	}
1901	#elif defined(IN_RING3)
1902	/**
1903	* Copies register state from the (common) exit context.
1904	*
1905	* ASSUMES no state copied yet.
1906	*
1907	* @param pVCpu The cross context per CPU structure.
1908	* @param pExitCtx The common exit context.
1909	* @sa nemHCWinCopyStateFromX64Header
1910	*/
1911	DECLINLINE(void) nemR3WinCopyStateFromX64Header(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx)
1912	{
1913	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT))
1914	== (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT));
1915	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pExitCtx->Cs);
1916	pVCpu->cpum.GstCtx.rip = pExitCtx->Rip;
1917	pVCpu->cpum.GstCtx.rflags.u = pExitCtx->Rflags;
1918
1919	pVCpu->nem.s.fLastInterruptShadow = pExitCtx->ExecutionState.InterruptShadow;
1920	if (!pExitCtx->ExecutionState.InterruptShadow)
1921	{
1922	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1923	{ /* likely */ }
1924	else
1925	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1926	}
1927	else
1928	EMSetInhibitInterruptsPC(pVCpu, pExitCtx->Rip);
1929
1930	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT);
1931	}
1932	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
1933
1934
1935	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
1936	/**
1937	* Deals with memory intercept message.
1938	*
1939	* @returns Strict VBox status code.
1940	* @param pVM The cross context VM structure.
1941	* @param pVCpu The cross context per CPU structure.
1942	* @param pMsg The message.
1943	* @sa nemR3WinHandleExitMemory
1944	*/
1945	NEM_TMPL_STATIC VBOXSTRICTRC
1946	nemHCWinHandleMessageMemory(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_MEMORY_INTERCEPT_MESSAGE const *pMsg)
1947	{
1948	uint64_t const uHostTsc = ASMReadTSC();
1949	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
1950	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
1951	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
1952
1953	/*
1954	* Whatever we do, we must clear pending event injection upon resume.
1955	*/
1956	if (pMsg->Header.ExecutionState.InterruptionPending)
1957	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
1958
1959	# if 0 /* Experiment: 20K -> 34K exit/s. */
1960	if ( pMsg->Header.ExecutionState.EferLma
1961	&& pMsg->Header.CsSegment.Long
1962	&& pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
1963	{
1964	if ( pMsg->Header.Rip - (uint64_t)0xf65a < (uint64_t)(0xf662 - 0xf65a)
1965	&& pMsg->InstructionBytes[0] == 0x89
1966	&& pMsg->InstructionBytes[1] == 0x03)
1967	{
1968	pVCpu->cpum.GstCtx.rip = pMsg->Header.Rip + 2;
1969	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
1970	AssertMsg(pMsg->Header.InstructionLength == 2, ("%#x\n", pMsg->Header.InstructionLength));
1971	//Log(("%RX64 msg:\n%.80Rhxd\n", pVCpu->cpum.GstCtx.rip, pMsg));
1972	return VINF_SUCCESS;
1973	}
1974	}
1975	# endif
1976
1977	/*
1978	* Ask PGM for information about the given GCPhys. We need to check if we're
1979	* out of sync first.
1980	*/
1981	NEMHCWINHMACPCCSTATE State = { pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE, false, false };
1982	PGMPHYSNEMPAGEINFO Info;
1983	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pMsg->GuestPhysicalAddress, State.fWriteAccess, &Info,
1984	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
1985	if (RT_SUCCESS(rc))
1986	{
1987	if (Info.fNemProt & ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
1988	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
1989	{
1990	if (State.fCanResume)
1991	{
1992	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
1993	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
1994	pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
1995	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
1996	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
1997	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
1998	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
1999	return VINF_SUCCESS;
2000	}
2001	}
2002	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2003	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2004	pMsg->GuestPhysicalAddress, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2005	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2006	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2007	}
2008	else
2009	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
2010	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2011	pMsg->GuestPhysicalAddress, rc, State.fDidSomething ? " modified-backing" : "",
2012	g_apszHvInterceptAccessTypes[pMsg->Header.InterceptAccessType]));
2013
2014	/*
2015	* Emulate the memory access, either access handler or special memory.
2016	*/
2017	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2018	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2019	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2020	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2021	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, uHostTsc);
2022	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2023	VBOXSTRICTRC rcStrict;
2024	# ifdef IN_RING0
2025	rcStrict = nemR0WinImportStateStrict(pVM, pVCpu,
2026	NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES, "MemExit");
2027	if (rcStrict != VINF_SUCCESS)
2028	return rcStrict;
2029	# else
2030	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2031	AssertRCReturn(rc, rc);
2032	# endif
2033
2034	if (pMsg->Reserved1)
2035	Log(("MemExit/Reserved1=%#x\n", pMsg->Reserved1));
2036	if (pMsg->Header.ExecutionState.Reserved0 \|\| pMsg->Header.ExecutionState.Reserved1)
2037	Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pMsg->Header.ExecutionState.Reserved0, pMsg->Header.ExecutionState.Reserved1));
2038
2039	if (!pExitRec)
2040	{
2041	//if (pMsg->InstructionByteCount > 0)
2042	// Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
2043	if (pMsg->InstructionByteCount > 0)
2044	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
2045	pMsg->InstructionBytes, pMsg->InstructionByteCount);
2046	else
2047	rcStrict = IEMExecOne(pVCpu);
2048	/** @todo do we need to do anything wrt debugging here? */
2049	}
2050	else
2051	{
2052	/* Frequent access or probing. */
2053	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2054	Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2055	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2056	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2057	}
2058	return rcStrict;
2059	}
2060	#elif defined(IN_RING3)
2061	/**
2062	* Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
2063	*
2064	* @returns Strict VBox status code.
2065	* @param pVM The cross context VM structure.
2066	* @param pVCpu The cross context per CPU structure.
2067	* @param pExit The VM exit information to handle.
2068	* @sa nemHCWinHandleMessageMemory
2069	*/
2070	NEM_TMPL_STATIC VBOXSTRICTRC
2071	nemR3WinHandleExitMemory(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2072	{
2073	uint64_t const uHostTsc = ASMReadTSC();
2074	Assert(pExit->MemoryAccess.AccessInfo.AccessType != 3);
2075
2076	/*
2077	* Whatever we do, we must clear pending event injection upon resume.
2078	*/
2079	if (pExit->VpContext.ExecutionState.InterruptionPending)
2080	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2081
2082	/*
2083	* Ask PGM for information about the given GCPhys. We need to check if we're
2084	* out of sync first.
2085	*/
2086	NEMHCWINHMACPCCSTATE State = { pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
2087	PGMPHYSNEMPAGEINFO Info;
2088	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pExit->MemoryAccess.Gpa, State.fWriteAccess, &Info,
2089	nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
2090	if (RT_SUCCESS(rc))
2091	{
2092	if (Info.fNemProt & ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2093	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
2094	{
2095	if (State.fCanResume)
2096	{
2097	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
2098	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2099	pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2100	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2101	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2102	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
2103	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
2104	return VINF_SUCCESS;
2105	}
2106	}
2107	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
2108	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2109	pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
2110	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
2111	State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2112	}
2113	else
2114	Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
2115	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2116	pExit->MemoryAccess.Gpa, rc, State.fDidSomething ? " modified-backing" : "",
2117	g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
2118
2119	/*
2120	* Emulate the memory access, either access handler or special memory.
2121	*/
2122	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2123	pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2124	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
2125	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
2126	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
2127	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2128	rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2129	AssertRCReturn(rc, rc);
2130	if (pExit->VpContext.ExecutionState.Reserved0 \|\| pExit->VpContext.ExecutionState.Reserved1)
2131	Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pExit->VpContext.ExecutionState.Reserved0, pExit->VpContext.ExecutionState.Reserved1));
2132
2133	VBOXSTRICTRC rcStrict;
2134	if (!pExitRec)
2135	{
2136	//if (pMsg->InstructionByteCount > 0)
2137	// Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
2138	if (pExit->MemoryAccess.InstructionByteCount > 0)
2139	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
2140	pExit->MemoryAccess.InstructionBytes, pExit->MemoryAccess.InstructionByteCount);
2141	else
2142	rcStrict = IEMExecOne(pVCpu);
2143	/** @todo do we need to do anything wrt debugging here? */
2144	}
2145	else
2146	{
2147	/* Frequent access or probing. */
2148	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2149	Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2150	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2151	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2152	}
2153	return rcStrict;
2154	}
2155	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2156
2157
2158	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2159	/**
2160	* Deals with I/O port intercept message.
2161	*
2162	* @returns Strict VBox status code.
2163	* @param pVM The cross context VM structure.
2164	* @param pVCpu The cross context per CPU structure.
2165	* @param pMsg The message.
2166	*/
2167	NEM_TMPL_STATIC VBOXSTRICTRC
2168	nemHCWinHandleMessageIoPort(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_IO_PORT_INTERCEPT_MESSAGE const *pMsg)
2169	{
2170	/*
2171	* Assert message sanity.
2172	*/
2173	Assert( pMsg->AccessInfo.AccessSize == 1
2174	\|\| pMsg->AccessInfo.AccessSize == 2
2175	\|\| pMsg->AccessInfo.AccessSize == 4);
2176	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
2177	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2178	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2179	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2180	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2181	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2182	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
2183	if (pMsg->AccessInfo.StringOp)
2184	{
2185	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterDs, pMsg->DsSegment);
2186	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterEs, pMsg->EsSegment);
2187	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
2188	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsi, pMsg->Rsi);
2189	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdi, pMsg->Rdi);
2190	}
2191
2192	/*
2193	* Whatever we do, we must clear pending event injection upon resume.
2194	*/
2195	if (pMsg->Header.ExecutionState.InterruptionPending)
2196	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2197
2198	/*
2199	* Add history first to avoid two paths doing EMHistoryExec calls.
2200	*/
2201	VBOXSTRICTRC rcStrict;
2202	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2203	!pMsg->AccessInfo.StringOp
2204	? ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2205	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
2206	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
2207	: ( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2208	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
2209	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
2210	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2211	if (!pExitRec)
2212	{
2213	if (!pMsg->AccessInfo.StringOp)
2214	{
2215	/*
2216	* Simple port I/O.
2217	*/
2218	static uint32_t const s_fAndMask[8] =
2219	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2220	uint32_t const fAndMask = s_fAndMask[pMsg->AccessInfo.AccessSize];
2221
2222	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2223	if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2224	{
2225	rcStrict = IOMIOPortWrite(pVM, pVCpu, pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize);
2226	Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2227	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2228	pMsg->PortNumber, (uint32_t)pMsg->Rax & fAndMask, pMsg->AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
2229	if (IOM_SUCCESS(rcStrict))
2230	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
2231	# ifdef IN_RING0
2232	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
2233	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
2234	/** @todo check for debug breakpoints */ )
2235	return EMRZSetPendingIoPortWrite(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
2236	pMsg->AccessInfo.AccessSize, (uint32_t)pMsg->Rax & fAndMask);
2237	# endif
2238	else
2239	{
2240	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2241	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2242	}
2243	}
2244	else
2245	{
2246	uint32_t uValue = 0;
2247	rcStrict = IOMIOPortRead(pVM, pVCpu, pMsg->PortNumber, &uValue, pMsg->AccessInfo.AccessSize);
2248	Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2249	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2250	pMsg->PortNumber, pMsg->AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2251	if (IOM_SUCCESS(rcStrict))
2252	{
2253	if (pMsg->AccessInfo.AccessSize != 4)
2254	pVCpu->cpum.GstCtx.rax = (pMsg->Rax & ~(uint64_t)fAndMask) \| (uValue & fAndMask);
2255	else
2256	pVCpu->cpum.GstCtx.rax = uValue;
2257	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2258	Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pMsg->Rax, pVCpu->cpum.GstCtx.rax));
2259	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 1);
2260	}
2261	else
2262	{
2263	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2264	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2265	# ifdef IN_RING0
2266	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
2267	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1TF
2268	/** @todo check for debug breakpoints */ )
2269	return EMRZSetPendingIoPortRead(pVCpu, pMsg->PortNumber, pMsg->Header.InstructionLength,
2270	pMsg->AccessInfo.AccessSize);
2271	# endif
2272	}
2273	}
2274	}
2275	else
2276	{
2277	/*
2278	* String port I/O.
2279	*/
2280	/** @todo Someone at Microsoft please explain how we can get the address mode
2281	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2282	* getting the default mode, it can always be overridden by a prefix. This
2283	* forces us to interpret the instruction from opcodes, which is suboptimal.
2284	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2285	* CPUs that are reasonably new.
2286	*
2287	* Of course, it's possible this is an undocumented and we just need to do some
2288	* experiments to figure out how it's communicated. Alternatively, we can scan
2289	* the opcode bytes for possible evil prefixes.
2290	*/
2291	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2292	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2293	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2294	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
2295	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
2296	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2297	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2298	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
2299	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
2300	# ifdef IN_RING0
2301	rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
2302	if (rcStrict != VINF_SUCCESS)
2303	return rcStrict;
2304	# else
2305	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2306	AssertRCReturn(rc, rc);
2307	# endif
2308
2309	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
2310	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2311	pMsg->AccessInfo.RepPrefix ? "REP " : "",
2312	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUTS" : "INS",
2313	pMsg->PortNumber, pMsg->AccessInfo.AccessSize ));
2314	rcStrict = IEMExecOne(pVCpu);
2315	}
2316	if (IOM_SUCCESS(rcStrict))
2317	{
2318	/*
2319	* Do debug checks.
2320	*/
2321	if ( pMsg->Header.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
2322	\|\| (pMsg->Header.Rflags & X86_EFL_TF)
2323	\|\| DBGFBpIsHwIoArmed(pVM) )
2324	{
2325	/** @todo Debugging. */
2326	}
2327	}
2328	return rcStrict;
2329	}
2330
2331	/*
2332	* Frequent exit or something needing probing.
2333	* Get state and call EMHistoryExec.
2334	*/
2335	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2336	if (!pMsg->AccessInfo.StringOp)
2337	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2338	else
2339	{
2340	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2341	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2342	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
2343	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pMsg->EsSegment);
2344	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2345	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
2346	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
2347	}
2348	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2349
2350	# ifdef IN_RING0
2351	rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "IOExit");
2352	if (rcStrict != VINF_SUCCESS)
2353	return rcStrict;
2354	# else
2355	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2356	AssertRCReturn(rc, rc);
2357	# endif
2358
2359	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
2360	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2361	pMsg->AccessInfo.RepPrefix ? "REP " : "",
2362	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "OUT" : "IN",
2363	pMsg->AccessInfo.StringOp ? "S" : "",
2364	pMsg->PortNumber, pMsg->AccessInfo.AccessSize));
2365	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2366	Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2367	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2368	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2369	return rcStrict;
2370	}
2371	#elif defined(IN_RING3)
2372	/**
2373	* Deals with I/O port access exits (WHvRunVpExitReasonX64IoPortAccess).
2374	*
2375	* @returns Strict VBox status code.
2376	* @param pVM The cross context VM structure.
2377	* @param pVCpu The cross context per CPU structure.
2378	* @param pExit The VM exit information to handle.
2379	* @sa nemHCWinHandleMessageIoPort
2380	*/
2381	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitIoPort(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2382	{
2383	Assert( pExit->IoPortAccess.AccessInfo.AccessSize == 1
2384	\|\| pExit->IoPortAccess.AccessInfo.AccessSize == 2
2385	\|\| pExit->IoPortAccess.AccessInfo.AccessSize == 4);
2386
2387	/*
2388	* Whatever we do, we must clear pending event injection upon resume.
2389	*/
2390	if (pExit->VpContext.ExecutionState.InterruptionPending)
2391	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
2392
2393	/*
2394	* Add history first to avoid two paths doing EMHistoryExec calls.
2395	*/
2396	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2397	!pExit->IoPortAccess.AccessInfo.StringOp
2398	? ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2399	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
2400	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
2401	: ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
2402	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
2403	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
2404	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2405	if (!pExitRec)
2406	{
2407	VBOXSTRICTRC rcStrict;
2408	if (!pExit->IoPortAccess.AccessInfo.StringOp)
2409	{
2410	/*
2411	* Simple port I/O.
2412	*/
2413	static uint32_t const s_fAndMask[8] =
2414	{ UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX };
2415	uint32_t const fAndMask = s_fAndMask[pExit->IoPortAccess.AccessInfo.AccessSize];
2416	if (pExit->IoPortAccess.AccessInfo.IsWrite)
2417	{
2418	rcStrict = IOMIOPortWrite(pVM, pVCpu, pExit->IoPortAccess.PortNumber,
2419	(uint32_t)pExit->IoPortAccess.Rax & fAndMask,
2420	pExit->IoPortAccess.AccessInfo.AccessSize);
2421	Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
2422	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2423	pExit->IoPortAccess.PortNumber, (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
2424	pExit->IoPortAccess.AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
2425	if (IOM_SUCCESS(rcStrict))
2426	{
2427	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2428	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
2429	}
2430	}
2431	else
2432	{
2433	uint32_t uValue = 0;
2434	rcStrict = IOMIOPortRead(pVM, pVCpu, pExit->IoPortAccess.PortNumber, &uValue,
2435	pExit->IoPortAccess.AccessInfo.AccessSize);
2436	Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
2437	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2438	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2439	if (IOM_SUCCESS(rcStrict))
2440	{
2441	if (pExit->IoPortAccess.AccessInfo.AccessSize != 4)
2442	pVCpu->cpum.GstCtx.rax = (pExit->IoPortAccess.Rax & ~(uint64_t)fAndMask) \| (uValue & fAndMask);
2443	else
2444	pVCpu->cpum.GstCtx.rax = uValue;
2445	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2446	Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pExit->IoPortAccess.Rax, pVCpu->cpum.GstCtx.rax));
2447	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2448	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
2449	}
2450	}
2451	}
2452	else
2453	{
2454	/*
2455	* String port I/O.
2456	*/
2457	/** @todo Someone at Microsoft please explain how we can get the address mode
2458	* from the IoPortAccess.VpContext. CS.Attributes is only sufficient for
2459	* getting the default mode, it can always be overridden by a prefix. This
2460	* forces us to interpret the instruction from opcodes, which is suboptimal.
2461	* Both AMD-V and VT-x includes the address size in the exit info, at least on
2462	* CPUs that are reasonably new.
2463	*
2464	* Of course, it's possible this is an undocumented and we just need to do some
2465	* experiments to figure out how it's communicated. Alternatively, we can scan
2466	* the opcode bytes for possible evil prefixes.
2467	*/
2468	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2469	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2470	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2471	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
2472	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
2473	pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
2474	pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
2475	pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
2476	pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
2477	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2478	AssertRCReturn(rc, rc);
2479
2480	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
2481	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2482	pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
2483	pExit->IoPortAccess.AccessInfo.IsWrite ? "OUTS" : "INS",
2484	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize ));
2485	rcStrict = IEMExecOne(pVCpu);
2486	}
2487	if (IOM_SUCCESS(rcStrict))
2488	{
2489	/*
2490	* Do debug checks.
2491	*/
2492	if ( pExit->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
2493	\|\| (pExit->VpContext.Rflags & X86_EFL_TF)
2494	\|\| DBGFBpIsHwIoArmed(pVM) )
2495	{
2496	/** @todo Debugging. */
2497	}
2498	}
2499	return rcStrict;
2500	}
2501
2502	/*
2503	* Frequent exit or something needing probing.
2504	* Get state and call EMHistoryExec.
2505	*/
2506	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2507	if (!pExit->IoPortAccess.AccessInfo.StringOp)
2508	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
2509	else
2510	{
2511	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDI \| CPUMCTX_EXTRN_RSI
2512	\| CPUMCTX_EXTRN_DS \| CPUMCTX_EXTRN_ES);
2513	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
2514	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
2515	pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
2516	pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
2517	pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
2518	}
2519	pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
2520	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2521	AssertRCReturn(rc, rc);
2522	Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
2523	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2524	pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
2525	pExit->IoPortAccess.AccessInfo.IsWrite ? "OUT" : "IN",
2526	pExit->IoPortAccess.AccessInfo.StringOp ? "S" : "",
2527	pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize));
2528	VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2529	Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2530	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2531	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2532	return rcStrict;
2533	}
2534	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2535
2536
2537	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2538	/**
2539	* Deals with interrupt window message.
2540	*
2541	* @returns Strict VBox status code.
2542	* @param pVM The cross context VM structure.
2543	* @param pVCpu The cross context per CPU structure.
2544	* @param pMsg The message.
2545	* @sa nemR3WinHandleExitInterruptWindow
2546	*/
2547	NEM_TMPL_STATIC VBOXSTRICTRC
2548	nemHCWinHandleMessageInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_INTERRUPT_WINDOW_MESSAGE const *pMsg)
2549	{
2550	/*
2551	* Assert message sanity.
2552	*/
2553	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE
2554	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ // READ & WRITE are probably not used here
2555	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2556	AssertMsg(pMsg->Type == HvX64PendingInterrupt \|\| pMsg->Type == HvX64PendingNmi, ("%#x\n", pMsg->Type));
2557
2558	/*
2559	* Just copy the state we've got and handle it in the loop for now.
2560	*/
2561	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2562	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2563
2564	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2565	Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d\n",
2566	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2567	pMsg->Type, RT_BOOL(pMsg->Header.Rflags & X86_EFL_IF), pMsg->Header.ExecutionState.InterruptShadow));
2568
2569	/** @todo call nemHCWinHandleInterruptFF */
2570	RT_NOREF(pVM);
2571	return VINF_SUCCESS;
2572	}
2573	#elif defined(IN_RING3)
2574	/**
2575	* Deals with interrupt window exits (WHvRunVpExitReasonX64InterruptWindow).
2576	*
2577	* @returns Strict VBox status code.
2578	* @param pVM The cross context VM structure.
2579	* @param pVCpu The cross context per CPU structure.
2580	* @param pExit The VM exit information to handle.
2581	* @sa nemHCWinHandleMessageInterruptWindow
2582	*/
2583	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2584	{
2585	/*
2586	* Assert message sanity.
2587	*/
2588	AssertMsg( pExit->InterruptWindow.DeliverableType == WHvX64PendingInterrupt
2589	\|\| pExit->InterruptWindow.DeliverableType == WHvX64PendingNmi,
2590	("%#x\n", pExit->InterruptWindow.DeliverableType));
2591
2592	/*
2593	* Just copy the state we've got and handle it in the loop for now.
2594	*/
2595	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
2596	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2597
2598	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2599	Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d\n",
2600	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2601	pExit->InterruptWindow.DeliverableType, RT_BOOL(pExit->VpContext.Rflags & X86_EFL_IF),
2602	pExit->VpContext.ExecutionState.InterruptShadow));
2603
2604	/** @todo call nemHCWinHandleInterruptFF */
2605	RT_NOREF(pVM);
2606	return VINF_SUCCESS;
2607	}
2608	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2609
2610
2611	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2612	/**
2613	* Deals with CPUID intercept message.
2614	*
2615	* @returns Strict VBox status code.
2616	* @param pVM The cross context VM structure.
2617	* @param pVCpu The cross context per CPU structure.
2618	* @param pMsg The message.
2619	* @sa nemR3WinHandleExitCpuId
2620	*/
2621	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageCpuId(PVMCC pVM, PVMCPUCC pVCpu, HV_X64_CPUID_INTERCEPT_MESSAGE const *pMsg)
2622	{
2623	/* Check message register value sanity. */
2624	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2625	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2626	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2627	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2628	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
2629	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
2630	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
2631	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbx, pMsg->Rbx);
2632
2633	/* Do exit history. */
2634	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
2635	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2636	if (!pExitRec)
2637	{
2638	/*
2639	* Soak up state and execute the instruction.
2640	*
2641	* Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
2642	* function and make everyone use it.
2643	*/
2644	/** @todo Combine implementations into IEMExecDecodedCpuId as this will
2645	* only get weirder with nested VT-x and AMD-V support. */
2646	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2647
2648	/* Copy in the low register values (top is always cleared). */
2649	pVCpu->cpum.GstCtx.rax = (uint32_t)pMsg->Rax;
2650	pVCpu->cpum.GstCtx.rcx = (uint32_t)pMsg->Rcx;
2651	pVCpu->cpum.GstCtx.rdx = (uint32_t)pMsg->Rdx;
2652	pVCpu->cpum.GstCtx.rbx = (uint32_t)pMsg->Rbx;
2653	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2654
2655	/* Get the correct values. */
2656	CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
2657	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
2658
2659	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
2660	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2661	pMsg->Rax, pMsg->Rcx, pMsg->Rdx, pMsg->Rbx,
2662	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
2663	pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
2664
2665	/* Move RIP and we're done. */
2666	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2667
2668	return VINF_SUCCESS;
2669	}
2670
2671	/*
2672	* Frequent exit or something needing probing.
2673	* Get state and call EMHistoryExec.
2674	*/
2675	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2676	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
2677	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
2678	pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
2679	pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
2680	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2681	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
2682	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2683	pMsg->Rax, pMsg->Rcx, pMsg->Rdx, pMsg->Rbx,
2684	pMsg->DefaultResultRax, pMsg->DefaultResultRcx, pMsg->DefaultResultRdx, pMsg->DefaultResultRbx));
2685	# ifdef IN_RING0
2686	VBOXSTRICTRC rcStrict = nemR0WinImportStateStrict(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM, "CpuIdExit");
2687	if (rcStrict != VINF_SUCCESS)
2688	return rcStrict;
2689	RT_NOREF(pVM);
2690	# else
2691	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2692	AssertRCReturn(rc, rc);
2693	# endif
2694	VBOXSTRICTRC rcStrictExec = EMHistoryExec(pVCpu, pExitRec, 0);
2695	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2696	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2697	VBOXSTRICTRC_VAL(rcStrictExec), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2698	return rcStrictExec;
2699	}
2700	#elif defined(IN_RING3)
2701	/**
2702	* Deals with CPUID exits (WHvRunVpExitReasonX64Cpuid).
2703	*
2704	* @returns Strict VBox status code.
2705	* @param pVM The cross context VM structure.
2706	* @param pVCpu The cross context per CPU structure.
2707	* @param pExit The VM exit information to handle.
2708	* @sa nemHCWinHandleMessageCpuId
2709	*/
2710	NEM_TMPL_STATIC VBOXSTRICTRC
2711	nemR3WinHandleExitCpuId(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2712	{
2713	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
2714	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2715	if (!pExitRec)
2716	{
2717	/*
2718	* Soak up state and execute the instruction.
2719	*
2720	* Note! If this grows slightly more complicated, combine into an IEMExecDecodedCpuId
2721	* function and make everyone use it.
2722	*/
2723	/** @todo Combine implementations into IEMExecDecodedCpuId as this will
2724	* only get weirder with nested VT-x and AMD-V support. */
2725	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2726
2727	/* Copy in the low register values (top is always cleared). */
2728	pVCpu->cpum.GstCtx.rax = (uint32_t)pExit->CpuidAccess.Rax;
2729	pVCpu->cpum.GstCtx.rcx = (uint32_t)pExit->CpuidAccess.Rcx;
2730	pVCpu->cpum.GstCtx.rdx = (uint32_t)pExit->CpuidAccess.Rdx;
2731	pVCpu->cpum.GstCtx.rbx = (uint32_t)pExit->CpuidAccess.Rbx;
2732	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2733
2734	/* Get the correct values. */
2735	CPUMGetGuestCpuId(pVCpu, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx,
2736	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
2737
2738	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
2739	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2740	pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx,
2741	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx,
2742	pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
2743
2744	/* Move RIP and we're done. */
2745	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
2746
2747	RT_NOREF_PV(pVM);
2748	return VINF_SUCCESS;
2749	}
2750
2751	/*
2752	* Frequent exit or something needing probing.
2753	* Get state and call EMHistoryExec.
2754	*/
2755	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2756	pVCpu->cpum.GstCtx.rax = pExit->CpuidAccess.Rax;
2757	pVCpu->cpum.GstCtx.rcx = pExit->CpuidAccess.Rcx;
2758	pVCpu->cpum.GstCtx.rdx = pExit->CpuidAccess.Rdx;
2759	pVCpu->cpum.GstCtx.rbx = pExit->CpuidAccess.Rbx;
2760	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
2761	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
2762	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2763	pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx,
2764	pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
2765	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
2766	AssertRCReturn(rc, rc);
2767	VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2768	Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2769	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2770	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2771	return rcStrict;
2772	}
2773	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
2774
2775
2776	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
2777	/**
2778	* Deals with MSR intercept message.
2779	*
2780	* @returns Strict VBox status code.
2781	* @param pVCpu The cross context per CPU structure.
2782	* @param pMsg The message.
2783	* @sa nemR3WinHandleExitMsr
2784	*/
2785	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleMessageMsr(PVMCPUCC pVCpu, HV_X64_MSR_INTERCEPT_MESSAGE const *pMsg)
2786	{
2787	/*
2788	* A wee bit of sanity first.
2789	*/
2790	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
2791	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE);
2792	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
2793	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
2794	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
2795	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
2796	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
2797	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
2798
2799	/*
2800	* Check CPL as that's common to both RDMSR and WRMSR.
2801	*/
2802	VBOXSTRICTRC rcStrict;
2803	if (pMsg->Header.ExecutionState.Cpl == 0)
2804	{
2805	/*
2806	* Get all the MSR state. Since we're getting EFER, we also need to
2807	* get CR0, CR4 and CR3.
2808	*/
2809	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2810	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
2811	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
2812	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2813	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
2814
2815	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
2816	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
2817	(!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
2818	\| CPUMCTX_EXTRN_ALL_MSRS \| CPUMCTX_EXTRN_CR0
2819	\| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4,
2820	"MSRs");
2821	if (rcStrict == VINF_SUCCESS)
2822	{
2823	if (!pExitRec)
2824	{
2825	/*
2826	* Handle writes.
2827	*/
2828	if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2829	{
2830	rcStrict = CPUMSetGuestMsr(pVCpu, pMsg->MsrNumber, RT_MAKE_U64((uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx));
2831	Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n",
2832	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2833	pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2834	if (rcStrict == VINF_SUCCESS)
2835	{
2836	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2837	return VINF_SUCCESS;
2838	}
2839	# ifndef IN_RING3
2840	/* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
2841	if (rcStrict == VERR_CPUM_RAISE_GP_0)
2842	rcStrict = VINF_CPUM_R3_MSR_WRITE;
2843	return rcStrict;
2844	# else
2845	LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n",
2846	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2847	pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2848	# endif
2849	}
2850	/*
2851	* Handle reads.
2852	*/
2853	else
2854	{
2855	uint64_t uValue = 0;
2856	rcStrict = CPUMQueryGuestMsr(pVCpu, pMsg->MsrNumber, &uValue);
2857	Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
2858	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2859	pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2860	if (rcStrict == VINF_SUCCESS)
2861	{
2862	pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
2863	pVCpu->cpum.GstCtx.rdx = uValue >> 32;
2864	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
2865	nemHCWinAdvanceGuestRipAndClearRF(pVCpu, &pMsg->Header, 2);
2866	return VINF_SUCCESS;
2867	}
2868	# ifndef IN_RING3
2869	/* move to ring-3 and handle the trap/whatever there, as we want to LogRel this. */
2870	if (rcStrict == VERR_CPUM_RAISE_GP_0)
2871	rcStrict = VINF_CPUM_R3_MSR_READ;
2872	return rcStrict;
2873	# else
2874	LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n",
2875	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2876	pMsg->MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2877	# endif
2878	}
2879	}
2880	else
2881	{
2882	/*
2883	* Handle frequent exit or something needing probing.
2884	*/
2885	Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
2886	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2887	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD", pMsg->MsrNumber));
2888	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
2889	Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
2890	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2891	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
2892	return rcStrict;
2893	}
2894	}
2895	else
2896	{
2897	LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
2898	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2899	pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE ? "WR" : "RD",
2900	pMsg->MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
2901	return rcStrict;
2902	}
2903	}
2904	else if (pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE)
2905	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n",
2906	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2907	pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber, (uint32_t)pMsg->Rax, (uint32_t)pMsg->Rdx ));
2908	else
2909	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n",
2910	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
2911	pMsg->Header.ExecutionState.Cpl, pMsg->MsrNumber));
2912
2913	/*
2914	* If we get down here, we're supposed to #GP(0).
2915	*/
2916	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL_MSRS, "MSR");
2917	if (rcStrict == VINF_SUCCESS)
2918	{
2919	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
2920	if (rcStrict == VINF_IEM_RAISED_XCPT)
2921	rcStrict = VINF_SUCCESS;
2922	else if (rcStrict != VINF_SUCCESS)
2923	Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
2924	}
2925	return rcStrict;
2926	}
2927	#elif defined(IN_RING3)
2928	/**
2929	* Deals with MSR access exits (WHvRunVpExitReasonX64MsrAccess).
2930	*
2931	* @returns Strict VBox status code.
2932	* @param pVM The cross context VM structure.
2933	* @param pVCpu The cross context per CPU structure.
2934	* @param pExit The VM exit information to handle.
2935	* @sa nemHCWinHandleMessageMsr
2936	*/
2937	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitMsr(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2938	{
2939	/*
2940	* Check CPL as that's common to both RDMSR and WRMSR.
2941	*/
2942	VBOXSTRICTRC rcStrict;
2943	if (pExit->VpContext.ExecutionState.Cpl == 0)
2944	{
2945	/*
2946	* Get all the MSR state. Since we're getting EFER, we also need to
2947	* get CR0, CR4 and CR3.
2948	*/
2949	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
2950	pExit->MsrAccess.AccessInfo.IsWrite
2951	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
2952	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
2953	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
2954	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2955	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
2956	(!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
2957	\| CPUMCTX_EXTRN_ALL_MSRS \| CPUMCTX_EXTRN_CR0
2958	\| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4,
2959	"MSRs");
2960	if (rcStrict == VINF_SUCCESS)
2961	{
2962	if (!pExitRec)
2963	{
2964	/*
2965	* Handle writes.
2966	*/
2967	if (pExit->MsrAccess.AccessInfo.IsWrite)
2968	{
2969	rcStrict = CPUMSetGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber,
2970	RT_MAKE_U64((uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx));
2971	Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
2972	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
2973	(uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
2974	if (rcStrict == VINF_SUCCESS)
2975	{
2976	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
2977	return VINF_SUCCESS;
2978	}
2979	LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n", pVCpu->idCpu,
2980	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2981	pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx,
2982	VBOXSTRICTRC_VAL(rcStrict) ));
2983	}
2984	/*
2985	* Handle reads.
2986	*/
2987	else
2988	{
2989	uint64_t uValue = 0;
2990	rcStrict = CPUMQueryGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, &uValue);
2991	Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu,
2992	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
2993	pExit->MsrAccess.MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
2994	if (rcStrict == VINF_SUCCESS)
2995	{
2996	pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
2997	pVCpu->cpum.GstCtx.rdx = uValue >> 32;
2998	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
2999	nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
3000	return VINF_SUCCESS;
3001	}
3002	LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3003	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
3004	uValue, VBOXSTRICTRC_VAL(rcStrict) ));
3005	}
3006	}
3007	else
3008	{
3009	/*
3010	* Handle frequent exit or something needing probing.
3011	*/
3012	Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
3013	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3014	pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber));
3015	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
3016	Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
3017	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3018	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
3019	return rcStrict;
3020	}
3021	}
3022	else
3023	{
3024	LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
3025	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3026	pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
3027	return rcStrict;
3028	}
3029	}
3030	else if (pExit->MsrAccess.AccessInfo.IsWrite)
3031	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3032	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
3033	pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx ));
3034	else
3035	Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3036	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
3037	pExit->MsrAccess.MsrNumber));
3038
3039	/*
3040	* If we get down here, we're supposed to #GP(0).
3041	*/
3042	rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL_MSRS, "MSR");
3043	if (rcStrict == VINF_SUCCESS)
3044	{
3045	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
3046	if (rcStrict == VINF_IEM_RAISED_XCPT)
3047	rcStrict = VINF_SUCCESS;
3048	else if (rcStrict != VINF_SUCCESS)
3049	Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
3050	}
3051
3052	RT_NOREF_PV(pVM);
3053	return rcStrict;
3054	}
3055	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3056
3057
3058	/**
3059	* Worker for nemHCWinHandleMessageException & nemR3WinHandleExitException that
3060	* checks if the given opcodes are of interest at all.
3061	*
3062	* @returns true if interesting, false if not.
3063	* @param cbOpcodes Number of opcode bytes available.
3064	* @param pbOpcodes The opcode bytes.
3065	* @param f64BitMode Whether we're in 64-bit mode.
3066	*/
3067	DECLINLINE(bool) nemHcWinIsInterestingUndefinedOpcode(uint8_t cbOpcodes, uint8_t const *pbOpcodes, bool f64BitMode)
3068	{
3069	/*
3070	* Currently only interested in VMCALL and VMMCALL.
3071	*/
3072	while (cbOpcodes >= 3)
3073	{
3074	switch (pbOpcodes[0])
3075	{
3076	case 0x0f:
3077	switch (pbOpcodes[1])
3078	{
3079	case 0x01:
3080	switch (pbOpcodes[2])
3081	{
3082	case 0xc1: /* 0f 01 c1 VMCALL */
3083	return true;
3084	case 0xd9: /* 0f 01 d9 VMMCALL */
3085	return true;
3086	default:
3087	break;
3088	}
3089	break;
3090	}
3091	break;
3092
3093	default:
3094	return false;
3095
3096	/* prefixes */
3097	case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47:
3098	case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
3099	if (!f64BitMode)
3100	return false;
3101	RT_FALL_THRU();
3102	case X86_OP_PRF_CS:
3103	case X86_OP_PRF_SS:
3104	case X86_OP_PRF_DS:
3105	case X86_OP_PRF_ES:
3106	case X86_OP_PRF_FS:
3107	case X86_OP_PRF_GS:
3108	case X86_OP_PRF_SIZE_OP:
3109	case X86_OP_PRF_SIZE_ADDR:
3110	case X86_OP_PRF_LOCK:
3111	case X86_OP_PRF_REPZ:
3112	case X86_OP_PRF_REPNZ:
3113	cbOpcodes--;
3114	pbOpcodes++;
3115	continue;
3116	}
3117	break;
3118	}
3119	return false;
3120	}
3121
3122
3123	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3124	/**
3125	* Copies state included in a exception intercept message.
3126	*
3127	* @param pVCpu The cross context per CPU structure.
3128	* @param pMsg The message.
3129	* @param fClearXcpt Clear pending exception.
3130	*/
3131	DECLINLINE(void)
3132	nemHCWinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg, bool fClearXcpt)
3133	{
3134	nemHCWinCopyStateFromX64Header(pVCpu, &pMsg->Header);
3135	pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_DS
3136	\| (fClearXcpt ? CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT : 0) );
3137	pVCpu->cpum.GstCtx.rax = pMsg->Rax;
3138	pVCpu->cpum.GstCtx.rcx = pMsg->Rcx;
3139	pVCpu->cpum.GstCtx.rdx = pMsg->Rdx;
3140	pVCpu->cpum.GstCtx.rbx = pMsg->Rbx;
3141	pVCpu->cpum.GstCtx.rsp = pMsg->Rsp;
3142	pVCpu->cpum.GstCtx.rbp = pMsg->Rbp;
3143	pVCpu->cpum.GstCtx.rsi = pMsg->Rsi;
3144	pVCpu->cpum.GstCtx.rdi = pMsg->Rdi;
3145	pVCpu->cpum.GstCtx.r8 = pMsg->R8;
3146	pVCpu->cpum.GstCtx.r9 = pMsg->R9;
3147	pVCpu->cpum.GstCtx.r10 = pMsg->R10;
3148	pVCpu->cpum.GstCtx.r11 = pMsg->R11;
3149	pVCpu->cpum.GstCtx.r12 = pMsg->R12;
3150	pVCpu->cpum.GstCtx.r13 = pMsg->R13;
3151	pVCpu->cpum.GstCtx.r14 = pMsg->R14;
3152	pVCpu->cpum.GstCtx.r15 = pMsg->R15;
3153	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pMsg->DsSegment);
3154	NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ss, pMsg->SsSegment);
3155	}
3156	#elif defined(IN_RING3)
3157	/**
3158	* Copies state included in a exception intercept exit.
3159	*
3160	* @param pVCpu The cross context per CPU structure.
3161	* @param pExit The VM exit information.
3162	* @param fClearXcpt Clear pending exception.
3163	*/
3164	DECLINLINE(void) nemR3WinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit, bool fClearXcpt)
3165	{
3166	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3167	if (fClearXcpt)
3168	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
3169	}
3170	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3171
3172
3173	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3174	/**
3175	* Deals with exception intercept message (HvMessageTypeX64ExceptionIntercept).
3176	*
3177	* @returns Strict VBox status code.
3178	* @param pVCpu The cross context per CPU structure.
3179	* @param pMsg The message.
3180	* @sa nemR3WinHandleExitMsr
3181	*/
3182	NEM_TMPL_STATIC VBOXSTRICTRC
3183	nemHCWinHandleMessageException(PVMCPUCC pVCpu, HV_X64_EXCEPTION_INTERCEPT_MESSAGE const *pMsg)
3184	{
3185	/*
3186	* Assert sanity.
3187	*/
3188	Assert( pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_READ
3189	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_WRITE
3190	\|\| pMsg->Header.InterceptAccessType == HV_INTERCEPT_ACCESS_EXECUTE);
3191	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsg->Header.CsSegment);
3192	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsg->Header.Rip);
3193	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsg->Header.Rflags);
3194	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsg->Header.Cr8);
3195	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterDs, pMsg->DsSegment);
3196	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterSs, pMsg->SsSegment);
3197	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRax, pMsg->Rax);
3198	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRcx, pMsg->Rcx);
3199	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdx, pMsg->Rdx);
3200	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbx, pMsg->Rbx);
3201	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsp, pMsg->Rsp);
3202	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRbp, pMsg->Rbp);
3203	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRsi, pMsg->Rsi);
3204	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRdi, pMsg->Rdi);
3205	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR8, pMsg->R8);
3206	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR9, pMsg->R9);
3207	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR10, pMsg->R10);
3208	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR11, pMsg->R11);
3209	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR12, pMsg->R12);
3210	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR13, pMsg->R13);
3211	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR14, pMsg->R14);
3212	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterR15, pMsg->R15);
3213
3214	/*
3215	* Get most of the register state since we'll end up making IEM inject the
3216	* event. The exception isn't normally flaged as a pending event, so duh.
3217	*
3218	* Note! We can optimize this later with event injection.
3219	*/
3220	Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n",
3221	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header),
3222	pMsg->ExceptionVector, pMsg->ErrorCode, pMsg->ExceptionParameter));
3223	nemHCWinCopyStateFromExceptionMessage(pVCpu, pMsg, true /fClearXcpt/);
3224	uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
3225	if (pMsg->ExceptionVector == X86_XCPT_DB)
3226	fWhat \|= CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_DR6;
3227	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
3228	if (rcStrict != VINF_SUCCESS)
3229	return rcStrict;
3230
3231	/*
3232	* Handle the intercept.
3233	*/
3234	TRPMEVENT enmEvtType = TRPM_TRAP;
3235	switch (pMsg->ExceptionVector)
3236	{
3237	/*
3238	* We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
3239	* and need to turn them over to GIM.
3240	*
3241	* Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
3242	* #UD for handling non-native hypercall instructions. (IEM will
3243	* decode both and let the GIM provider decide whether to accept it.)
3244	*/
3245	case X86_XCPT_UD:
3246	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
3247	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
3248	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3249
3250	if (nemHcWinIsInterestingUndefinedOpcode(pMsg->InstructionByteCount, pMsg->InstructionBytes,
3251	pMsg->Header.ExecutionState.EferLma && pMsg->Header.CsSegment.Long ))
3252	{
3253	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pMsg->Header.Rip,
3254	pMsg->InstructionBytes, pMsg->InstructionByteCount);
3255	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
3256	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3257	nemHCWinExecStateToLogStr(&pMsg->Header), VBOXSTRICTRC_VAL(rcStrict) ));
3258	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3259	return rcStrict;
3260	}
3261	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
3262	pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->InstructionByteCount, pMsg->InstructionBytes ));
3263	break;
3264
3265	/*
3266	* Filter debug exceptions.
3267	*/
3268	case X86_XCPT_DB:
3269	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
3270	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
3271	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3272	Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
3273	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header) ));
3274	break;
3275
3276	case X86_XCPT_BP:
3277	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
3278	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
3279	pMsg->Header.Rip + pMsg->Header.CsSegment.Base, ASMReadTSC());
3280	Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pMsg->Header.CsSegment.Selector,
3281	pMsg->Header.Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->Header.InstructionLength));
3282	enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
3283	break;
3284
3285	/* This shouldn't happen. */
3286	default:
3287	AssertLogRelMsgFailedReturn(("ExceptionVector=%#x\n", pMsg->ExceptionVector), VERR_IEM_IPE_6);
3288	}
3289
3290	/*
3291	* Inject it.
3292	*/
3293	rcStrict = IEMInjectTrap(pVCpu, pMsg->ExceptionVector, enmEvtType, pMsg->ErrorCode,
3294	pMsg->ExceptionParameter /??/, pMsg->Header.InstructionLength);
3295	Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
3296	pVCpu->idCpu, pMsg->Header.CsSegment.Selector, pMsg->Header.Rip,
3297	nemHCWinExecStateToLogStr(&pMsg->Header), pMsg->ExceptionVector, VBOXSTRICTRC_VAL(rcStrict) ));
3298	return rcStrict;
3299	}
3300	#elif defined(IN_RING3)
3301	/**
3302	* Deals with MSR access exits (WHvRunVpExitReasonException).
3303	*
3304	* @returns Strict VBox status code.
3305	* @param pVM The cross context VM structure.
3306	* @param pVCpu The cross context per CPU structure.
3307	* @param pExit The VM exit information to handle.
3308	* @sa nemR3WinHandleExitException
3309	*/
3310	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3311	{
3312	/*
3313	* Get most of the register state since we'll end up making IEM inject the
3314	* event. The exception isn't normally flaged as a pending event, so duh.
3315	*
3316	* Note! We can optimize this later with event injection.
3317	*/
3318	Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3319	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType,
3320	pExit->VpException.ErrorCode, pExit->VpException.ExceptionParameter ));
3321	nemR3WinCopyStateFromExceptionMessage(pVCpu, pExit, true /fClearXcpt/);
3322	uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
3323	if (pExit->VpException.ExceptionType == X86_XCPT_DB)
3324	fWhat \|= CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR7 \| CPUMCTX_EXTRN_DR6;
3325	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
3326	if (rcStrict != VINF_SUCCESS)
3327	return rcStrict;
3328
3329	/*
3330	* Handle the intercept.
3331	*/
3332	TRPMEVENT enmEvtType = TRPM_TRAP;
3333	switch (pExit->VpException.ExceptionType)
3334	{
3335	/*
3336	* We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
3337	* and need to turn them over to GIM.
3338	*
3339	* Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
3340	* #UD for handling non-native hypercall instructions. (IEM will
3341	* decode both and let the GIM provider decide whether to accept it.)
3342	*/
3343	case X86_XCPT_UD:
3344	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
3345	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
3346	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3347	if (nemHcWinIsInterestingUndefinedOpcode(pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes,
3348	pExit->VpContext.ExecutionState.EferLma && pExit->VpContext.Cs.Long ))
3349	{
3350	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), pExit->VpContext.Rip,
3351	pExit->VpException.InstructionBytes,
3352	pExit->VpException.InstructionByteCount);
3353	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
3354	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3355	nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
3356	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
3357	return rcStrict;
3358	}
3359
3360	Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu,
3361	pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
3362	pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes ));
3363	break;
3364
3365	/*
3366	* Filter debug exceptions.
3367	*/
3368	case X86_XCPT_DB:
3369	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
3370	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
3371	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3372	Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
3373	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext) ));
3374	break;
3375
3376	case X86_XCPT_BP:
3377	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
3378	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
3379	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3380	Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3381	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.InstructionLength));
3382	enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
3383	break;
3384
3385	/* This shouldn't happen. */
3386	default:
3387	AssertLogRelMsgFailedReturn(("ExceptionType=%#x\n", pExit->VpException.ExceptionType), VERR_IEM_IPE_6);
3388	}
3389
3390	/*
3391	* Inject it.
3392	*/
3393	rcStrict = IEMInjectTrap(pVCpu, pExit->VpException.ExceptionType, enmEvtType, pExit->VpException.ErrorCode,
3394	pExit->VpException.ExceptionParameter /??/, pExit->VpContext.InstructionLength);
3395	Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
3396	pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
3397	nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, VBOXSTRICTRC_VAL(rcStrict) ));
3398
3399	RT_NOREF_PV(pVM);
3400	return rcStrict;
3401	}
3402	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3403
3404
3405	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3406	/**
3407	* Deals with unrecoverable exception (triple fault).
3408	*
3409	* Seen WRMSR 0x201 (IA32_MTRR_PHYSMASK0) writes from grub / debian9 ending up
3410	* here too. So we'll leave it to IEM to decide.
3411	*
3412	* @returns Strict VBox status code.
3413	* @param pVCpu The cross context per CPU structure.
3414	* @param pMsgHdr The message header.
3415	* @sa nemR3WinHandleExitUnrecoverableException
3416	*/
3417	NEM_TMPL_STATIC VBOXSTRICTRC
3418	nemHCWinHandleMessageUnrecoverableException(PVMCPUCC pVCpu, HV_X64_INTERCEPT_MESSAGE_HEADER const *pMsgHdr)
3419	{
3420	/* Check message register value sanity. */
3421	NEMWIN_ASSERT_MSG_REG_SEG( pVCpu, HvX64RegisterCs, pMsgHdr->CsSegment);
3422	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRip, pMsgHdr->Rip);
3423	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterRflags, pMsgHdr->Rflags);
3424	NEMWIN_ASSERT_MSG_REG_VAL64(pVCpu, HvX64RegisterCr8, (uint64_t)pMsgHdr->Cr8);
3425
3426	# if 0
3427	/*
3428	* Just copy the state we've got and handle it in the loop for now.
3429	*/
3430	nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
3431	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n",
3432	pVCpu->idCpu, pMsgHdr->CsSegment.Selector, pMsgHdr->Rip, nemHCWinExecStateToLogStr(&pMsg->Header), pMsgHdr->Rflags));
3433	return VINF_EM_TRIPLE_FAULT;
3434	# else
3435	/*
3436	* Let IEM decide whether this is really it.
3437	*/
3438	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
3439	pMsgHdr->Rip + pMsgHdr->CsSegment.Base, ASMReadTSC());
3440	nemHCWinCopyStateFromX64Header(pVCpu, pMsgHdr);
3441	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL, "TripleExit");
3442	if (rcStrict == VINF_SUCCESS)
3443	{
3444	rcStrict = IEMExecOne(pVCpu);
3445	if (rcStrict == VINF_SUCCESS)
3446	{
3447	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3448	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags ));
3449	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
3450	return VINF_SUCCESS;
3451	}
3452	if (rcStrict == VINF_EM_TRIPLE_FAULT)
3453	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3454	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3455	else
3456	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3457	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3458	}
3459	else
3460	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pMsgHdr->CsSegment.Selector,
3461	pMsgHdr->Rip, nemHCWinExecStateToLogStr(pMsgHdr), pMsgHdr->Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3462	return rcStrict;
3463	# endif
3464	}
3465	#elif defined(IN_RING3)
3466	/**
3467	* Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException).
3468	*
3469	* @returns Strict VBox status code.
3470	* @param pVM The cross context VM structure.
3471	* @param pVCpu The cross context per CPU structure.
3472	* @param pExit The VM exit information to handle.
3473	* @sa nemHCWinHandleMessageUnrecoverableException
3474	*/
3475	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3476	{
3477	# if 0
3478	/*
3479	* Just copy the state we've got and handle it in the loop for now.
3480	*/
3481	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3482	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3483	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
3484	RT_NOREF_PV(pVM);
3485	return VINF_EM_TRIPLE_FAULT;
3486	# else
3487	/*
3488	* Let IEM decide whether this is really it.
3489	*/
3490	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
3491	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3492	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
3493	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM \| CPUMCTX_EXTRN_ALL, "TripleExit");
3494	if (rcStrict == VINF_SUCCESS)
3495	{
3496	rcStrict = IEMExecOne(pVCpu);
3497	if (rcStrict == VINF_SUCCESS)
3498	{
3499	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3500	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
3501	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
3502	return VINF_SUCCESS;
3503	}
3504	if (rcStrict == VINF_EM_TRIPLE_FAULT)
3505	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3506	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3507	else
3508	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3509	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3510	}
3511	else
3512	Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
3513	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
3514	RT_NOREF_PV(pVM);
3515	return rcStrict;
3516	# endif
3517
3518	}
3519	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3520
3521
3522	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3523	/**
3524	* Handles messages (VM exits).
3525	*
3526	* @returns Strict VBox status code.
3527	* @param pVM The cross context VM structure.
3528	* @param pVCpu The cross context per CPU structure.
3529	* @param pMappingHeader The message slot mapping.
3530	* @sa nemR3WinHandleExit
3531	*/
3532	NEM_TMPL_STATIC VBOXSTRICTRC
3533	nemHCWinHandleMessage(PVMCC pVM, PVMCPUCC pVCpu, VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
3534	{
3535	if (pMappingHeader->enmVidMsgType == VidMessageHypervisorMessage)
3536	{
3537	AssertMsg(pMappingHeader->cbMessage == HV_MESSAGE_SIZE, ("%#x\n", pMappingHeader->cbMessage));
3538	HV_MESSAGE const pMsg = (HV_MESSAGE const )(pMappingHeader + 1);
3539	switch (pMsg->Header.MessageType)
3540	{
3541	case HvMessageTypeUnmappedGpa:
3542	Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
3543	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
3544	return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept);
3545
3546	case HvMessageTypeGpaIntercept:
3547	Assert(pMsg->Header.PayloadSize == RT_UOFFSETOF(HV_X64_MEMORY_INTERCEPT_MESSAGE, DsSegment));
3548	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemIntercept);
3549	return nemHCWinHandleMessageMemory(pVM, pVCpu, &pMsg->X64MemoryIntercept);
3550
3551	case HvMessageTypeX64IoPortIntercept:
3552	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64IoPortIntercept));
3553	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
3554	return nemHCWinHandleMessageIoPort(pVM, pVCpu, &pMsg->X64IoPortIntercept);
3555
3556	case HvMessageTypeX64Halt:
3557	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
3558	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
3559	pMsg->X64InterceptHeader.Rip + pMsg->X64InterceptHeader.CsSegment.Base, ASMReadTSC());
3560	Log4(("HaltExit\n"));
3561	return VINF_EM_HALT;
3562
3563	case HvMessageTypeX64InterruptWindow:
3564	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterruptWindow));
3565	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
3566	return nemHCWinHandleMessageInterruptWindow(pVM, pVCpu, &pMsg->X64InterruptWindow);
3567
3568	case HvMessageTypeX64CpuidIntercept:
3569	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64CpuIdIntercept));
3570	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
3571	return nemHCWinHandleMessageCpuId(pVM, pVCpu, &pMsg->X64CpuIdIntercept);
3572
3573	case HvMessageTypeX64MsrIntercept:
3574	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64MsrIntercept));
3575	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
3576	return nemHCWinHandleMessageMsr(pVCpu, &pMsg->X64MsrIntercept);
3577
3578	case HvMessageTypeX64ExceptionIntercept:
3579	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64ExceptionIntercept));
3580	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
3581	return nemHCWinHandleMessageException(pVCpu, &pMsg->X64ExceptionIntercept);
3582
3583	case HvMessageTypeUnrecoverableException:
3584	Assert(pMsg->Header.PayloadSize == sizeof(pMsg->X64InterceptHeader));
3585	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
3586	return nemHCWinHandleMessageUnrecoverableException(pVCpu, &pMsg->X64InterceptHeader);
3587
3588	case HvMessageTypeInvalidVpRegisterValue:
3589	case HvMessageTypeUnsupportedFeature:
3590	case HvMessageTypeTlbPageSizeMismatch:
3591	LogRel(("Unimplemented msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3592	AssertLogRelMsgFailedReturn(("Message type %#x not implemented!\n%.32Rhxd\n", pMsg->Header.MessageType, pMsg),
3593	VERR_NEM_IPE_3);
3594
3595	case HvMessageTypeX64ApicEoi:
3596	case HvMessageTypeX64LegacyFpError:
3597	case HvMessageTypeX64RegisterIntercept:
3598	case HvMessageTypeApicEoi:
3599	case HvMessageTypeFerrAsserted:
3600	case HvMessageTypeEventLogBufferComplete:
3601	case HvMessageTimerExpired:
3602	LogRel(("Unexpected msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3603	AssertLogRelMsgFailedReturn(("Unexpected message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
3604	VERR_NEM_IPE_3);
3605
3606	default:
3607	LogRel(("Unknown msg:\n%.Rhxd\n", (int)sizeof(pMsg), pMsg));
3608	AssertLogRelMsgFailedReturn(("Unknown message on CPU #%u: %#x\n", pVCpu->idCpu, pMsg->Header.MessageType),
3609	VERR_NEM_IPE_3);
3610	}
3611	}
3612	else
3613	AssertLogRelMsgFailedReturn(("Unexpected VID message type on CPU #%u: %#x LB %u\n",
3614	pVCpu->idCpu, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage),
3615	VERR_NEM_IPE_4);
3616	}
3617	#elif defined(IN_RING3)
3618	/**
3619	* Handles VM exits.
3620	*
3621	* @returns Strict VBox status code.
3622	* @param pVM The cross context VM structure.
3623	* @param pVCpu The cross context per CPU structure.
3624	* @param pExit The VM exit information to handle.
3625	* @sa nemHCWinHandleMessage
3626	*/
3627	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
3628	{
3629	switch (pExit->ExitReason)
3630	{
3631	case WHvRunVpExitReasonMemoryAccess:
3632	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
3633	return nemR3WinHandleExitMemory(pVM, pVCpu, pExit);
3634
3635	case WHvRunVpExitReasonX64IoPortAccess:
3636	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
3637	return nemR3WinHandleExitIoPort(pVM, pVCpu, pExit);
3638
3639	case WHvRunVpExitReasonX64Halt:
3640	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
3641	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
3642	pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
3643	Log4(("HaltExit\n"));
3644	return VINF_EM_HALT;
3645
3646	case WHvRunVpExitReasonCanceled:
3647	return VINF_SUCCESS;
3648
3649	case WHvRunVpExitReasonX64InterruptWindow:
3650	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
3651	return nemR3WinHandleExitInterruptWindow(pVM, pVCpu, pExit);
3652
3653	case WHvRunVpExitReasonX64Cpuid:
3654	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
3655	return nemR3WinHandleExitCpuId(pVM, pVCpu, pExit);
3656
3657	case WHvRunVpExitReasonX64MsrAccess:
3658	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
3659	return nemR3WinHandleExitMsr(pVM, pVCpu, pExit);
3660
3661	case WHvRunVpExitReasonException:
3662	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
3663	return nemR3WinHandleExitException(pVM, pVCpu, pExit);
3664
3665	case WHvRunVpExitReasonUnrecoverableException:
3666	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
3667	return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit);
3668
3669	case WHvRunVpExitReasonUnsupportedFeature:
3670	case WHvRunVpExitReasonInvalidVpRegisterValue:
3671	LogRel(("Unimplemented exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
3672	AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n",
3673	pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3);
3674
3675	/* Undesired exits: */
3676	case WHvRunVpExitReasonNone:
3677	default:
3678	LogRel(("Unknown exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
3679	AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3);
3680	}
3681	}
3682	#endif /* IN_RING3 && !NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3683
3684
3685	#ifdef IN_RING0
3686	/**
3687	* Perform an I/O control operation on the partition handle (VID.SYS),
3688	* restarting on alert-like behaviour.
3689	*
3690	* @returns NT status code.
3691	* @param pGVM The ring-0 VM structure.
3692	* @param pGVCpu The global (ring-0) per CPU structure.
3693	* @param fFlags The wait flags.
3694	* @param cMillies The timeout in milliseconds
3695	*/
3696	static NTSTATUS nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(PGVM pGVM, PGVMCPU pGVCpu, uint32_t fFlags, uint32_t cMillies)
3697	{
3698	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
3699	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = fFlags;
3700	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
3701	NTSTATUS rcNt = nemR0NtPerformIoControl(pGVM, pGVCpu, pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
3702	&pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
3703	pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
3704	NULL, 0);
3705	if (rcNt == STATUS_SUCCESS)
3706	{ /* likely */ }
3707	/*
3708	* Generally, if we get down here, we have been interrupted between ACK'ing
3709	* a message and waiting for the next due to a NtAlertThread call. So, we
3710	* should stop ACK'ing the previous message and get on waiting on the next.
3711	* See similar stuff in nemHCWinRunGC().
3712	*/
3713	else if ( rcNt == STATUS_TIMEOUT
3714	\|\| rcNt == STATUS_ALERTED /* just in case */
3715	\|\| rcNt == STATUS_KERNEL_APC /* just in case */
3716	\|\| rcNt == STATUS_USER_APC /* just in case */)
3717	{
3718	DBGFTRACE_CUSTOM(pGVCpu->CTX_SUFF(pVM), "IoCtlMessageSlotHandleAndGetNextRestart/1 %#x (f=%#x)", rcNt, fFlags);
3719	STAM_REL_COUNTER_INC(&pGVCpu->nem.s.StatStopCpuPendingAlerts);
3720	Assert(fFlags & VID_MSHAGN_F_GET_NEXT_MESSAGE);
3721
3722	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pGVCpu->idCpu;
3723	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = fFlags & ~VID_MSHAGN_F_HANDLE_MESSAGE;
3724	pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMillies;
3725	rcNt = nemR0NtPerformIoControl(pGVM, pGVCpu, pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
3726	&pGVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
3727	pGVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
3728	NULL, 0);
3729	DBGFTRACE_CUSTOM(pGVM, "IoCtlMessageSlotHandleAndGetNextRestart/2 %#x", rcNt);
3730	}
3731	return rcNt;
3732	}
3733	#endif /* IN_RING0 */
3734
3735
3736	#ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
3737	/**
3738	* Worker for nemHCWinRunGC that stops the execution on the way out.
3739	*
3740	* The CPU was running the last time we checked, no there are no messages that
3741	* needs being marked handled/whatever. Caller checks this.
3742	*
3743	* @returns rcStrict on success, error status on failure.
3744	* @param pVM The cross context VM structure.
3745	* @param pVCpu The cross context per CPU structure.
3746	* @param rcStrict The nemHCWinRunGC return status. This is a little
3747	* bit unnecessary, except in internal error cases,
3748	* since we won't need to stop the CPU if we took an
3749	* exit.
3750	* @param pMappingHeader The message slot mapping.
3751	*/
3752	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinStopCpu(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict,
3753	VID_MESSAGE_MAPPING_HEADER volatile *pMappingHeader)
3754	{
3755	# ifdef DBGFTRACE_ENABLED
3756	HV_MESSAGE const volatile pMsgForTrace = (HV_MESSAGE const volatile )(pMappingHeader + 1);
3757	# endif
3758
3759	/*
3760	* Try stopping the processor. If we're lucky we manage to do this before it
3761	* does another VM exit.
3762	*/
3763	DBGFTRACE_CUSTOM(pVM, "nemStop#0");
3764	# ifdef IN_RING0
3765	pVCpu->nem.s.uIoCtlBuf.idCpu = pVCpu->idCpu;
3766	NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlStopVirtualProcessor.uFunction,
3767	&pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
3768	NULL, 0);
3769	if (NT_SUCCESS(rcNt))
3770	{
3771	DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay (%#x)", rcNt);
3772	Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
3773	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
3774	return rcStrict;
3775	}
3776	# else
3777	BOOL fRet = VidStopVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu);
3778	if (fRet)
3779	{
3780	DBGFTRACE_CUSTOM(pVM, "nemStop#0: okay");
3781	Log8(("nemHCWinStopCpu: Stopping CPU succeeded (cpu status %u)\n", nemHCWinCpuGetRunningStatus(pVCpu) ));
3782	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuSuccess);
3783	return rcStrict;
3784	}
3785	# endif
3786
3787	/*
3788	* Dang. The CPU stopped by itself and we got a couple of message to deal with.
3789	*/
3790	# ifdef IN_RING0
3791	DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", rcNt);
3792	AssertLogRelMsgReturn(rcNt == ERROR_VID_STOP_PENDING, ("rcNt=%#x\n", rcNt),
3793	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3794	# else
3795	DWORD dwErr = RTNtLastErrorValue();
3796	DBGFTRACE_CUSTOM(pVM, "nemStop#0: pending (%#x)", dwErr);
3797	AssertLogRelMsgReturn(dwErr == ERROR_VID_STOP_PENDING, ("dwErr=%#u (%#x)\n", dwErr, dwErr),
3798	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3799	# endif
3800	Log8(("nemHCWinStopCpu: Stopping CPU #%u pending...\n", pVCpu->idCpu));
3801	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPending);
3802
3803	/*
3804	* First message: Exit or similar, sometimes VidMessageStopRequestComplete.
3805	* Note! We can safely ASSUME that rcStrict isn't an important information one.
3806	*/
3807	# ifdef IN_RING0
3808	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu, VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3809	DBGFTRACE_CUSTOM(pVM, "nemStop#1: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
3810	pMsgForTrace->Header.MessageType);
3811	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3812	("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3813	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3814	# else
3815	BOOL fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
3816	VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3817	DBGFTRACE_CUSTOM(pVM, "nemStop#1: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3818	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3819	AssertLogRelMsgReturn(fWait, ("1st VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
3820	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3821	# endif
3822
3823	VID_MESSAGE_TYPE enmVidMsgType = pMappingHeader->enmVidMsgType;
3824	if (enmVidMsgType != VidMessageStopRequestComplete)
3825	{
3826	VBOXSTRICTRC rcStrict2 = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader);
3827	if (rcStrict2 != VINF_SUCCESS && RT_SUCCESS(rcStrict))
3828	rcStrict = rcStrict2;
3829	DBGFTRACE_CUSTOM(pVM, "nemStop#1: handled %#x -> %d", pMsgForTrace->Header.MessageType, VBOXSTRICTRC_VAL(rcStrict));
3830
3831	/*
3832	* Mark it as handled and get the stop request completed message, then mark
3833	* that as handled too. CPU is back into fully stopped stated then.
3834	*/
3835	# ifdef IN_RING0
3836	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu,
3837	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE,
3838	30000 /ms/);
3839	DBGFTRACE_CUSTOM(pVM, "nemStop#2: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType, pMappingHeader->cbMessage,
3840	pMsgForTrace->Header.MessageType);
3841	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3842	("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3843	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3844	# else
3845	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
3846	VID_MSHAGN_F_HANDLE_MESSAGE \| VID_MSHAGN_F_GET_NEXT_MESSAGE, 30000 /ms/);
3847	DBGFTRACE_CUSTOM(pVM, "nemStop#2: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3848	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3849	AssertLogRelMsgReturn(fWait, ("2nd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
3850	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3851	# endif
3852
3853	/* It should be a stop request completed message. */
3854	enmVidMsgType = pMappingHeader->enmVidMsgType;
3855	AssertLogRelMsgReturn(enmVidMsgType == VidMessageStopRequestComplete,
3856	("Unexpected 2nd message following ERROR_VID_STOP_PENDING: %#x LB %#x\n",
3857	enmVidMsgType, pMappingHeader->cbMessage),
3858	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3859
3860	/*
3861	* Mark the VidMessageStopRequestComplete message as handled.
3862	*/
3863	# ifdef IN_RING0
3864	rcNt = nemR0NtPerformIoCtlMessageSlotHandleAndGetNext(pVM, pVCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /ms/);
3865	DBGFTRACE_CUSTOM(pVM, "nemStop#3: %#x / %#x %#x %#x", rcNt, pMappingHeader->enmVidMsgType,
3866	pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3867	AssertLogRelMsgReturn(rcNt == STATUS_SUCCESS,
3868	("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %#x\n", rcNt),
3869	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3870	# else
3871	fWait = g_pfnVidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu, VID_MSHAGN_F_HANDLE_MESSAGE, 30000 /ms/);
3872	DBGFTRACE_CUSTOM(pVM, "nemStop#3: %d+%#x / %#x %#x %#x", fWait, RTNtLastErrorValue(), pMappingHeader->enmVidMsgType,
3873	pMsgForTrace->Header.MessageType, pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3874	AssertLogRelMsgReturn(fWait, ("3rd VidMessageSlotHandleAndGetNext after ERROR_VID_STOP_PENDING failed: %u\n", RTNtLastErrorValue()),
3875	RT_SUCCESS(rcStrict) ? VERR_NEM_IPE_5 : rcStrict);
3876	# endif
3877	Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict) ));
3878	}
3879	else
3880	{
3881	/** @todo I'm not so sure about this now... */
3882	DBGFTRACE_CUSTOM(pVM, "nemStop#9: %#x %#x %#x", pMappingHeader->enmVidMsgType,
3883	pMappingHeader->cbMessage, pMsgForTrace->Header.MessageType);
3884	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatStopCpuPendingOdd);
3885	Log8(("nemHCWinStopCpu: Stopped the CPU (rcStrict=%Rrc) - 1st VidMessageSlotHandleAndGetNext got VidMessageStopRequestComplete.\n",
3886	VBOXSTRICTRC_VAL(rcStrict) ));
3887	}
3888	return rcStrict;
3889	}
3890	#endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
3891
3892	#if defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3)
3893
3894	/**
3895	* Deals with pending interrupt related force flags, may inject interrupt.
3896	*
3897	* @returns VBox strict status code.
3898	* @param pVM The cross context VM structure.
3899	* @param pVCpu The cross context per CPU structure.
3900	* @param pfInterruptWindows Where to return interrupt window flags.
3901	*/
3902	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleInterruptFF(PVMCC pVM, PVMCPUCC pVCpu, uint8_t *pfInterruptWindows)
3903	{
3904	Assert(!TRPMHasTrap(pVCpu));
3905	RT_NOREF_PV(pVM);
3906
3907	/*
3908	* First update APIC. We ASSUME this won't need TPR/CR8.
3909	*/
3910	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3911	{
3912	APICUpdatePendingInterrupts(pVCpu);
3913	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
3914	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
3915	return VINF_SUCCESS;
3916	}
3917
3918	/*
3919	* We don't currently implement SMIs.
3920	*/
3921	AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
3922
3923	/*
3924	* Check if we've got the minimum of state required for deciding whether we
3925	* can inject interrupts and NMIs. If we don't have it, get all we might require
3926	* for injection via IEM.
3927	*/
3928	bool const fPendingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3929	uint64_t fNeedExtrn = CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
3930	\| (fPendingNmi ? CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI : 0);
3931	if (pVCpu->cpum.GstCtx.fExtrn & fNeedExtrn)
3932	{
3933	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "IntFF");
3934	if (rcStrict != VINF_SUCCESS)
3935	return rcStrict;
3936	}
3937	bool const fInhibitInterrupts = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
3938	&& EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip;
3939
3940	/*
3941	* NMI? Try deliver it first.
3942	*/
3943	if (fPendingNmi)
3944	{
3945	if ( !fInhibitInterrupts
3946	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3947	{
3948	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
3949	if (rcStrict == VINF_SUCCESS)
3950	{
3951	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3952	rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_NMI, TRPM_HARDWARE_INT, 0, 0, 0);
3953	Log8(("Injected NMI on %u (%d)\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
3954	}
3955	return rcStrict;
3956	}
3957	*pfInterruptWindows \|= NEM_WIN_INTW_F_NMI;
3958	Log8(("NMI window pending on %u\n", pVCpu->idCpu));
3959	}
3960
3961	/*
3962	* APIC or PIC interrupt?
3963	*/
3964	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
3965	{
3966	if ( !fInhibitInterrupts
3967	&& pVCpu->cpum.GstCtx.rflags.Bits.u1IF)
3968	{
3969	AssertCompile(NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT & CPUMCTX_EXTRN_APIC_TPR);
3970	VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
3971	if (rcStrict == VINF_SUCCESS)
3972	{
3973	uint8_t bInterrupt;
3974	int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
3975	if (RT_SUCCESS(rc))
3976	{
3977	rcStrict = IEMInjectTrap(pVCpu, bInterrupt, TRPM_HARDWARE_INT, 0, 0, 0);
3978	Log8(("Injected interrupt %#x on %u (%d)\n", bInterrupt, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
3979	}
3980	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3981	{
3982	pfInterruptWindows \|= (bInterrupt >> 4 /??*/) << NEM_WIN_INTW_F_PRIO_SHIFT;
3983	Log8(("VERR_APIC_INTR_MASKED_BY_TPR: pfInterruptWindows=%#x\n", pfInterruptWindows));
3984	}
3985	else
3986	Log8(("PDMGetInterrupt failed -> %d\n", rc));
3987	}
3988	return rcStrict;
3989	}
3990	*pfInterruptWindows \|= NEM_WIN_INTW_F_REGULAR;
3991	Log8(("Interrupt window pending on %u\n", pVCpu->idCpu));
3992	}
3993
3994	return VINF_SUCCESS;
3995	}
3996
3997
3998	/**
3999	* Inner NEM runloop for windows.
4000	*
4001	* @returns Strict VBox status code.
4002	* @param pVM The cross context VM structure.
4003	* @param pVCpu The cross context per CPU structure.
4004	*/
4005	NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVMCC pVM, PVMCPUCC pVCpu)
4006	{
4007	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
4008	# ifdef LOG_ENABLED
4009	if (LogIs3Enabled())
4010	nemHCWinLogState(pVM, pVCpu);
4011	# endif
4012
4013	/*
4014	* Try switch to NEM runloop state.
4015	*/
4016	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
4017	{ /* likely */ }
4018	else
4019	{
4020	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4021	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
4022	return VINF_SUCCESS;
4023	}
4024
4025	/*
4026	* The run loop.
4027	*
4028	* Current approach to state updating to use the sledgehammer and sync
4029	* everything every time. This will be optimized later.
4030	*/
4031	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4032	VID_MESSAGE_MAPPING_HEADER volatile pMappingHeader = (VID_MESSAGE_MAPPING_HEADER volatile )pVCpu->nem.s.pvMsgSlotMapping;
4033	# endif
4034	const bool fSingleStepping = DBGFIsStepping(pVCpu);
4035	// const uint32_t fCheckVmFFs = !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK
4036	// : VM_FF_HP_R0_PRE_HM_STEP_MASK;
4037	// const uint32_t fCheckCpuFFs = !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK;
4038	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4039	for (unsigned iLoop = 0;; iLoop++)
4040	{
4041	# ifndef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4042	/*
4043	* Hack alert!
4044	*/
4045	uint32_t const cMappedPages = pVM->nem.s.cMappedPages;
4046	if (cMappedPages >= 4000)
4047	{
4048	PGMPhysNemEnumPagesByState(pVM, pVCpu, NEM_WIN_PAGE_STATE_READABLE, nemR3WinWHvUnmapOnePageCallback, NULL);
4049	Log(("nemHCWinRunGC: Unmapped all; cMappedPages=%u -> %u\n", cMappedPages, pVM->nem.s.cMappedPages));
4050	}
4051	# endif
4052
4053	/*
4054	* Pending interrupts or such? Need to check and deal with this prior
4055	* to the state syncing.
4056	*/
4057	pVCpu->nem.s.fDesiredInterruptWindows = 0;
4058	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC
4059	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4060	{
4061	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4062	/* Make sure the CPU isn't executing. */
4063	if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
4064	{
4065	pVCpu->nem.s.fHandleAndGetFlags = 0;
4066	rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
4067	if (rcStrict == VINF_SUCCESS)
4068	{ /* likely */ }
4069	else
4070	{
4071	LogFlow(("NEM/%u: breaking: nemHCWinStopCpu -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4072	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4073	break;
4074	}
4075	}
4076	# endif
4077
4078	/* Try inject interrupt. */
4079	rcStrict = nemHCWinHandleInterruptFF(pVM, pVCpu, &pVCpu->nem.s.fDesiredInterruptWindows);
4080	if (rcStrict == VINF_SUCCESS)
4081	{ /* likely */ }
4082	else
4083	{
4084	LogFlow(("NEM/%u: breaking: nemHCWinHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4085	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4086	break;
4087	}
4088	}
4089
4090	/*
4091	* Ensure that hyper-V has the whole state.
4092	* (We always update the interrupt windows settings when active as hyper-V seems
4093	* to forget about it after an exit.)
4094	*/
4095	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK))
4096	!= (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK)
4097	\|\| ( ( pVCpu->nem.s.fDesiredInterruptWindows
4098	\|\| pVCpu->nem.s.fCurrentInterruptWindows != pVCpu->nem.s.fDesiredInterruptWindows)
4099	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4100	&& pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */
4101	# endif
4102	)
4103	)
4104	{
4105	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4106	AssertMsg(pVCpu->nem.s.fHandleAndGetFlags != VID_MSHAGN_F_GET_NEXT_MESSAGE /* not running */,
4107	("%#x fExtrn=%#RX64 (%#RX64) fDesiredInterruptWindows=%d fCurrentInterruptWindows=%#x vs %#x\n",
4108	pVCpu->nem.s.fHandleAndGetFlags, pVCpu->cpum.GstCtx.fExtrn, ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_NEM_WIN_MASK),
4109	pVCpu->nem.s.fDesiredInterruptWindows, pVCpu->nem.s.fCurrentInterruptWindows, pVCpu->nem.s.fDesiredInterruptWindows));
4110	# endif
4111	# ifdef IN_RING0
4112	int rc2 = nemR0WinExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx);
4113	# else
4114	int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu);
4115	# endif
4116	AssertRCReturn(rc2, rc2);
4117	}
4118
4119	/*
4120	* Poll timers and run for a bit.
4121	*
4122	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
4123	* so we take the time of the next timer event and uses that as a deadline.
4124	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
4125	*/
4126	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
4127	* the whole polling job when timers have changed... */
4128	uint64_t offDeltaIgnored;
4129	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
4130	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4131	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4132	{
4133	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4134	if (pVCpu->nem.s.fHandleAndGetFlags)
4135	{ /* Very likely that the CPU does NOT need starting (pending msg, running). */ }
4136	else
4137	{
4138	# ifdef IN_RING0
4139	pVCpu->nem.s.uIoCtlBuf.idCpu = pVCpu->idCpu;
4140	NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlStartVirtualProcessor.uFunction,
4141	&pVCpu->nem.s.uIoCtlBuf.idCpu, sizeof(pVCpu->nem.s.uIoCtlBuf.idCpu),
4142	NULL, 0);
4143	LogFlow(("NEM/%u: IoCtlStartVirtualProcessor -> %#x\n", pVCpu->idCpu, rcNt));
4144	AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("VidStartVirtualProcessor failed for CPU #%u: %#x\n", pVCpu->idCpu, rcNt),
4145	VERR_NEM_IPE_5);
4146	# else
4147	AssertLogRelMsgReturn(g_pfnVidStartVirtualProcessor(pVM->nem.s.hPartitionDevice, pVCpu->idCpu),
4148	("VidStartVirtualProcessor failed for CPU #%u: %u (%#x, rcNt=%#x)\n",
4149	pVCpu->idCpu, RTNtLastErrorValue(), RTNtLastErrorValue(), RTNtLastStatusValue()),
4150	VERR_NEM_IPE_5);
4151	# endif
4152	pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
4153	}
4154	# endif /* NEM_WIN_TEMPLATE_MODE_OWN_RUN_API */
4155
4156	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
4157	{
4158	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4159	uint64_t const nsNow = RTTimeNanoTS();
4160	int64_t const cNsNextTimerEvt = nsNow - nsNextTimerEvt;
4161	uint32_t cMsWait;
4162	if (cNsNextTimerEvt < 100000 /* ns */)
4163	cMsWait = 0;
4164	else if ((uint64_t)cNsNextTimerEvt < RT_NS_1SEC)
4165	{
4166	if ((uint32_t)cNsNextTimerEvt < 2*RT_NS_1MS)
4167	cMsWait = 1;
4168	else
4169	cMsWait = ((uint32_t)cNsNextTimerEvt - 100000 /ns/) / RT_NS_1MS;
4170	}
4171	else
4172	cMsWait = RT_MS_1SEC;
4173	# ifdef IN_RING0
4174	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.iCpu = pVCpu->idCpu;
4175	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.fFlags = pVCpu->nem.s.fHandleAndGetFlags;
4176	pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext.cMillies = cMsWait;
4177	NTSTATUS rcNt = nemR0NtPerformIoControl(pVM, pVCpu, pVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.uFunction,
4178	&pVCpu->nem.s.uIoCtlBuf.MsgSlotHandleAndGetNext,
4179	pVM->nemr0.s.IoCtlMessageSlotHandleAndGetNext.cbInput,
4180	NULL, 0);
4181	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4182	if (rcNt == STATUS_SUCCESS)
4183	# else
4184	BOOL fRet = VidMessageSlotHandleAndGetNext(pVM->nem.s.hPartitionDevice, pVCpu->idCpu,
4185	pVCpu->nem.s.fHandleAndGetFlags, cMsWait);
4186	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4187	if (fRet)
4188	# endif
4189	# else
4190	WHV_RUN_VP_EXIT_CONTEXT ExitReason;
4191	RT_ZERO(ExitReason);
4192	HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
4193	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
4194	if (SUCCEEDED(hrc))
4195	# endif
4196	{
4197	/*
4198	* Deal with the message.
4199	*/
4200	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4201	rcStrict = nemHCWinHandleMessage(pVM, pVCpu, pMappingHeader);
4202	pVCpu->nem.s.fHandleAndGetFlags \|= VID_MSHAGN_F_HANDLE_MESSAGE;
4203	# else
4204	rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason);
4205	# endif
4206	if (rcStrict == VINF_SUCCESS)
4207	{ /* hopefully likely */ }
4208	else
4209	{
4210	LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
4211	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
4212	break;
4213	}
4214	}
4215	else
4216	{
4217	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4218
4219	/* VID.SYS merges STATUS_ALERTED and STATUS_USER_APC into STATUS_TIMEOUT,
4220	so after NtAlertThread we end up here with a STATUS_TIMEOUT. And yeah,
4221	the error code conversion is into WAIT_XXX, i.e. NT status codes. */
4222	# ifndef IN_RING0
4223	DWORD rcNt = GetLastError();
4224	# endif
4225	LogFlow(("NEM/%u: VidMessageSlotHandleAndGetNext -> %#x\n", pVCpu->idCpu, rcNt));
4226	AssertLogRelMsgReturn( rcNt == STATUS_TIMEOUT
4227	\|\| rcNt == STATUS_ALERTED /* just in case */
4228	\|\| rcNt == STATUS_USER_APC /* ditto */
4229	\|\| rcNt == STATUS_KERNEL_APC /* ditto */
4230	, ("VidMessageSlotHandleAndGetNext failed for CPU #%u: %#x (%u)\n",
4231	pVCpu->idCpu, rcNt, rcNt),
4232	VERR_NEM_IPE_0);
4233	pVCpu->nem.s.fHandleAndGetFlags = VID_MSHAGN_F_GET_NEXT_MESSAGE;
4234	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatGetMsgTimeout);
4235	# else
4236	AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n",
4237	pVCpu->idCpu, hrc, GetLastError()),
4238	VERR_NEM_IPE_0);
4239	# endif
4240	}
4241
4242	/*
4243	* If no relevant FFs are pending, loop.
4244	*/
4245	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
4246	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4247	continue;
4248
4249	/** @todo Try handle pending flags, not just return to EM loops. Take care
4250	* not to set important RCs here unless we've handled a message. */
4251	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
4252	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
4253	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
4254	}
4255	else
4256	{
4257	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
4258	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
4259	}
4260	}
4261	else
4262	{
4263	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
4264	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
4265	}
4266	break;
4267	} /* the run loop */
4268
4269
4270	/*
4271	* If the CPU is running, make sure to stop it before we try sync back the
4272	* state and return to EM. We don't sync back the whole state if we can help it.
4273	*/
4274	# ifdef NEM_WIN_TEMPLATE_MODE_OWN_RUN_API
4275	if (pVCpu->nem.s.fHandleAndGetFlags == VID_MSHAGN_F_GET_NEXT_MESSAGE)
4276	{
4277	pVCpu->nem.s.fHandleAndGetFlags = 0;
4278	rcStrict = nemHCWinStopCpu(pVM, pVCpu, rcStrict, pMappingHeader);
4279	}
4280	# endif
4281
4282	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
4283	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
4284
4285	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
4286	{
4287	/* Try anticipate what we might need. */
4288	uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_NMI;
4289	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
4290	\|\| RT_FAILURE(rcStrict))
4291	fImport = CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
4292	# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
4293	else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
4294	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
4295	fImport = CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT;
4296	else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
4297	fImport = CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_NEM_WIN_INHIBIT_INT;
4298	# endif
4299	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
4300	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
4301	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
4302
4303	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
4304	{
4305	# ifdef IN_RING0
4306	int rc2 = nemR0WinImportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, fImport \| CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT,
4307	true /fCanUpdateCr3/);
4308	if (RT_SUCCESS(rc2))
4309	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4310	else if (rc2 == VERR_NEM_FLUSH_TLB)
4311	{
4312	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4313	if (rcStrict == VINF_SUCCESS \|\| rcStrict == -rc2)
4314	rcStrict = -rc2;
4315	else
4316	{
4317	pVCpu->nem.s.rcPending = -rc2;
4318	LogFlow(("NEM/%u: rcPending=%Rrc (rcStrict=%Rrc)\n", pVCpu->idCpu, rc2, VBOXSTRICTRC_VAL(rcStrict) ));
4319	}
4320	}
4321	# else
4322	int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport \| CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
4323	if (RT_SUCCESS(rc2))
4324	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
4325	# endif
4326	else if (RT_SUCCESS(rcStrict))
4327	rcStrict = rc2;
4328	if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL \| (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
4329	pVCpu->cpum.GstCtx.fExtrn = 0;
4330	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
4331	}
4332	else
4333	{
4334	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4335	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
4336	}
4337	}
4338	else
4339	{
4340	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
4341	pVCpu->cpum.GstCtx.fExtrn = 0;
4342	}
4343
4344	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
4345	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
4346	return rcStrict;
4347	}
4348
4349	#endif /* defined(NEM_WIN_TEMPLATE_MODE_OWN_RUN_API) \|\| defined(IN_RING3) */
4350
4351	/**
4352	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
4353	*/
4354	NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
4355	PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
4356	{
4357	/* We'll just unmap the memory. */
4358	if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
4359	{
4360	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4361	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhys);
4362	AssertRC(rc);
4363	if (RT_SUCCESS(rc))
4364	#else
4365	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
4366	if (SUCCEEDED(hrc))
4367	#endif
4368	{
4369	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4370	Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
4371	pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
4372	}
4373	else
4374	{
4375	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4376	LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp rc=%Rrc\n", GCPhys, rc));
4377	return rc;
4378	#else
4379	LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4380	GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4381	return VERR_NEM_IPE_2;
4382	#endif
4383	}
4384	}
4385	RT_NOREF(pVCpu, pvUser);
4386	return VINF_SUCCESS;
4387	}
4388
4389
4390	/**
4391	* Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
4392	*
4393	* @returns The PGMPhysNemQueryPageInfo result.
4394	* @param pVM The cross context VM structure.
4395	* @param pVCpu The cross context virtual CPU structure.
4396	* @param GCPhys The page to unmap.
4397	*/
4398	NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
4399	{
4400	PGMPHYSNEMPAGEINFO Info;
4401	return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /fMakeWritable/, &Info,
4402	nemHCWinUnsetForA20CheckerCallback, NULL);
4403	}
4404
4405
4406	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
4407	{
4408	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
4409	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
4410	}
4411
4412
4413	void nemHCNativeNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
4414	int fRestoreAsRAM, bool fRestoreAsRAM2)
4415	{
4416	Log5(("nemHCNativeNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d fRestoreAsRAM=%d fRestoreAsRAM2=%d\n",
4417	GCPhys, cb, enmKind, fRestoreAsRAM, fRestoreAsRAM2));
4418	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); NOREF(fRestoreAsRAM); NOREF(fRestoreAsRAM2);
4419	}
4420
4421
4422	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
4423	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
4424	{
4425	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
4426	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
4427	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
4428	}
4429
4430
4431	/**
4432	* Worker that maps pages into Hyper-V.
4433	*
4434	* This is used by the PGM physical page notifications as well as the memory
4435	* access VMEXIT handlers.
4436	*
4437	* @returns VBox status code.
4438	* @param pVM The cross context VM structure.
4439	* @param pVCpu The cross context virtual CPU structure of the
4440	* calling EMT.
4441	* @param GCPhysSrc The source page address.
4442	* @param GCPhysDst The hyper-V destination page. This may differ from
4443	* GCPhysSrc when A20 is disabled.
4444	* @param fPageProt NEM_PAGE_PROT_XXX.
4445	* @param pu2State Our page state (input/output).
4446	* @param fBackingChanged Set if the page backing is being changed.
4447	* @thread EMT(pVCpu)
4448	*/
4449	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
4450	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
4451	{
4452	#ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4453	/*
4454	* When using the hypercalls instead of the ring-3 APIs, we don't need to
4455	* unmap memory before modifying it. We still want to track the state though,
4456	* since unmap will fail when called an unmapped page and we don't want to redo
4457	* upgrades/downgrades.
4458	*/
4459	uint8_t const u2OldState = *pu2State;
4460	int rc;
4461	if (fPageProt == NEM_PAGE_PROT_NONE)
4462	{
4463	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
4464	{
4465	rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4466	if (RT_SUCCESS(rc))
4467	{
4468	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4469	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4470	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4471	}
4472	else
4473	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4474	}
4475	else
4476	rc = VINF_SUCCESS;
4477	}
4478	else if (fPageProt & NEM_PAGE_PROT_WRITE)
4479	{
4480	if (u2OldState != NEM_WIN_PAGE_STATE_WRITABLE \|\| fBackingChanged)
4481	{
4482	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4483	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
4484	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4485	if (RT_SUCCESS(rc))
4486	{
4487	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4488	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
4489	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
4490	Log5(("NEM GPA writable/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4491	NOREF(cMappedPages);
4492	}
4493	else
4494	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4495	}
4496	else
4497	rc = VINF_SUCCESS;
4498	}
4499	else
4500	{
4501	if (u2OldState != NEM_WIN_PAGE_STATE_READABLE \|\| fBackingChanged)
4502	{
4503	rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4504	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4505	if (RT_SUCCESS(rc))
4506	{
4507	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4508	uint32_t cMappedPages = u2OldState <= NEM_WIN_PAGE_STATE_UNMAPPED
4509	? ASMAtomicIncU32(&pVM->nem.s.cMappedPages) : pVM->nem.s.cMappedPages;
4510	Log5(("NEM GPA read+exec/set: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4511	NOREF(cMappedPages);
4512	}
4513	else
4514	AssertLogRelMsgFailed(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4515	}
4516	else
4517	rc = VINF_SUCCESS;
4518	}
4519
4520	return VINF_SUCCESS;
4521
4522	#else
4523	/*
4524	* Looks like we need to unmap a page before we can change the backing
4525	* or even modify the protection. This is going to be REALLY efficient.
4526	* PGM lends us two bits to keep track of the state here.
4527	*/
4528	uint8_t const u2OldState = *pu2State;
4529	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
4530	: fPageProt & NEM_PAGE_PROT_READ ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
4531	if ( fBackingChanged
4532	\|\| u2NewState != u2OldState)
4533	{
4534	if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
4535	{
4536	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4537	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4538	AssertRC(rc);
4539	if (RT_SUCCESS(rc))
4540	{
4541	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4542	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4543	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
4544	{
4545	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
4546	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4547	return VINF_SUCCESS;
4548	}
4549	}
4550	else
4551	{
4552	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4553	return rc;
4554	}
4555	# else
4556	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
4557	if (SUCCEEDED(hrc))
4558	{
4559	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4560	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4561	if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
4562	{
4563	Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
4564	GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
4565	return VINF_SUCCESS;
4566	}
4567	}
4568	else
4569	{
4570	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4571	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4572	return VERR_NEM_INIT_FAILED;
4573	}
4574	# endif
4575	}
4576	}
4577
4578	/*
4579	* Writeable mapping?
4580	*/
4581	if (fPageProt & NEM_PAGE_PROT_WRITE)
4582	{
4583	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4584	int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4585	HV_MAP_GPA_READABLE \| HV_MAP_GPA_WRITABLE
4586	\| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4587	AssertRC(rc);
4588	if (RT_SUCCESS(rc))
4589	{
4590	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4591	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4592	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4593	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4594	return VINF_SUCCESS;
4595	}
4596	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4597	return rc;
4598	# else
4599	void *pvPage;
4600	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
4601	if (RT_SUCCESS(rc))
4602	{
4603	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
4604	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute \| WHvMapGpaRangeFlagWrite);
4605	if (SUCCEEDED(hrc))
4606	{
4607	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
4608	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4609	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4610	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4611	return VINF_SUCCESS;
4612	}
4613	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4614	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4615	return VERR_NEM_INIT_FAILED;
4616	}
4617	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
4618	return rc;
4619	# endif
4620	}
4621
4622	if (fPageProt & NEM_PAGE_PROT_READ)
4623	{
4624	# ifdef NEM_WIN_USE_HYPERCALLS_FOR_PAGES
4625	int rc = nemHCWinHypercallMapPage(pVM, pVCpu, GCPhysSrc, GCPhysDst,
4626	HV_MAP_GPA_READABLE \| HV_MAP_GPA_EXECUTABLE \| HV_MAP_GPA_EXECUTABLE_AGAIN);
4627	AssertRC(rc);
4628	if (RT_SUCCESS(rc))
4629	{
4630	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4631	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4632	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4633	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4634	return VINF_SUCCESS;
4635	}
4636	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4637	return rc;
4638	# else
4639	const void *pvPage;
4640	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
4641	if (RT_SUCCESS(rc))
4642	{
4643	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
4644	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
4645	if (SUCCEEDED(hrc))
4646	{
4647	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
4648	uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4649	Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
4650	GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
4651	return VINF_SUCCESS;
4652	}
4653	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
4654	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4655	return VERR_NEM_INIT_FAILED;
4656	}
4657	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
4658	return rc;
4659	# endif
4660	}
4661
4662	/* We already unmapped it above. */
4663	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4664	return VINF_SUCCESS;
4665	#endif /* !NEM_WIN_USE_HYPERCALLS_FOR_PAGES */
4666	}
4667
4668
4669	NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
4670	{
4671	if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
4672	{
4673	Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
4674	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4675	return VINF_SUCCESS;
4676	}
4677
4678	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4679	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4680	int rc = nemHCWinHypercallUnmapPage(pVM, pVCpu, GCPhysDst);
4681	AssertRC(rc);
4682	if (RT_SUCCESS(rc))
4683	{
4684	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4685	Log5(("NEM GPA unmapped/just: %RGp (was %s, cMappedPages=%u)\n", GCPhysDst, g_apszPageStates[*pu2State], cMappedPages));
4686	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4687	return VINF_SUCCESS;
4688	}
4689	LogRel(("nemHCJustUnmapPageFromHyperV/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
4690	return rc;
4691	#else
4692	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
4693	if (SUCCEEDED(hrc))
4694	{
4695	uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
4696	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
4697	Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
4698	return VINF_SUCCESS;
4699	}
4700	LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
4701	GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
4702	return VERR_NEM_IPE_6;
4703	#endif
4704	}
4705
4706
4707	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
4708	PGMPAGETYPE enmType, uint8_t *pu2State)
4709	{
4710	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4711	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4712	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
4713
4714	int rc;
4715	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4716	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4717	if ( pVM->nem.s.fA20Enabled
4718	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4719	rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4720	else
4721	{
4722	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
4723	rc = nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
4724	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys) && RT_SUCCESS(rc))
4725	rc = nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4726
4727	}
4728	#else
4729	RT_NOREF_PV(fPageProt);
4730	if ( pVM->nem.s.fA20Enabled
4731	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4732	rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4733	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4734	rc = nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4735	else
4736	rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
4737	#endif
4738	return rc;
4739	}
4740
4741
4742	void nemHCNativeNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
4743	PGMPAGETYPE enmType, uint8_t *pu2State)
4744	{
4745	Log5(("nemHCNativeNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4746	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
4747	RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
4748
4749	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4750	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4751	if ( pVM->nem.s.fA20Enabled
4752	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4753	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
4754	else
4755	{
4756	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
4757	nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
4758	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4759	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, false /fBackingChanged/);
4760	}
4761	#else
4762	RT_NOREF_PV(fPageProt);
4763	if ( pVM->nem.s.fA20Enabled
4764	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4765	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4766	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4767	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4768	/* else: ignore since we've got the alias page at this address. */
4769	#endif
4770	}
4771
4772
4773	void nemHCNativeNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
4774	uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
4775	{
4776	Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
4777	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
4778	RT_NOREF_PV(HCPhysPrev); RT_NOREF_PV(HCPhysNew); RT_NOREF_PV(enmType);
4779
4780	#if defined(NEM_WIN_USE_HYPERCALLS_FOR_PAGES) \|\| defined(IN_RING0)
4781	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4782	if ( pVM->nem.s.fA20Enabled
4783	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4784	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4785	else
4786	{
4787	/* To keep effort at a minimum, we unmap the HMA page alias and resync it lazily when needed. */
4788	nemHCWinUnmapPageForA20Gate(pVM, pVCpu, GCPhys \| RT_BIT_32(20));
4789	if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4790	nemHCNativeSetPhysPage(pVM, pVCpu, GCPhys, GCPhys, fPageProt, pu2State, true /fBackingChanged/);
4791	}
4792	#else
4793	RT_NOREF_PV(fPageProt);
4794	if ( pVM->nem.s.fA20Enabled
4795	\|\| !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
4796	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4797	else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
4798	nemR3JustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
4799	/* else: ignore since we've got the alias page at this address. */
4800	#endif
4801	}
4802

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

© 2023 Oracle
Contact – Privacy policy – Terms of Use