NEMR3Native-win-armv8.cpp@ 108763

Last change on this file since 108763 was 108763, checked in by vboxsync, 5 weeks ago
VMM/NEMR3Native-win-armv8.cpp: Cleanup the state import export methods to use a table driven style where possible to keep them shorter and more in line with the other ARMv8 NEM backends, bugref:10392
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 127.9 KB

Line
1	/* $Id: NEMR3Native-win-armv8.cpp 108763 2025-03-27 10:22:31Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 Windows backend.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	* Log group 6: Ring-0 memory management
9	* Log group 12: API intercepts.
10	*/
11
12	/*
13	* Copyright (C) 2018-2024 Oracle and/or its affiliates.
14	*
15	* This file is part of VirtualBox base platform packages, as
16	* available from https://www.virtualbox.org.
17	*
18	* This program is free software; you can redistribute it and/or
19	* modify it under the terms of the GNU General Public License
20	* as published by the Free Software Foundation, in version 3 of the
21	* License.
22	*
23	* This program is distributed in the hope that it will be useful, but
24	* WITHOUT ANY WARRANTY; without even the implied warranty of
25	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
26	* General Public License for more details.
27	*
28	* You should have received a copy of the GNU General Public License
29	* along with this program; if not, see <https://www.gnu.org/licenses>.
30	*
31	* SPDX-License-Identifier: GPL-3.0-only
32	*/
33
34
35	/*********************************************************************************************************************************
36	* Header Files *
37	*********************************************************************************************************************************/
38	#define LOG_GROUP LOG_GROUP_NEM
39	#define VMCPU_INCL_CPUM_GST_CTX
40	#include <iprt/nt/nt-and-windows.h>
41	#include <iprt/nt/hyperv.h>
42	#include <WinHvPlatform.h>
43
44	#ifndef _WIN32_WINNT_WIN10
45	# error "Missing _WIN32_WINNT_WIN10"
46	#endif
47	#ifndef _WIN32_WINNT_WIN10_RS1 /* Missing define, causing trouble for us. */
48	# define _WIN32_WINNT_WIN10_RS1 (_WIN32_WINNT_WIN10 + 1)
49	#endif
50	#include <sysinfoapi.h>
51	#include <debugapi.h>
52	#include <errhandlingapi.h>
53	#include <fileapi.h>
54	#include <winerror.h> /* no api header for this. */
55
56	#include <VBox/dis.h>
57	#include <VBox/vmm/nem.h>
58	#include <VBox/vmm/iem.h>
59	#include <VBox/vmm/em.h>
60	#include <VBox/vmm/pdmapic.h>
61	#include <VBox/vmm/pdm.h>
62	#include <VBox/vmm/dbgftrace.h>
63	#include "NEMInternal.h"
64	#include <VBox/vmm/vmcc.h>
65
66	#include <iprt/formats/arm-psci.h>
67
68	#include <iprt/ldr.h>
69	#include <iprt/path.h>
70	#include <iprt/string.h>
71	#include <iprt/system.h>
72	#include <iprt/utf16.h>
73
74	#ifndef NTDDI_WIN10_VB /* Present in W10 2004 SDK, quite possibly earlier. */
75	HRESULT WINAPI WHvQueryGpaRangeDirtyBitmap(WHV_PARTITION_HANDLE, WHV_GUEST_PHYSICAL_ADDRESS, UINT64, UINT64 *, UINT32);
76	# define WHvMapGpaRangeFlagTrackDirtyPages ((WHV_MAP_GPA_RANGE_FLAGS)0x00000008)
77	#endif
78
79	/** Our saved state version for Hyper-V specific things. */
80	#define NEM_HV_SAVED_STATE_VERSION 1
81
82
83	/*
84	* The following definitions appeared in build 27744 allow configuring the base address of the GICv3 controller,
85	* (there is no official SDK for this yet).
86	*/
87	/** @todo Better way of defining these which doesn't require casting later on when calling APIs. */
88	#define WHV_PARTITION_PROPERTY_CODE_ARM64_IC_PARAMETERS UINT32_C(0x00001012)
89	/** No GIC present. */
90	#define WHV_ARM64_IC_EMULATION_MODE_NONE 0
91	/** Hyper-V emulates a GICv3. */
92	#define WHV_ARM64_IC_EMULATION_MODE_GICV3 1
93
94	/**
95	* Configures the interrupt controller emulated by Hyper-V.
96	*/
97	typedef struct MY_WHV_ARM64_IC_PARAMETERS
98	{
99	uint32_t u32EmulationMode;
100	uint32_t u32Rsvd;
101	union
102	{
103	struct
104	{
105	RTGCPHYS GCPhysGicdBase;
106	RTGCPHYS GCPhysGitsTranslaterBase;
107	uint32_t u32Rsvd;
108	uint32_t cLpiIntIdBits;
109	uint32_t u32PpiCntvOverflw;
110	uint32_t u32PpiPmu;
111	uint32_t au32Rsvd[6];
112	} GicV3;
113	} u;
114	} MY_WHV_ARM64_IC_PARAMETERS;
115	AssertCompileSize(MY_WHV_ARM64_IC_PARAMETERS, 64);
116
117
118	/**
119	* The hypercall exit context.
120	*/
121	typedef struct MY_WHV_HYPERCALL_CONTEXT
122	{
123	WHV_INTERCEPT_MESSAGE_HEADER Header;
124	uint16_t Immediate;
125	uint16_t u16Rsvd;
126	uint32_t u32Rsvd;
127	uint64_t X[18];
128	} MY_WHV_HYPERCALL_CONTEXT;
129	typedef MY_WHV_HYPERCALL_CONTEXT *PMY_WHV_HYPERCALL_CONTEXT;
130	AssertCompileSize(MY_WHV_HYPERCALL_CONTEXT, 24 + 19 * sizeof(uint64_t));
131
132
133	/**
134	* The ARM64 reset context.
135	*/
136	typedef struct MY_WHV_ARM64_RESET_CONTEXT
137	{
138	WHV_INTERCEPT_MESSAGE_HEADER Header;
139	uint32_t ResetType;
140	uint32_t u32Rsvd;
141	} MY_WHV_ARM64_RESET_CONTEXT;
142	typedef MY_WHV_ARM64_RESET_CONTEXT *PMY_WHV_ARM64_RESET_CONTEXT;
143	AssertCompileSize(MY_WHV_ARM64_RESET_CONTEXT, 24 + 2 * sizeof(uint32_t));
144
145
146	#define WHV_ARM64_RESET_CONTEXT_TYPE_POWER_OFF 0
147	#define WHV_ARM64_RESET_CONTEXT_TYPE_RESET 1
148
149
150	/**
151	* The exit reason context for arm64, the size is different
152	* from the default SDK we build against.
153	*/
154	typedef struct MY_WHV_RUN_VP_EXIT_CONTEXT
155	{
156	WHV_RUN_VP_EXIT_REASON ExitReason;
157	uint32_t u32Rsvd;
158	uint64_t u64Rsvd;
159	union
160	{
161	WHV_MEMORY_ACCESS_CONTEXT MemoryAccess;
162	WHV_RUN_VP_CANCELED_CONTEXT CancelReason;
163	MY_WHV_HYPERCALL_CONTEXT Hypercall;
164	WHV_UNRECOVERABLE_EXCEPTION_CONTEXT UnrecoverableException;
165	MY_WHV_ARM64_RESET_CONTEXT Arm64Reset;
166	uint64_t au64Rsvd2[32];
167	};
168	} MY_WHV_RUN_VP_EXIT_CONTEXT;
169	typedef MY_WHV_RUN_VP_EXIT_CONTEXT *PMY_WHV_RUN_VP_EXIT_CONTEXT;
170	AssertCompileSize(MY_WHV_RUN_VP_EXIT_CONTEXT, 272);
171
172	#define My_WHvArm64RegisterGicrBaseGpa ((WHV_REGISTER_NAME)UINT32_C(0x00063000))
173	#define My_WHvArm64RegisterActlrEl1 ((WHV_REGISTER_NAME)UINT32_C(0x00040003))
174
175
176	/*********************************************************************************************************************************
177	* Defined Constants And Macros *
178	*********************************************************************************************************************************/
179
180
181	/*********************************************************************************************************************************
182	* Global Variables *
183	*********************************************************************************************************************************/
184	/** @name APIs imported from WinHvPlatform.dll
185	* @{ */
186	static decltype(WHvGetCapability) * g_pfnWHvGetCapability;
187	static decltype(WHvCreatePartition) * g_pfnWHvCreatePartition;
188	static decltype(WHvSetupPartition) * g_pfnWHvSetupPartition;
189	static decltype(WHvDeletePartition) * g_pfnWHvDeletePartition;
190	static decltype(WHvGetPartitionProperty) * g_pfnWHvGetPartitionProperty;
191	static decltype(WHvSetPartitionProperty) * g_pfnWHvSetPartitionProperty;
192	static decltype(WHvMapGpaRange) * g_pfnWHvMapGpaRange;
193	static decltype(WHvUnmapGpaRange) * g_pfnWHvUnmapGpaRange;
194	static decltype(WHvTranslateGva) * g_pfnWHvTranslateGva;
195	static decltype(WHvQueryGpaRangeDirtyBitmap) * g_pfnWHvQueryGpaRangeDirtyBitmap;
196	static decltype(WHvCreateVirtualProcessor) * g_pfnWHvCreateVirtualProcessor;
197	static decltype(WHvDeleteVirtualProcessor) * g_pfnWHvDeleteVirtualProcessor;
198	static decltype(WHvRunVirtualProcessor) * g_pfnWHvRunVirtualProcessor;
199	static decltype(WHvCancelRunVirtualProcessor) * g_pfnWHvCancelRunVirtualProcessor;
200	static decltype(WHvGetVirtualProcessorRegisters) * g_pfnWHvGetVirtualProcessorRegisters;
201	static decltype(WHvSetVirtualProcessorRegisters) * g_pfnWHvSetVirtualProcessorRegisters;
202	static decltype(WHvSuspendPartitionTime) * g_pfnWHvSuspendPartitionTime;
203	static decltype(WHvResumePartitionTime) * g_pfnWHvResumePartitionTime;
204	decltype(WHvGetVirtualProcessorState) * g_pfnWHvGetVirtualProcessorState;
205	decltype(WHvSetVirtualProcessorState) * g_pfnWHvSetVirtualProcessorState;
206	decltype(WHvRequestInterrupt) * g_pfnWHvRequestInterrupt;
207	/** @} */
208
209	/** The Windows build number. */
210	static uint32_t g_uBuildNo = 17134;
211
212
213
214	/**
215	* Import instructions.
216	*/
217	static const struct
218	{
219	uint8_t idxDll; /*< 0 for WinHvPlatform.dll, 1 for vid.dll. /
220	bool fOptional; /*< Set if import is optional. /
221	PFNRT ppfn; /< The function pointer variable. /
222	const char pszName; /< The function name. /
223	} g_aImports[] =
224	{
225	#define NEM_WIN_IMPORT(a_idxDll, a_fOptional, a_Name) { (a_idxDll), (a_fOptional), (PFNRT *)&RT_CONCAT(g_pfn,a_Name), #a_Name }
226	NEM_WIN_IMPORT(0, false, WHvGetCapability),
227	NEM_WIN_IMPORT(0, false, WHvCreatePartition),
228	NEM_WIN_IMPORT(0, false, WHvSetupPartition),
229	NEM_WIN_IMPORT(0, false, WHvDeletePartition),
230	NEM_WIN_IMPORT(0, false, WHvGetPartitionProperty),
231	NEM_WIN_IMPORT(0, false, WHvSetPartitionProperty),
232	NEM_WIN_IMPORT(0, false, WHvMapGpaRange),
233	NEM_WIN_IMPORT(0, false, WHvUnmapGpaRange),
234	NEM_WIN_IMPORT(0, false, WHvTranslateGva),
235	NEM_WIN_IMPORT(0, true, WHvQueryGpaRangeDirtyBitmap),
236	NEM_WIN_IMPORT(0, false, WHvCreateVirtualProcessor),
237	NEM_WIN_IMPORT(0, false, WHvDeleteVirtualProcessor),
238	NEM_WIN_IMPORT(0, false, WHvRunVirtualProcessor),
239	NEM_WIN_IMPORT(0, false, WHvCancelRunVirtualProcessor),
240	NEM_WIN_IMPORT(0, false, WHvGetVirtualProcessorRegisters),
241	NEM_WIN_IMPORT(0, false, WHvSetVirtualProcessorRegisters),
242	NEM_WIN_IMPORT(0, false, WHvSuspendPartitionTime),
243	NEM_WIN_IMPORT(0, false, WHvResumePartitionTime),
244	NEM_WIN_IMPORT(0, false, WHvGetVirtualProcessorState),
245	NEM_WIN_IMPORT(0, false, WHvSetVirtualProcessorState),
246	NEM_WIN_IMPORT(0, false, WHvRequestInterrupt),
247	#undef NEM_WIN_IMPORT
248	};
249
250
251	/*
252	* Let the preprocessor alias the APIs to import variables for better autocompletion.
253	*/
254	#ifndef IN_SLICKEDIT
255	# define WHvGetCapability g_pfnWHvGetCapability
256	# define WHvCreatePartition g_pfnWHvCreatePartition
257	# define WHvSetupPartition g_pfnWHvSetupPartition
258	# define WHvDeletePartition g_pfnWHvDeletePartition
259	# define WHvGetPartitionProperty g_pfnWHvGetPartitionProperty
260	# define WHvSetPartitionProperty g_pfnWHvSetPartitionProperty
261	# define WHvMapGpaRange g_pfnWHvMapGpaRange
262	# define WHvUnmapGpaRange g_pfnWHvUnmapGpaRange
263	# define WHvTranslateGva g_pfnWHvTranslateGva
264	# define WHvQueryGpaRangeDirtyBitmap g_pfnWHvQueryGpaRangeDirtyBitmap
265	# define WHvCreateVirtualProcessor g_pfnWHvCreateVirtualProcessor
266	# define WHvDeleteVirtualProcessor g_pfnWHvDeleteVirtualProcessor
267	# define WHvRunVirtualProcessor g_pfnWHvRunVirtualProcessor
268	# define WHvGetRunExitContextSize g_pfnWHvGetRunExitContextSize
269	# define WHvCancelRunVirtualProcessor g_pfnWHvCancelRunVirtualProcessor
270	# define WHvGetVirtualProcessorRegisters g_pfnWHvGetVirtualProcessorRegisters
271	# define WHvSetVirtualProcessorRegisters g_pfnWHvSetVirtualProcessorRegisters
272	# define WHvSuspendPartitionTime g_pfnWHvSuspendPartitionTime
273	# define WHvResumePartitionTime g_pfnWHvResumePartitionTime
274	# define WHvGetVirtualProcessorState g_pfnWHvGetVirtualProcessorState
275	# define WHvSetVirtualProcessorState g_pfnWHvSetVirtualProcessorState
276	# define WHvRequestInterrupt g_pfnWHvRequestInterrupt
277	#endif
278
279
280	#define WHV_REGNM(a_Suffix) WHvArm64Register ## a_Suffix
281	/** The general registers. */
282	static const struct
283	{
284	WHV_REGISTER_NAME enmWHvReg;
285	uint32_t fCpumExtrn;
286	uintptr_t offCpumCtx;
287	} s_aCpumRegs[] =
288	{
289	#define CPUM_GREG_EMIT_X0_X3(a_Idx) { WHV_REGNM(X ## a_Idx), CPUMCTX_EXTRN_X ## a_Idx, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
290	#define CPUM_GREG_EMIT_X4_X28(a_Idx) { WHV_REGNM(X ## a_Idx), CPUMCTX_EXTRN_X4_X28, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
291	CPUM_GREG_EMIT_X0_X3(0),
292	CPUM_GREG_EMIT_X0_X3(1),
293	CPUM_GREG_EMIT_X0_X3(2),
294	CPUM_GREG_EMIT_X0_X3(3),
295	CPUM_GREG_EMIT_X4_X28(4),
296	CPUM_GREG_EMIT_X4_X28(5),
297	CPUM_GREG_EMIT_X4_X28(6),
298	CPUM_GREG_EMIT_X4_X28(7),
299	CPUM_GREG_EMIT_X4_X28(8),
300	CPUM_GREG_EMIT_X4_X28(9),
301	CPUM_GREG_EMIT_X4_X28(10),
302	CPUM_GREG_EMIT_X4_X28(11),
303	CPUM_GREG_EMIT_X4_X28(12),
304	CPUM_GREG_EMIT_X4_X28(13),
305	CPUM_GREG_EMIT_X4_X28(14),
306	CPUM_GREG_EMIT_X4_X28(15),
307	CPUM_GREG_EMIT_X4_X28(16),
308	CPUM_GREG_EMIT_X4_X28(17),
309	CPUM_GREG_EMIT_X4_X28(18),
310	CPUM_GREG_EMIT_X4_X28(19),
311	CPUM_GREG_EMIT_X4_X28(20),
312	CPUM_GREG_EMIT_X4_X28(21),
313	CPUM_GREG_EMIT_X4_X28(22),
314	CPUM_GREG_EMIT_X4_X28(23),
315	CPUM_GREG_EMIT_X4_X28(24),
316	CPUM_GREG_EMIT_X4_X28(25),
317	CPUM_GREG_EMIT_X4_X28(26),
318	CPUM_GREG_EMIT_X4_X28(27),
319	CPUM_GREG_EMIT_X4_X28(28),
320	{ WHV_REGNM(Fp), CPUMCTX_EXTRN_FP, RT_UOFFSETOF(CPUMCTX, aGRegs[29].x) },
321	{ WHV_REGNM(Lr), CPUMCTX_EXTRN_LR, RT_UOFFSETOF(CPUMCTX, aGRegs[30].x) },
322	{ WHV_REGNM(Pc), CPUMCTX_EXTRN_PC, RT_UOFFSETOF(CPUMCTX, Pc.u64) },
323	{ WHV_REGNM(Fpcr), CPUMCTX_EXTRN_FPCR, RT_UOFFSETOF(CPUMCTX, fpcr) },
324	{ WHV_REGNM(Fpsr), CPUMCTX_EXTRN_FPSR, RT_UOFFSETOF(CPUMCTX, fpsr) }
325	#undef CPUM_GREG_EMIT_X0_X3
326	#undef CPUM_GREG_EMIT_X4_X28
327	};
328	/** SIMD/FP registers. */
329	static const struct
330	{
331	WHV_REGISTER_NAME enmWHvReg;
332	uintptr_t offCpumCtx;
333	} s_aCpumFpRegs[] =
334	{
335	#define CPUM_VREG_EMIT(a_Idx) { WHV_REGNM(Q ## a_Idx), RT_UOFFSETOF(CPUMCTX, aVRegs[a_Idx].v) }
336	CPUM_VREG_EMIT(0),
337	CPUM_VREG_EMIT(1),
338	CPUM_VREG_EMIT(2),
339	CPUM_VREG_EMIT(3),
340	CPUM_VREG_EMIT(4),
341	CPUM_VREG_EMIT(5),
342	CPUM_VREG_EMIT(6),
343	CPUM_VREG_EMIT(7),
344	CPUM_VREG_EMIT(8),
345	CPUM_VREG_EMIT(9),
346	CPUM_VREG_EMIT(10),
347	CPUM_VREG_EMIT(11),
348	CPUM_VREG_EMIT(12),
349	CPUM_VREG_EMIT(13),
350	CPUM_VREG_EMIT(14),
351	CPUM_VREG_EMIT(15),
352	CPUM_VREG_EMIT(16),
353	CPUM_VREG_EMIT(17),
354	CPUM_VREG_EMIT(18),
355	CPUM_VREG_EMIT(19),
356	CPUM_VREG_EMIT(20),
357	CPUM_VREG_EMIT(21),
358	CPUM_VREG_EMIT(22),
359	CPUM_VREG_EMIT(23),
360	CPUM_VREG_EMIT(24),
361	CPUM_VREG_EMIT(25),
362	CPUM_VREG_EMIT(26),
363	CPUM_VREG_EMIT(27),
364	CPUM_VREG_EMIT(28),
365	CPUM_VREG_EMIT(29),
366	CPUM_VREG_EMIT(30),
367	CPUM_VREG_EMIT(31)
368	#undef CPUM_VREG_EMIT
369	};
370	/** PAuth key system registers. */
371	static const struct
372	{
373	WHV_REGISTER_NAME enmWHvReg;
374	uintptr_t offCpumCtx;
375	} s_aCpumPAuthKeyRegs[] =
376	{
377	{ WHV_REGNM(ApdAKeyLoEl1), RT_UOFFSETOF(CPUMCTX, Apda.Low.u64) },
378	{ WHV_REGNM(ApdAKeyHiEl1), RT_UOFFSETOF(CPUMCTX, Apda.High.u64) },
379	{ WHV_REGNM(ApdBKeyLoEl1), RT_UOFFSETOF(CPUMCTX, Apdb.Low.u64) },
380	{ WHV_REGNM(ApdBKeyHiEl1), RT_UOFFSETOF(CPUMCTX, Apdb.High.u64) },
381	{ WHV_REGNM(ApgAKeyLoEl1), RT_UOFFSETOF(CPUMCTX, Apga.Low.u64) },
382	{ WHV_REGNM(ApgAKeyHiEl1), RT_UOFFSETOF(CPUMCTX, Apga.High.u64) },
383	{ WHV_REGNM(ApiAKeyLoEl1), RT_UOFFSETOF(CPUMCTX, Apia.Low.u64) },
384	{ WHV_REGNM(ApiAKeyHiEl1), RT_UOFFSETOF(CPUMCTX, Apia.High.u64) },
385	{ WHV_REGNM(ApiBKeyLoEl1), RT_UOFFSETOF(CPUMCTX, Apib.Low.u64) },
386	{ WHV_REGNM(ApiBKeyHiEl1), RT_UOFFSETOF(CPUMCTX, Apib.High.u64) }
387	};
388	/** System registers. */
389	static const struct
390	{
391	WHV_REGISTER_NAME enmWHvReg;
392	uint32_t fCpumExtrn;
393	uintptr_t offCpumCtx;
394	} s_aCpumSysRegs[] =
395	{
396	{ WHV_REGNM(SpEl0), CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[0].u64) },
397	{ WHV_REGNM(SpEl1), CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[1].u64) },
398	{ WHV_REGNM(SpsrEl1), CPUMCTX_EXTRN_SPSR, RT_UOFFSETOF(CPUMCTX, Spsr.u64) },
399	{ WHV_REGNM(ElrEl1), CPUMCTX_EXTRN_ELR, RT_UOFFSETOF(CPUMCTX, Elr.u64) },
400	{ WHV_REGNM(SctlrEl1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Sctlr.u64) },
401	{ WHV_REGNM(TcrEl1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Tcr.u64) },
402	{ WHV_REGNM(Ttbr0El1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Ttbr0.u64) },
403	{ WHV_REGNM(Ttbr1El1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Ttbr1.u64) },
404	{ WHV_REGNM(VbarEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, VBar.u64) },
405	{ WHV_REGNM(CntkctlEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, CntKCtl.u64) },
406	{ WHV_REGNM(ContextidrEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, ContextIdr.u64) },
407	{ WHV_REGNM(CpacrEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Cpacr.u64) },
408	{ WHV_REGNM(CsselrEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Csselr.u64) },
409	{ WHV_REGNM(EsrEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Esr.u64) },
410	{ WHV_REGNM(FarEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Far.u64) },
411	{ WHV_REGNM(MairEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Mair.u64) },
412	{ WHV_REGNM(ParEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Par.u64) },
413	{ WHV_REGNM(TpidrroEl0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, TpIdrRoEl0.u64) },
414	{ WHV_REGNM(TpidrEl0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[0].u64) },
415	{ WHV_REGNM(TpidrEl1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[1].u64) },
416	{ My_WHvArm64RegisterActlrEl1, CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Actlr.u64) }
417	#if 0 /* Not available in Hyper-V */
418	{ WHV_REGNM(), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr0.u64) },
419	{ WHV_REGNM(), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr1.u64) },
420	{ WHV_REGNM(), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Amair.u64) },
421	{ WHV_REGNM(), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, MDccInt.u64) }
422	#endif
423	};
424
425
426	/*********************************************************************************************************************************
427	* Internal Functions *
428	*********************************************************************************************************************************/
429	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv);
430	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv);
431
432
433	/**
434	* Worker for nemR3NativeInit that probes and load the native API.
435	*
436	* @returns VBox status code.
437	* @param fForced Whether the HMForced flag is set and we should
438	* fail if we cannot initialize.
439	* @param pErrInfo Where to always return error info.
440	*/
441	static int nemR3WinInitProbeAndLoad(bool fForced, PRTERRINFO pErrInfo)
442	{
443	/*
444	* Check that the DLL files we need are present, but without loading them.
445	* We'd like to avoid loading them unnecessarily.
446	*/
447	WCHAR wszPath[MAX_PATH + 64];
448	UINT cwcPath = GetSystemDirectoryW(wszPath, MAX_PATH);
449	if (cwcPath >= MAX_PATH \|\| cwcPath < 2)
450	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "GetSystemDirectoryW failed (%#x / %u)", cwcPath, GetLastError());
451
452	if (wszPath[cwcPath - 1] != '\\' \|\| wszPath[cwcPath - 1] != '/')
453	wszPath[cwcPath++] = '\\';
454	RTUtf16CopyAscii(&wszPath[cwcPath], RT_ELEMENTS(wszPath) - cwcPath, "WinHvPlatform.dll");
455	if (GetFileAttributesW(wszPath) == INVALID_FILE_ATTRIBUTES)
456	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "The native API dll was not found (%ls)", wszPath);
457
458	/*
459	* Check that we're in a VM and that the hypervisor identifies itself as Hyper-V.
460	*/
461	/** @todo */
462
463	/** @todo would be great if we could recognize a root partition from the
464	* CPUID info, but I currently don't dare do that. */
465
466	/*
467	* Now try load the DLLs and resolve the APIs.
468	*/
469	static const char * const s_apszDllNames[1] = { "WinHvPlatform.dll" };
470	RTLDRMOD ahMods[1] = { NIL_RTLDRMOD };
471	int rc = VINF_SUCCESS;
472	for (unsigned i = 0; i < RT_ELEMENTS(s_apszDllNames); i++)
473	{
474	int rc2 = RTLdrLoadSystem(s_apszDllNames[i], true /fNoUnload/, &ahMods[i]);
475	if (RT_FAILURE(rc2))
476	{
477	if (!RTErrInfoIsSet(pErrInfo))
478	RTErrInfoSetF(pErrInfo, rc2, "Failed to load API DLL: %s: %Rrc", s_apszDllNames[i], rc2);
479	else
480	RTErrInfoAddF(pErrInfo, rc2, "; %s: %Rrc", s_apszDllNames[i], rc2);
481	ahMods[i] = NIL_RTLDRMOD;
482	rc = VERR_NEM_INIT_FAILED;
483	}
484	}
485	if (RT_SUCCESS(rc))
486	{
487	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
488	{
489	int rc2 = RTLdrGetSymbol(ahMods[g_aImports[i].idxDll], g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
490	if (RT_SUCCESS(rc2))
491	{
492	if (g_aImports[i].fOptional)
493	LogRel(("NEM: info: Found optional import %s!%s.\n",
494	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName));
495	}
496	else
497	{
498	*g_aImports[i].ppfn = NULL;
499
500	LogRel(("NEM: %s: Failed to import %s!%s: %Rrc",
501	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
502	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName, rc2));
503	if (!g_aImports[i].fOptional)
504	{
505	if (RTErrInfoIsSet(pErrInfo))
506	RTErrInfoAddF(pErrInfo, rc2, ", %s!%s",
507	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
508	else
509	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: %s!%s",
510	s_apszDllNames[g_aImports[i].idxDll], g_aImports[i].pszName);
511	Assert(RT_FAILURE(rc));
512	}
513	}
514	}
515	if (RT_SUCCESS(rc))
516	{
517	Assert(!RTErrInfoIsSet(pErrInfo));
518	}
519	}
520
521	for (unsigned i = 0; i < RT_ELEMENTS(ahMods); i++)
522	RTLdrClose(ahMods[i]);
523	return rc;
524	}
525
526
527	/**
528	* Wrapper for different WHvGetCapability signatures.
529	*/
530	DECLINLINE(HRESULT) WHvGetCapabilityWrapper(WHV_CAPABILITY_CODE enmCap, WHV_CAPABILITY *pOutput, uint32_t cbOutput)
531	{
532	return g_pfnWHvGetCapability(enmCap, pOutput, cbOutput, NULL);
533	}
534
535
536	/**
537	* Worker for nemR3NativeInit that gets the hypervisor capabilities.
538	*
539	* @returns VBox status code.
540	* @param pVM The cross context VM structure.
541	* @param pErrInfo Where to always return error info.
542	*/
543	static int nemR3WinInitCheckCapabilities(PVM pVM, PRTERRINFO pErrInfo)
544	{
545	#define NEM_LOG_REL_CAP_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM: %-38s= " a_szFmt "\n", a_szField, a_Value))
546	#define NEM_LOG_REL_CAP_SUB_EX(a_szField, a_szFmt, a_Value) LogRel(("NEM: %36s: " a_szFmt "\n", a_szField, a_Value))
547	#define NEM_LOG_REL_CAP_SUB(a_szField, a_Value) NEM_LOG_REL_CAP_SUB_EX(a_szField, "%d", a_Value)
548
549	WHV_CAPABILITY Caps;
550	RT_ZERO(Caps);
551	SetLastError(0);
552	HRESULT hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeHypervisorPresent, &Caps, sizeof(Caps));
553	DWORD rcWin = GetLastError();
554	if (FAILED(hrc))
555	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
556	"WHvGetCapability/WHvCapabilityCodeHypervisorPresent failed: %Rhrc (Last=%#x/%u)",
557	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
558	if (!Caps.HypervisorPresent)
559	{
560	if (!RTPathExists(RTPATH_NT_PASSTHRU_PREFIX "Device\\VidExo"))
561	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE,
562	"WHvCapabilityCodeHypervisorPresent is FALSE! Make sure you have enabled the 'Windows Hypervisor Platform' feature.");
563	return RTErrInfoSetF(pErrInfo, VERR_NEM_NOT_AVAILABLE, "WHvCapabilityCodeHypervisorPresent is FALSE! (%u)", rcWin);
564	}
565	LogRel(("NEM: WHvCapabilityCodeHypervisorPresent is TRUE, so this might work...\n"));
566
567
568	/*
569	* Check what extended VM exits are supported.
570	*/
571	RT_ZERO(Caps);
572	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeExtendedVmExits, &Caps, sizeof(Caps));
573	if (FAILED(hrc))
574	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
575	"WHvGetCapability/WHvCapabilityCodeExtendedVmExits failed: %Rhrc (Last=%#x/%u)",
576	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
577	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeExtendedVmExits", "%'#018RX64", Caps.ExtendedVmExits.AsUINT64);
578	pVM->nem.s.fHypercallExit = RT_BOOL(Caps.ExtendedVmExits.HypercallExit);
579	pVM->nem.s.fGpaAccessFaultExit = RT_BOOL(Caps.ExtendedVmExits.GpaAccessFaultExit);
580	NEM_LOG_REL_CAP_SUB("fHypercallExit", pVM->nem.s.fHypercallExit);
581	NEM_LOG_REL_CAP_SUB("fGpaAccessFaultExit", pVM->nem.s.fGpaAccessFaultExit);
582	if (Caps.ExtendedVmExits.AsUINT64 & ~(uint64_t)7)
583	LogRel(("NEM: Warning! Unknown VM exit definitions: %#RX64\n", Caps.ExtendedVmExits.AsUINT64));
584	/** @todo RECHECK: WHV_EXTENDED_VM_EXITS typedef. */
585
586	/*
587	* Check features in case they end up defining any.
588	*/
589	RT_ZERO(Caps);
590	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeFeatures, &Caps, sizeof(Caps));
591	if (FAILED(hrc))
592	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
593	"WHvGetCapability/WHvCapabilityCodeFeatures failed: %Rhrc (Last=%#x/%u)",
594	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
595	if (Caps.Features.AsUINT64 & ~(uint64_t)0)
596	LogRel(("NEM: Warning! Unknown feature definitions: %#RX64\n", Caps.Features.AsUINT64));
597	/** @todo RECHECK: WHV_CAPABILITY_FEATURES typedef. */
598
599	/*
600	* Check that the CPU vendor is supported.
601	*/
602	RT_ZERO(Caps);
603	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorVendor, &Caps, sizeof(Caps));
604	if (FAILED(hrc))
605	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
606	"WHvGetCapability/WHvCapabilityCodeProcessorVendor failed: %Rhrc (Last=%#x/%u)",
607	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
608	switch (Caps.ProcessorVendor)
609	{
610	/** @todo RECHECK: WHV_PROCESSOR_VENDOR typedef. */
611	case WHvProcessorVendorArm:
612	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d - ARM", Caps.ProcessorVendor);
613	pVM->nem.s.enmCpuVendor = CPUMCPUVENDOR_UNKNOWN;
614	break;
615	default:
616	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorVendor", "%d", Caps.ProcessorVendor);
617	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unknown processor vendor: %d", Caps.ProcessorVendor);
618	}
619
620	/*
621	* CPU features, guessing these are virtual CPU features?
622	*/
623	RT_ZERO(Caps);
624	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorFeatures, &Caps, sizeof(Caps));
625	if (FAILED(hrc))
626	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
627	"WHvGetCapability/WHvCapabilityCodeProcessorFeatures failed: %Rhrc (Last=%#x/%u)",
628	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
629	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorFeatures", "%'#018RX64", Caps.ProcessorFeatures.AsUINT64);
630	#define NEM_LOG_REL_CPU_FEATURE(a_Field) NEM_LOG_REL_CAP_SUB(#a_Field, Caps.ProcessorFeatures.a_Field)
631	NEM_LOG_REL_CPU_FEATURE(Asid16);
632	NEM_LOG_REL_CPU_FEATURE(TGran16);
633	NEM_LOG_REL_CPU_FEATURE(TGran64);
634	NEM_LOG_REL_CPU_FEATURE(Haf);
635	NEM_LOG_REL_CPU_FEATURE(Hdbs);
636	NEM_LOG_REL_CPU_FEATURE(Pan);
637	NEM_LOG_REL_CPU_FEATURE(AtS1E1);
638	NEM_LOG_REL_CPU_FEATURE(Uao);
639	NEM_LOG_REL_CPU_FEATURE(El0Aarch32);
640	NEM_LOG_REL_CPU_FEATURE(Fp);
641	NEM_LOG_REL_CPU_FEATURE(FpHp);
642	NEM_LOG_REL_CPU_FEATURE(AdvSimd);
643	NEM_LOG_REL_CPU_FEATURE(AdvSimdHp);
644	NEM_LOG_REL_CPU_FEATURE(GicV3V4);
645	NEM_LOG_REL_CPU_FEATURE(GicV41);
646	NEM_LOG_REL_CPU_FEATURE(Ras);
647	NEM_LOG_REL_CPU_FEATURE(PmuV3);
648	NEM_LOG_REL_CPU_FEATURE(PmuV3ArmV81);
649	NEM_LOG_REL_CPU_FEATURE(PmuV3ArmV84);
650	NEM_LOG_REL_CPU_FEATURE(PmuV3ArmV85);
651	NEM_LOG_REL_CPU_FEATURE(Aes);
652	NEM_LOG_REL_CPU_FEATURE(PolyMul);
653	NEM_LOG_REL_CPU_FEATURE(Sha1);
654	NEM_LOG_REL_CPU_FEATURE(Sha256);
655	NEM_LOG_REL_CPU_FEATURE(Sha512);
656	NEM_LOG_REL_CPU_FEATURE(Crc32);
657	NEM_LOG_REL_CPU_FEATURE(Atomic);
658	NEM_LOG_REL_CPU_FEATURE(Rdm);
659	NEM_LOG_REL_CPU_FEATURE(Sha3);
660	NEM_LOG_REL_CPU_FEATURE(Sm3);
661	NEM_LOG_REL_CPU_FEATURE(Sm4);
662	NEM_LOG_REL_CPU_FEATURE(Dp);
663	NEM_LOG_REL_CPU_FEATURE(Fhm);
664	NEM_LOG_REL_CPU_FEATURE(DcCvap);
665	NEM_LOG_REL_CPU_FEATURE(DcCvadp);
666	NEM_LOG_REL_CPU_FEATURE(ApaBase);
667	NEM_LOG_REL_CPU_FEATURE(ApaEp);
668	NEM_LOG_REL_CPU_FEATURE(ApaEp2);
669	NEM_LOG_REL_CPU_FEATURE(ApaEp2Fp);
670	NEM_LOG_REL_CPU_FEATURE(ApaEp2Fpc);
671	NEM_LOG_REL_CPU_FEATURE(Jscvt);
672	NEM_LOG_REL_CPU_FEATURE(Fcma);
673	NEM_LOG_REL_CPU_FEATURE(RcpcV83);
674	NEM_LOG_REL_CPU_FEATURE(RcpcV84);
675	NEM_LOG_REL_CPU_FEATURE(Gpa);
676	NEM_LOG_REL_CPU_FEATURE(L1ipPipt);
677	NEM_LOG_REL_CPU_FEATURE(DzPermitted);
678
679	#undef NEM_LOG_REL_CPU_FEATURE
680	if (Caps.ProcessorFeatures.AsUINT64 & (~(RT_BIT_64(47) - 1)))
681	LogRel(("NEM: Warning! Unknown CPU features: %#RX64\n", Caps.ProcessorFeatures.AsUINT64));
682	pVM->nem.s.uCpuFeatures.u64 = Caps.ProcessorFeatures.AsUINT64;
683	/** @todo RECHECK: WHV_PROCESSOR_FEATURES typedef. */
684
685	/*
686	* The cache line flush size.
687	*/
688	RT_ZERO(Caps);
689	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodeProcessorClFlushSize, &Caps, sizeof(Caps));
690	if (FAILED(hrc))
691	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
692	"WHvGetCapability/WHvCapabilityCodeProcessorClFlushSize failed: %Rhrc (Last=%#x/%u)",
693	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
694	NEM_LOG_REL_CAP_EX("WHvCapabilityCodeProcessorClFlushSize", "2^%u", Caps.ProcessorClFlushSize);
695	if (Caps.ProcessorClFlushSize < 8 && Caps.ProcessorClFlushSize > 9)
696	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unsupported cache line flush size: %u", Caps.ProcessorClFlushSize);
697	pVM->nem.s.cCacheLineFlushShift = Caps.ProcessorClFlushSize;
698
699	RT_ZERO(Caps);
700	hrc = WHvGetCapabilityWrapper(WHvCapabilityCodePhysicalAddressWidth, &Caps, sizeof(Caps));
701	if (FAILED(hrc))
702	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
703	"WHvGetCapability/WHvCapabilityCodePhysicalAddressWidth failed: %Rhrc (Last=%#x/%u)",
704	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
705	NEM_LOG_REL_CAP_EX("WHvCapabilityCodePhysicalAddressWidth", "2^%u", Caps.PhysicalAddressWidth);
706	if (Caps.PhysicalAddressWidth < 32 && Caps.PhysicalAddressWidth > 52)
707	return RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED, "Unsupported physical address width: %u", Caps.ProcessorClFlushSize);
708	pVM->nem.s.cPhysicalAddressWidth = Caps.PhysicalAddressWidth;
709
710
711	/*
712	* See if they've added more properties that we're not aware of.
713	*/
714	/** @todo RECHECK: WHV_CAPABILITY_CODE typedef. */
715	if (!IsDebuggerPresent()) /* Too noisy when in debugger, so skip. */
716	{
717	static const struct
718	{
719	uint32_t iMin, iMax; } s_aUnknowns[] =
720	{
721	{ 0x0004, 0x000f },
722	{ 0x1003, 0x100f },
723	{ 0x2000, 0x200f },
724	{ 0x3000, 0x300f },
725	{ 0x4000, 0x400f },
726	};
727	for (uint32_t j = 0; j < RT_ELEMENTS(s_aUnknowns); j++)
728	for (uint32_t i = s_aUnknowns[j].iMin; i <= s_aUnknowns[j].iMax; i++)
729	{
730	RT_ZERO(Caps);
731	hrc = WHvGetCapabilityWrapper((WHV_CAPABILITY_CODE)i, &Caps, sizeof(Caps));
732	if (SUCCEEDED(hrc))
733	LogRel(("NEM: Warning! Unknown capability %#x returning: %.*Rhxs\n", i, sizeof(Caps), &Caps));
734	}
735	}
736
737	/*
738	* For proper operation, we require CPUID exits.
739	*/
740	/** @todo Any? */
741
742	#undef NEM_LOG_REL_CAP_EX
743	#undef NEM_LOG_REL_CAP_SUB_EX
744	#undef NEM_LOG_REL_CAP_SUB
745	return VINF_SUCCESS;
746	}
747
748
749	/**
750	* Initializes the GIC controller emulation provided by Hyper-V.
751	*
752	* @returns VBox status code.
753	* @param pVM The cross context VM structure.
754	*
755	* @note Needs to be done early when setting up the partition so this has to live here and not in GICNem-win.cpp
756	*/
757	static int nemR3WinGicCreate(PVM pVM)
758	{
759	PCFGMNODE pGicCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "Devices/gic-nem/0/Config");
760	AssertPtrReturn(pGicCfg, VERR_NEM_IPE_5);
761
762	/*
763	* Query the MMIO ranges.
764	*/
765	RTGCPHYS GCPhysMmioBaseDist = 0;
766	int rc = CFGMR3QueryU64(pGicCfg, "DistributorMmioBase", &GCPhysMmioBaseDist);
767	if (RT_FAILURE(rc))
768	return VMSetError(pVM, rc, RT_SRC_POS,
769	"Configuration error: Failed to get the \"DistributorMmioBase\" value\n");
770
771	RTGCPHYS GCPhysMmioBaseReDist = 0;
772	rc = CFGMR3QueryU64(pGicCfg, "RedistributorMmioBase", &GCPhysMmioBaseReDist);
773	if (RT_FAILURE(rc))
774	return VMSetError(pVM, rc, RT_SRC_POS,
775	"Configuration error: Failed to get the \"RedistributorMmioBase\" value\n");
776
777	RTGCPHYS GCPhysMmioBaseIts = 0;
778	rc = CFGMR3QueryU64(pGicCfg, "ItsMmioBase", &GCPhysMmioBaseIts);
779	if (RT_FAILURE(rc) && rc != VERR_CFGM_VALUE_NOT_FOUND)
780	return VMSetError(pVM, rc, RT_SRC_POS,
781	"Configuration error: Failed to get the \"ItsMmioBase\" value\n");
782	rc = VINF_SUCCESS;
783
784	/*
785	* One can only set the GIC distributor base. The re-distributor regions for the individual
786	* vCPUs are configured when the vCPUs are created, so we need to save the base of the MMIO region.
787	*/
788	pVM->nem.s.GCPhysMmioBaseReDist = GCPhysMmioBaseReDist;
789
790	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
791
792	MY_WHV_ARM64_IC_PARAMETERS Property; RT_ZERO(Property);
793	Property.u32EmulationMode = WHV_ARM64_IC_EMULATION_MODE_GICV3;
794	Property.u.GicV3.GCPhysGicdBase = GCPhysMmioBaseDist;
795	Property.u.GicV3.GCPhysGitsTranslaterBase = GCPhysMmioBaseIts;
796	Property.u.GicV3.cLpiIntIdBits = 1; /** @todo LPIs are currently not supported with our device emulations. */
797	Property.u.GicV3.u32PpiCntvOverflw = pVM->nem.s.u32GicPpiVTimer + 16; /* Calculate the absolute timer INTID. */
798	Property.u.GicV3.u32PpiPmu = 23; /** @todo Configure dynamically (from SBSA, needs a PMU/NEM emulation just like with the GIC probably). */
799	HRESULT hrc = WHvSetPartitionProperty(hPartition, (WHV_PARTITION_PROPERTY_CODE)WHV_PARTITION_PROPERTY_CODE_ARM64_IC_PARAMETERS, &Property, sizeof(Property));
800	if (FAILED(hrc))
801	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
802	"Failed to set WHvPartitionPropertyCodeArm64IcParameters: %Rhrc (Last=%#x/%u)",
803	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
804
805	return rc;
806	}
807
808
809	/**
810	* Creates and sets up a Hyper-V (exo) partition.
811	*
812	* @returns VBox status code.
813	* @param pVM The cross context VM structure.
814	* @param pErrInfo Where to always return error info.
815	*/
816	static int nemR3WinInitCreatePartition(PVM pVM, PRTERRINFO pErrInfo)
817	{
818	AssertReturn(!pVM->nem.s.hPartition, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
819	AssertReturn(!pVM->nem.s.hPartitionDevice, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
820
821	/*
822	* Create the partition.
823	*/
824	WHV_PARTITION_HANDLE hPartition;
825	HRESULT hrc = WHvCreatePartition(&hPartition);
826	if (FAILED(hrc))
827	return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "WHvCreatePartition failed with %Rhrc (Last=%#x/%u)",
828	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
829
830	int rc;
831
832	/*
833	* Set partition properties, most importantly the CPU count.
834	*/
835	/**
836	* @todo Someone at Microsoft please explain another weird API:
837	* - Why this API doesn't take the WHV_PARTITION_PROPERTY_CODE value as an
838	* argument rather than as part of the struct. That is so weird if you've
839	* used any other NT or windows API, including WHvGetCapability().
840	* - Why use PVOID when WHV_PARTITION_PROPERTY is what's expected. We
841	* technically only need 9 bytes for setting/getting
842	* WHVPartitionPropertyCodeProcessorClFlushSize, but the API insists on 16. */
843	WHV_PARTITION_PROPERTY Property;
844	RT_ZERO(Property);
845	Property.ProcessorCount = pVM->cCpus;
846	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorCount, &Property, sizeof(Property));
847	if (SUCCEEDED(hrc))
848	{
849	RT_ZERO(Property);
850	Property.ExtendedVmExits.HypercallExit = pVM->nem.s.fHypercallExit;
851	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeExtendedVmExits, &Property, sizeof(Property));
852	if (SUCCEEDED(hrc))
853	{
854	/*
855	* We'll continue setup in nemR3NativeInitAfterCPUM.
856	*/
857	pVM->nem.s.fCreatedEmts = false;
858	pVM->nem.s.hPartition = hPartition;
859	LogRel(("NEM: Created partition %p.\n", hPartition));
860	return VINF_SUCCESS;
861	}
862
863	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
864	"Failed setting WHvPartitionPropertyCodeExtendedVmExits to %'#RX64: %Rhrc",
865	Property.ExtendedVmExits.AsUINT64, hrc);
866	}
867	else
868	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED,
869	"Failed setting WHvPartitionPropertyCodeProcessorCount to %u: %Rhrc (Last=%#x/%u)",
870	pVM->cCpus, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
871	WHvDeletePartition(hPartition);
872
873	Assert(!pVM->nem.s.hPartitionDevice);
874	Assert(!pVM->nem.s.hPartition);
875	return rc;
876	}
877
878
879	static int nemR3NativeInitSetupVm(PVM pVM)
880	{
881	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
882	AssertReturn(hPartition != NULL, VERR_WRONG_ORDER);
883	AssertReturn(!pVM->nem.s.hPartitionDevice, VERR_WRONG_ORDER);
884	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
885
886	/*
887	* Continue setting up the partition now that we've got most of the CPUID feature stuff.
888	*/
889	WHV_PARTITION_PROPERTY Property;
890	HRESULT hrc;
891
892	/* Not sure if we really need to set the cache line flush size. */
893	RT_ZERO(Property);
894	Property.ProcessorClFlushSize = pVM->nem.s.cCacheLineFlushShift;
895	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorClFlushSize, &Property, sizeof(Property));
896	if (FAILED(hrc))
897	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
898	"Failed to set WHvPartitionPropertyCodeProcessorClFlushSize to %u: %Rhrc (Last=%#x/%u)",
899	pVM->nem.s.cCacheLineFlushShift, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
900
901	/*
902	* Sync CPU features with CPUM.
903	*/
904	/** @todo sync CPU features with CPUM. */
905
906	/* Set the partition property. */
907	RT_ZERO(Property);
908	Property.ProcessorFeatures.AsUINT64 = pVM->nem.s.uCpuFeatures.u64;
909	hrc = WHvSetPartitionProperty(hPartition, WHvPartitionPropertyCodeProcessorFeatures, &Property, sizeof(Property));
910	if (FAILED(hrc))
911	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
912	"Failed to set WHvPartitionPropertyCodeProcessorFeatures to %'#RX64: %Rhrc (Last=%#x/%u)",
913	pVM->nem.s.uCpuFeatures.u64, hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
914
915	/* Configure the GIC. */
916	int rc = nemR3WinGicCreate(pVM);
917	if (RT_FAILURE(rc))
918	return rc;
919
920	/*
921	* Set up the partition.
922	*
923	* Seems like this is where the partition is actually instantiated and we get
924	* a handle to it.
925	*/
926	hrc = WHvSetupPartition(hPartition);
927	if (FAILED(hrc))
928	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
929	"Call to WHvSetupPartition failed: %Rhrc (Last=%#x/%u)",
930	hrc, RTNtLastStatusValue(), RTNtLastErrorValue());
931
932	/*
933	* Setup the EMTs.
934	*/
935	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
936	{
937	hrc = WHvCreateVirtualProcessor(hPartition, idCpu, 0 /fFlags/);
938	if (FAILED(hrc))
939	{
940	NTSTATUS const rcNtLast = RTNtLastStatusValue();
941	DWORD const dwErrLast = RTNtLastErrorValue();
942	while (idCpu-- > 0)
943	{
944	HRESULT hrc2 = WHvDeleteVirtualProcessor(hPartition, idCpu);
945	AssertLogRelMsg(SUCCEEDED(hrc2), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
946	hPartition, idCpu, hrc2, RTNtLastStatusValue(),
947	RTNtLastErrorValue()));
948	}
949	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
950	"Call to WHvCreateVirtualProcessor failed: %Rhrc (Last=%#x/%u)", hrc, rcNtLast, dwErrLast);
951	}
952
953	if (idCpu == 0)
954	{
955	/*
956	* Need to query the ID registers and populate CPUM,
957	* these are partition wide registers and need to be queried/set with WHV_ANY_VP.
958	*/
959	CPUMARMV8IDREGS IdRegs; RT_ZERO(IdRegs);
960
961	WHV_REGISTER_NAME aenmNames[10];
962	WHV_REGISTER_VALUE aValues[10];
963	RT_ZERO(aValues);
964
965	aenmNames[0] = WHvArm64RegisterIdAa64Dfr0El1;
966	aenmNames[1] = WHvArm64RegisterIdAa64Dfr1El1;
967	aenmNames[2] = WHvArm64RegisterIdAa64Isar0El1;
968	aenmNames[3] = WHvArm64RegisterIdAa64Isar1El1;
969	aenmNames[4] = WHvArm64RegisterIdAa64Isar2El1;
970	aenmNames[5] = WHvArm64RegisterIdAa64Mmfr0El1;
971	aenmNames[6] = WHvArm64RegisterIdAa64Mmfr1El1;
972	aenmNames[7] = WHvArm64RegisterIdAa64Mmfr2El1;
973	aenmNames[8] = WHvArm64RegisterIdAa64Pfr0El1;
974	aenmNames[9] = WHvArm64RegisterIdAa64Pfr1El1;
975
976	hrc = WHvGetVirtualProcessorRegisters(hPartition, WHV_ANY_VP /idCpu/, aenmNames, RT_ELEMENTS(aenmNames), aValues);
977	AssertLogRelMsgReturn(SUCCEEDED(hrc),
978	("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
979	hPartition, WHV_ANY_VP, RT_ELEMENTS(aenmNames), hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
980	, VERR_NEM_GET_REGISTERS_FAILED);
981
982	IdRegs.u64RegIdAa64Pfr0El1 = aValues[8].Reg64;
983	IdRegs.u64RegIdAa64Pfr1El1 = aValues[9].Reg64;
984	IdRegs.u64RegIdAa64Dfr0El1 = aValues[0].Reg64;
985	IdRegs.u64RegIdAa64Dfr1El1 = aValues[1].Reg64;
986	IdRegs.u64RegIdAa64Isar0El1 = aValues[2].Reg64;
987	IdRegs.u64RegIdAa64Isar1El1 = aValues[3].Reg64;
988	IdRegs.u64RegIdAa64Isar2El1 = aValues[4].Reg64;
989	IdRegs.u64RegIdAa64Mmfr0El1 = aValues[5].Reg64;
990	IdRegs.u64RegIdAa64Mmfr1El1 = aValues[6].Reg64;
991	IdRegs.u64RegIdAa64Mmfr2El1 = aValues[7].Reg64;
992
993	rc = CPUMR3PopulateFeaturesByIdRegisters(pVM, &IdRegs);
994	if (RT_FAILURE(rc))
995	return rc;
996
997	/* Apply any overrides to the partition. */
998	PCCPUMARMV8IDREGS pIdRegsGst = NULL;
999	rc = CPUMR3QueryGuestIdRegs(pVM, &pIdRegsGst);
1000	AssertRCReturn(rc, rc);
1001
1002	aValues[0].Reg64 = pIdRegsGst->u64RegIdAa64Dfr0El1;
1003	aValues[1].Reg64 = pIdRegsGst->u64RegIdAa64Dfr1El1;
1004	aValues[2].Reg64 = pIdRegsGst->u64RegIdAa64Isar0El1;
1005	aValues[3].Reg64 = pIdRegsGst->u64RegIdAa64Isar1El1;
1006	aValues[4].Reg64 = pIdRegsGst->u64RegIdAa64Isar2El1;
1007	aValues[5].Reg64 = pIdRegsGst->u64RegIdAa64Mmfr0El1;
1008	aValues[6].Reg64 = pIdRegsGst->u64RegIdAa64Mmfr1El1;
1009	aValues[7].Reg64 = pIdRegsGst->u64RegIdAa64Mmfr2El1;
1010	aValues[8].Reg64 = pIdRegsGst->u64RegIdAa64Pfr0El1;
1011	aValues[9].Reg64 = pIdRegsGst->u64RegIdAa64Pfr1El1;
1012
1013	hrc = WHvSetVirtualProcessorRegisters(hPartition, WHV_ANY_VP /idCpu/, aenmNames, RT_ELEMENTS(aenmNames), aValues);
1014	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1015	("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
1016	hPartition, WHV_ANY_VP, RT_ELEMENTS(aenmNames), hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1017	, VERR_NEM_SET_REGISTERS_FAILED);
1018
1019	/* Save the amount of break-/watchpoints supported for syncing the guest register state later. */
1020	pVM->nem.s.cBreakpoints = RT_BF_GET(pIdRegsGst->u64RegIdAa64Dfr0El1, ARMV8_ID_AA64DFR0_EL1_BRPS) + 1;
1021	pVM->nem.s.cWatchpoints = RT_BF_GET(pIdRegsGst->u64RegIdAa64Dfr0El1, ARMV8_ID_AA64DFR0_EL1_WRPS) + 1;
1022	}
1023
1024	/* Configure the GIC re-distributor region for the GIC. */
1025	WHV_REGISTER_NAME enmName = My_WHvArm64RegisterGicrBaseGpa;
1026	WHV_REGISTER_VALUE Value;
1027	Value.Reg64 = pVM->nem.s.GCPhysMmioBaseReDist + idCpu * _128K;
1028
1029	hrc = WHvSetVirtualProcessorRegisters(hPartition, idCpu, &enmName, 1, &Value);
1030	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1031	("WHvSetVirtualProcessorRegisters(%p, %u, WHvArm64RegisterGicrBaseGpa,) -> %Rhrc (Last=%#x/%u)\n",
1032	hPartition, idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1033	, VERR_NEM_SET_REGISTERS_FAILED);
1034	}
1035
1036	pVM->nem.s.fCreatedEmts = true;
1037
1038	LogRel(("NEM: Successfully set up partition\n"));
1039	return VINF_SUCCESS;
1040	}
1041
1042
1043	/**
1044	* Try initialize the native API.
1045	*
1046	* This may only do part of the job, more can be done in
1047	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
1048	*
1049	* @returns VBox status code.
1050	* @param pVM The cross context VM structure.
1051	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
1052	* the latter we'll fail if we cannot initialize.
1053	* @param fForced Whether the HMForced flag is set and we should
1054	* fail if we cannot initialize.
1055	*/
1056	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
1057	{
1058	g_uBuildNo = RTSystemGetNtBuildNo();
1059
1060	/*
1061	* Error state.
1062	* The error message will be non-empty on failure and 'rc' will be set too.
1063	*/
1064	RTERRINFOSTATIC ErrInfo;
1065	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
1066	int rc = nemR3WinInitProbeAndLoad(fForced, pErrInfo);
1067	if (RT_SUCCESS(rc))
1068	{
1069	/*
1070	* Check the capabilties of the hypervisor, starting with whether it's present.
1071	*/
1072	rc = nemR3WinInitCheckCapabilities(pVM, pErrInfo);
1073	if (RT_SUCCESS(rc))
1074	{
1075	/*
1076	* Create and initialize a partition.
1077	*/
1078	rc = nemR3WinInitCreatePartition(pVM, pErrInfo);
1079	if (RT_SUCCESS(rc))
1080	{
1081	rc = nemR3NativeInitSetupVm(pVM);
1082	if (RT_SUCCESS(rc))
1083	{
1084	/*
1085	* Set ourselves as the execution engine and make config adjustments.
1086	*/
1087	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
1088	Log(("NEM: Marked active!\n"));
1089	PGMR3EnableNemMode(pVM);
1090
1091	/*
1092	* Register release statistics
1093	*/
1094	STAMR3Register(pVM, (void *)&pVM->nem.s.cMappedPages, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1095	"/NEM/PagesCurrentlyMapped", STAMUNIT_PAGES, "Number guest pages currently mapped by the VM");
1096	STAMR3Register(pVM, (void *)&pVM->nem.s.StatMapPage, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1097	"/NEM/PagesMapCalls", STAMUNIT_PAGES, "Calls to WHvMapGpaRange/HvCallMapGpaPages");
1098	STAMR3Register(pVM, (void *)&pVM->nem.s.StatMapPageFailed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1099	"/NEM/PagesMapFails", STAMUNIT_PAGES, "Calls to WHvMapGpaRange/HvCallMapGpaPages that failed");
1100	STAMR3Register(pVM, (void *)&pVM->nem.s.StatUnmapPage, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1101	"/NEM/PagesUnmapCalls", STAMUNIT_PAGES, "Calls to WHvUnmapGpaRange/HvCallUnmapGpaPages");
1102	STAMR3Register(pVM, (void *)&pVM->nem.s.StatUnmapPageFailed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1103	"/NEM/PagesUnmapFails", STAMUNIT_PAGES, "Calls to WHvUnmapGpaRange/HvCallUnmapGpaPages that failed");
1104	STAMR3Register(pVM, &pVM->nem.s.StatProfMapGpaRange, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS,
1105	"/NEM/PagesMapGpaRange", STAMUNIT_TICKS_PER_CALL, "Profiling calls to WHvMapGpaRange for bigger stuff");
1106	STAMR3Register(pVM, &pVM->nem.s.StatProfUnmapGpaRange, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS,
1107	"/NEM/PagesUnmapGpaRange", STAMUNIT_TICKS_PER_CALL, "Profiling calls to WHvUnmapGpaRange for bigger stuff");
1108	STAMR3Register(pVM, &pVM->nem.s.StatProfMapGpaRangePage, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS,
1109	"/NEM/PagesMapGpaRangePage", STAMUNIT_TICKS_PER_CALL, "Profiling calls to WHvMapGpaRange for single pages");
1110	STAMR3Register(pVM, &pVM->nem.s.StatProfUnmapGpaRangePage, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS,
1111	"/NEM/PagesUnmapGpaRangePage", STAMUNIT_TICKS_PER_CALL, "Profiling calls to WHvUnmapGpaRange for single pages");
1112
1113	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1114	{
1115	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
1116	STAMR3RegisterF(pVM, &pNemCpu->StatExitPortIo, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of port I/O exits", "/NEM/CPU%u/ExitPortIo", idCpu);
1117	STAMR3RegisterF(pVM, &pNemCpu->StatExitMemUnmapped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unmapped memory exits", "/NEM/CPU%u/ExitMemUnmapped", idCpu);
1118	STAMR3RegisterF(pVM, &pNemCpu->StatExitMemIntercept, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of intercepted memory exits", "/NEM/CPU%u/ExitMemIntercept", idCpu);
1119	STAMR3RegisterF(pVM, &pNemCpu->StatExitHalt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of HLT exits", "/NEM/CPU%u/ExitHalt", idCpu);
1120	STAMR3RegisterF(pVM, &pNemCpu->StatExitInterruptWindow, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of interrupt window exits", "/NEM/CPU%u/ExitInterruptWindow", idCpu);
1121	STAMR3RegisterF(pVM, &pNemCpu->StatExitCpuId, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of CPUID exits", "/NEM/CPU%u/ExitCpuId", idCpu);
1122	STAMR3RegisterF(pVM, &pNemCpu->StatExitMsr, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of MSR access exits", "/NEM/CPU%u/ExitMsr", idCpu);
1123	STAMR3RegisterF(pVM, &pNemCpu->StatExitException, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of exception exits", "/NEM/CPU%u/ExitException", idCpu);
1124	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionBp, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #BP exits", "/NEM/CPU%u/ExitExceptionBp", idCpu);
1125	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionDb, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #DB exits", "/NEM/CPU%u/ExitExceptionDb", idCpu);
1126	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionGp, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #GP exits", "/NEM/CPU%u/ExitExceptionGp", idCpu);
1127	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionGpMesa, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #GP exits from mesa driver", "/NEM/CPU%u/ExitExceptionGpMesa", idCpu);
1128	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUd, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of #UD exits", "/NEM/CPU%u/ExitExceptionUd", idCpu);
1129	STAMR3RegisterF(pVM, &pNemCpu->StatExitExceptionUdHandled, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of handled #UD exits", "/NEM/CPU%u/ExitExceptionUdHandled", idCpu);
1130	STAMR3RegisterF(pVM, &pNemCpu->StatExitUnrecoverable, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of unrecoverable exits", "/NEM/CPU%u/ExitUnrecoverable", idCpu);
1131	STAMR3RegisterF(pVM, &pNemCpu->StatGetMsgTimeout, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of get message timeouts/alerts", "/NEM/CPU%u/GetMsgTimeout", idCpu);
1132	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuSuccess, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of successful CPU stops", "/NEM/CPU%u/StopCpuSuccess", idCpu);
1133	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPending, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stops", "/NEM/CPU%u/StopCpuPending", idCpu);
1134	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingAlerts,STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pending CPU stop alerts", "/NEM/CPU%u/StopCpuPendingAlerts", idCpu);
1135	STAMR3RegisterF(pVM, &pNemCpu->StatStopCpuPendingOdd, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of odd pending CPU stops (see code)", "/NEM/CPU%u/StopCpuPendingOdd", idCpu);
1136	STAMR3RegisterF(pVM, &pNemCpu->StatCancelChangedState, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel changed state", "/NEM/CPU%u/CancelChangedState", idCpu);
1137	STAMR3RegisterF(pVM, &pNemCpu->StatCancelAlertedThread, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel alerted EMT", "/NEM/CPU%u/CancelAlertedEMT", idCpu);
1138	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPre, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of pre execution FF breaks", "/NEM/CPU%u/BreakOnFFPre", idCpu);
1139	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnFFPost, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of post execution FF breaks", "/NEM/CPU%u/BreakOnFFPost", idCpu);
1140	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnCancel, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of cancel execution breaks", "/NEM/CPU%u/BreakOnCancel", idCpu);
1141	STAMR3RegisterF(pVM, &pNemCpu->StatBreakOnStatus, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of status code breaks", "/NEM/CPU%u/BreakOnStatus", idCpu);
1142	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnDemand, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of on-demand state imports", "/NEM/CPU%u/ImportOnDemand", idCpu);
1143	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of state imports on loop return", "/NEM/CPU%u/ImportOnReturn", idCpu);
1144	STAMR3RegisterF(pVM, &pNemCpu->StatImportOnReturnSkipped, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of skipped state imports on loop return", "/NEM/CPU%u/ImportOnReturnSkipped", idCpu);
1145	STAMR3RegisterF(pVM, &pNemCpu->StatQueryCpuTick, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of TSC queries", "/NEM/CPU%u/QueryCpuTick", idCpu);
1146	}
1147
1148	#if defined(VBOX_WITH_R0_MODULES) && !defined(VBOX_WITH_MINIMAL_R0)
1149	if (!SUPR3IsDriverless())
1150	{
1151	PUVM pUVM = pVM->pUVM;
1152	STAMR3RegisterRefresh(pUVM, &pVM->nem.s.R0Stats.cPagesAvailable, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,
1153	STAMUNIT_PAGES, STAM_REFRESH_GRP_NEM, "Free pages available to the hypervisor",
1154	"/NEM/R0Stats/cPagesAvailable");
1155	STAMR3RegisterRefresh(pUVM, &pVM->nem.s.R0Stats.cPagesInUse, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,
1156	STAMUNIT_PAGES, STAM_REFRESH_GRP_NEM, "Pages in use by hypervisor",
1157	"/NEM/R0Stats/cPagesInUse");
1158	}
1159	#endif /* VBOX_WITH_R0_MODULES && !VBOX_WITH_MINIMAL_R0 */
1160	}
1161	}
1162	}
1163	}
1164
1165	/*
1166	* We only fail if in forced mode, otherwise just log the complaint and return.
1167	*/
1168	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
1169	if ( (fForced \|\| !fFallback)
1170	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
1171	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
1172
1173	if (RTErrInfoIsSet(pErrInfo))
1174	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
1175	return VINF_SUCCESS;
1176	}
1177
1178
1179	/**
1180	* This is called after CPUMR3Init is done.
1181	*
1182	* @returns VBox status code.
1183	* @param pVM The VM handle..
1184	*/
1185	int nemR3NativeInitAfterCPUM(PVM pVM)
1186	{
1187	/*
1188	* Validate sanity.
1189	*/
1190	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
1191
1192	/** @todo */
1193
1194	/*
1195	* Any hyper-v statistics we can get at now? HvCallMapStatsPage isn't accessible any more.
1196	*/
1197	/** @todo stats */
1198
1199	/*
1200	* Adjust features.
1201	*
1202	* Note! We've already disabled X2APIC and MONITOR/MWAIT via CFGM during
1203	* the first init call.
1204	*/
1205
1206	return VINF_SUCCESS;
1207	}
1208
1209
1210	/**
1211	* Execute state save operation.
1212	*
1213	* @returns VBox status code.
1214	* @param pVM The cross context VM structure.
1215	* @param pSSM SSM operation handle.
1216	*/
1217	static DECLCALLBACK(int) nemR3Save(PVM pVM, PSSMHANDLE pSSM)
1218	{
1219	/*
1220	* Save the Hyper-V activity state for all CPUs.
1221	*/
1222	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1223	{
1224	PVMCPUCC pVCpu = pVM->apCpusR3[i];
1225
1226	static const WHV_REGISTER_NAME s_Name = WHvRegisterInternalActivityState;
1227	WHV_REGISTER_VALUE Reg;
1228
1229	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, &s_Name, 1, &Reg);
1230	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1231	("WHvSetVirtualProcessorRegisters(%p, 0,{WHvRegisterInternalActivityState}, 1,) -> %Rhrc (Last=%#x/%u)\n",
1232	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1233	, VERR_NEM_IPE_9);
1234
1235	SSMR3PutU64(pSSM, Reg.Reg64);
1236	}
1237
1238	return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
1239	}
1240
1241
1242	/**
1243	* Execute state load operation.
1244	*
1245	* @returns VBox status code.
1246	* @param pVM The cross context VM structure.
1247	* @param pSSM SSM operation handle.
1248	* @param uVersion Data layout version.
1249	* @param uPass The data pass.
1250	*/
1251	static DECLCALLBACK(int) nemR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
1252	{
1253	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
1254
1255	/*
1256	* Validate version.
1257	*/
1258	if (uVersion != 1)
1259	{
1260	AssertMsgFailed(("nemR3Load: Invalid version uVersion=%u!\n", uVersion));
1261	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
1262	}
1263
1264	/*
1265	* Restore the Hyper-V activity states for all vCPUs.
1266	*/
1267	VMCPU_SET_STATE(pVM->apCpusR3[0], VMCPUSTATE_STARTED);
1268	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1269	{
1270	PVMCPUCC pVCpu = pVM->apCpusR3[i];
1271
1272	static const WHV_REGISTER_NAME s_Name = WHvRegisterInternalActivityState;
1273	WHV_REGISTER_VALUE Reg;
1274	int rc = SSMR3GetU64(pSSM, &Reg.Reg64);
1275	if (RT_FAILURE(rc))
1276	return rc;
1277
1278	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, &s_Name, 1, &Reg);
1279	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1280	("WHvSetVirtualProcessorRegisters(%p, 0,{WHvRegisterInternalActivityState}, 1,) -> %Rhrc (Last=%#x/%u)\n",
1281	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1282	, VERR_NEM_IPE_9);
1283	}
1284
1285	/* terminator */
1286	uint32_t u32;
1287	int rc = SSMR3GetU32(pSSM, &u32);
1288	if (RT_FAILURE(rc))
1289	return rc;
1290	if (u32 != UINT32_MAX)
1291	{
1292	AssertMsgFailed(("u32=%#x\n", u32));
1293	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1294	}
1295	return VINF_SUCCESS;
1296	}
1297
1298
1299	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
1300	{
1301	//BOOL fRet = SetThreadPriority(GetCurrentThread(), 0);
1302	//AssertLogRel(fRet);
1303
1304	if (enmWhat == VMINITCOMPLETED_RING3)
1305	{
1306	/*
1307	* Register the saved state data unit.
1308	*/
1309	int rc = SSMR3RegisterInternal(pVM, "nem-win", 1, NEM_HV_SAVED_STATE_VERSION,
1310	sizeof(uint64_t),
1311	NULL, NULL, NULL,
1312	NULL, nemR3Save, NULL,
1313	NULL, nemR3Load, NULL);
1314	if (RT_FAILURE(rc))
1315	return rc;
1316	}
1317
1318	NOREF(pVM); NOREF(enmWhat);
1319	return VINF_SUCCESS;
1320	}
1321
1322
1323	int nemR3NativeTerm(PVM pVM)
1324	{
1325	/*
1326	* Delete the partition.
1327	*/
1328	WHV_PARTITION_HANDLE hPartition = pVM->nem.s.hPartition;
1329	pVM->nem.s.hPartition = NULL;
1330	pVM->nem.s.hPartitionDevice = NULL;
1331	if (hPartition != NULL)
1332	{
1333	VMCPUID idCpu = pVM->nem.s.fCreatedEmts ? pVM->cCpus : 0;
1334	LogRel(("NEM: Destroying partition %p with its %u VCpus...\n", hPartition, idCpu));
1335	while (idCpu-- > 0)
1336	{
1337	HRESULT hrc = WHvDeleteVirtualProcessor(hPartition, idCpu);
1338	AssertLogRelMsg(SUCCEEDED(hrc), ("WHvDeleteVirtualProcessor(%p, %u) -> %Rhrc (Last=%#x/%u)\n",
1339	hPartition, idCpu, hrc, RTNtLastStatusValue(),
1340	RTNtLastErrorValue()));
1341	}
1342	WHvDeletePartition(hPartition);
1343	}
1344	pVM->nem.s.fCreatedEmts = false;
1345	return VINF_SUCCESS;
1346	}
1347
1348
1349	/**
1350	* VM reset notification.
1351	*
1352	* @param pVM The cross context VM structure.
1353	*/
1354	void nemR3NativeReset(PVM pVM)
1355	{
1356	RT_NOREF(pVM);
1357	}
1358
1359
1360	/**
1361	* Reset CPU due to INIT IPI or hot (un)plugging.
1362	*
1363	* @param pVCpu The cross context virtual CPU structure of the CPU being
1364	* reset.
1365	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
1366	*/
1367	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
1368	{
1369	RT_NOREF(pVCpu, fInitIpi);
1370	}
1371
1372
1373	NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVMCC pVM, PVMCPUCC pVCpu)
1374	{
1375	WHV_REGISTER_NAME aenmNames[128];
1376	WHV_REGISTER_VALUE aValues[128];
1377
1378	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL;
1379	if (!fWhat)
1380	return VINF_SUCCESS;
1381	uintptr_t iReg = 0;
1382
1383	#define ADD_REG64(a_enmName, a_uValue) do { \
1384	aenmNames[iReg] = (a_enmName); \
1385	aValues[iReg].Reg128.High64 = 0; \
1386	aValues[iReg].Reg64 = (a_uValue).x; \
1387	iReg++; \
1388	} while (0)
1389	#define ADD_REG64_RAW(a_enmName, a_uValue) do { \
1390	aenmNames[iReg] = (a_enmName); \
1391	aValues[iReg].Reg128.High64 = 0; \
1392	aValues[iReg].Reg64 = (a_uValue); \
1393	iReg++; \
1394	} while (0)
1395	#define ADD_SYSREG64(a_enmName, a_uValue) do { \
1396	aenmNames[iReg] = (a_enmName); \
1397	aValues[iReg].Reg128.High64 = 0; \
1398	aValues[iReg].Reg64 = (a_uValue).u64; \
1399	iReg++; \
1400	} while (0)
1401	#define ADD_REG128(a_enmName, a_uValue) do { \
1402	aenmNames[iReg] = (a_enmName); \
1403	aValues[iReg].Reg128.Low64 = (a_uValue).au64[0]; \
1404	aValues[iReg].Reg128.High64 = (a_uValue).au64[1]; \
1405	iReg++; \
1406	} while (0)
1407
1408	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_FPCR \| CPUMCTX_EXTRN_FPSR))
1409	!= (CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_FPCR \| CPUMCTX_EXTRN_FPSR))
1410	{
1411	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
1412	{
1413	if (!(s_aCpumRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
1414	{
1415	const CPUMCTXGREG pReg = (const CPUMCTXGREG )((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
1416	ADD_REG64(s_aCpumRegs[i].enmWHvReg, *pReg);
1417	}
1418	}
1419	}
1420
1421	if (fWhat & CPUMCTX_EXTRN_V0_V31)
1422	{
1423	/* SIMD/FP registers. */
1424	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
1425	{
1426	PCCPUMCTXVREG pVReg = (PCCPUMCTXVREG)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
1427	ADD_REG128(s_aCpumFpRegs[i].enmWHvReg, *pVReg);
1428	}
1429	}
1430
1431	if (fWhat & CPUMCTX_EXTRN_SYSREG_DEBUG)
1432	{
1433	for (uint32_t i = 0; i < pVM->nem.s.cBreakpoints; i++)
1434	{
1435	ADD_SYSREG64((WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgbcr0El1 + i), pVCpu->cpum.GstCtx.aBp[i].Ctrl);
1436	ADD_SYSREG64((WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgbvr0El1 + i), pVCpu->cpum.GstCtx.aBp[i].Value);
1437	}
1438
1439	for (uint32_t i = 0; i < pVM->nem.s.cWatchpoints; i++)
1440	{
1441	ADD_SYSREG64((WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgwcr0El1 + i), pVCpu->cpum.GstCtx.aWp[i].Ctrl);
1442	ADD_SYSREG64((WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgwvr0El1 + i), pVCpu->cpum.GstCtx.aWp[i].Value);
1443	}
1444
1445	ADD_SYSREG64(WHvArm64RegisterMdscrEl1, pVCpu->cpum.GstCtx.Mdscr);
1446	}
1447
1448	if (fWhat & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS)
1449	{
1450	/* PAuth registers. */
1451	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
1452	{
1453	const CPUMCTXSYSREG pReg = (const CPUMCTXSYSREG )((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
1454	ADD_SYSREG64(s_aCpumPAuthKeyRegs[i].enmWHvReg, *pReg);
1455	}
1456	}
1457
1458	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_SPSR \| CPUMCTX_EXTRN_ELR \| CPUMCTX_EXTRN_SP \| CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_MISC))
1459	!= (CPUMCTX_EXTRN_SPSR \| CPUMCTX_EXTRN_ELR \| CPUMCTX_EXTRN_SP \| CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_MISC))
1460	{
1461	/* System registers. */
1462	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
1463	{
1464	if (!(s_aCpumSysRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
1465	{
1466	const CPUMCTXSYSREG pReg = (const CPUMCTXSYSREG )((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
1467	ADD_SYSREG64(s_aCpumSysRegs[i].enmWHvReg, *pReg);
1468	}
1469	}
1470	}
1471
1472	if (fWhat & CPUMCTX_EXTRN_PSTATE)
1473	ADD_REG64_RAW(WHvArm64RegisterPstate, pVCpu->cpum.GstCtx.fPState);
1474
1475	#undef ADD_REG64
1476	#undef ADD_REG64_RAW
1477	#undef ADD_REG128
1478
1479	/*
1480	* Set the registers.
1481	*/
1482	Assert(iReg < RT_ELEMENTS(aValues));
1483	Assert(iReg < RT_ELEMENTS(aenmNames));
1484	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
1485	if (SUCCEEDED(hrc))
1486	{
1487	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_NEM;
1488	return VINF_SUCCESS;
1489	}
1490	AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
1491	pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
1492	hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
1493	return VERR_INTERNAL_ERROR;
1494	}
1495
1496
1497	NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
1498	{
1499	WHV_REGISTER_NAME aenmNames[256];
1500
1501	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
1502	if (!fWhat)
1503	return VINF_SUCCESS;
1504
1505	uintptr_t iReg = 0;
1506
1507	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_FPCR \| CPUMCTX_EXTRN_FPSR))
1508	!= (CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_FPCR \| CPUMCTX_EXTRN_FPSR))
1509	{
1510	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
1511	{
1512	if (!(s_aCpumRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
1513	aenmNames[iReg++] = s_aCpumRegs[i].enmWHvReg;
1514	}
1515	}
1516
1517	if (fWhat & CPUMCTX_EXTRN_V0_V31)
1518	{
1519	/* SIMD/FP registers. */
1520	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
1521	{
1522	aenmNames[iReg++] = s_aCpumFpRegs[i].enmWHvReg;
1523	}
1524	}
1525
1526	if (fWhat & CPUMCTX_EXTRN_SYSREG_DEBUG)
1527	{
1528	for (uint32_t i = 0; i < pVM->nem.s.cBreakpoints; i++)
1529	{
1530	aenmNames[iReg++] = (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgbcr0El1 + i);
1531	aenmNames[iReg++] = (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgbvr0El1 + i);
1532	}
1533
1534	for (uint32_t i = 0; i < pVM->nem.s.cWatchpoints; i++)
1535	{
1536	aenmNames[iReg++] = (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgwcr0El1 + i);
1537	aenmNames[iReg++] = (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgwvr0El1 + i);
1538	}
1539
1540	aenmNames[iReg++] = WHvArm64RegisterMdscrEl1;
1541	}
1542
1543	if (fWhat & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS)
1544	{
1545	/* PAuth registers. */
1546	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
1547	{
1548	aenmNames[iReg++] = s_aCpumPAuthKeyRegs[i].enmWHvReg;
1549	}
1550	}
1551
1552	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_SPSR \| CPUMCTX_EXTRN_ELR \| CPUMCTX_EXTRN_SP \| CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_MISC))
1553	!= (CPUMCTX_EXTRN_SPSR \| CPUMCTX_EXTRN_ELR \| CPUMCTX_EXTRN_SP \| CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_MISC))
1554	{
1555	/* System registers. */
1556	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
1557	{
1558	if (!(s_aCpumSysRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
1559	aenmNames[iReg++] = s_aCpumSysRegs[i].enmWHvReg;
1560	}
1561	}
1562
1563	if (fWhat & CPUMCTX_EXTRN_PSTATE)
1564	aenmNames[iReg++] = WHvArm64RegisterPstate;
1565
1566	size_t const cRegs = iReg;
1567	Assert(cRegs < RT_ELEMENTS(aenmNames));
1568
1569	/*
1570	* Get the registers.
1571	*/
1572	WHV_REGISTER_VALUE aValues[256];
1573	RT_ZERO(aValues);
1574	Assert(RT_ELEMENTS(aValues) >= cRegs);
1575	Assert(RT_ELEMENTS(aenmNames) >= cRegs);
1576	HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues);
1577	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1578	("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
1579	pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1580	, VERR_NEM_GET_REGISTERS_FAILED);
1581
1582	iReg = 0;
1583	#define GET_REG64(a_DstVar, a_enmName) do { \
1584	Assert(aenmNames[iReg] == (a_enmName)); \
1585	(a_DstVar)->x = aValues[iReg].Reg64; \
1586	iReg++; \
1587	} while (0)
1588	#define GET_REG64_RAW(a_DstVar, a_enmName) do { \
1589	Assert(aenmNames[iReg] == (a_enmName)); \
1590	*(a_DstVar) = aValues[iReg].Reg64; \
1591	iReg++; \
1592	} while (0)
1593	#define GET_SYSREG64(a_DstVar, a_enmName) do { \
1594	Assert(aenmNames[iReg] == (a_enmName)); \
1595	(a_DstVar)->u64 = aValues[iReg].Reg64; \
1596	iReg++; \
1597	} while (0)
1598	#define GET_REG128(a_DstVar, a_enmName) do { \
1599	Assert(aenmNames[iReg] == a_enmName); \
1600	(a_DstVar)->au64[0] = aValues[iReg].Reg128.Low64; \
1601	(a_DstVar)->au64[1] = aValues[iReg].Reg128.High64; \
1602	iReg++; \
1603	} while (0)
1604
1605	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_FPCR \| CPUMCTX_EXTRN_FPSR))
1606	!= (CPUMCTX_EXTRN_GPRS_MASK \| CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_FPCR \| CPUMCTX_EXTRN_FPSR))
1607	{
1608	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
1609	{
1610	if (!(s_aCpumRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
1611	{
1612	CPUMCTXGREG pReg = (CPUMCTXGREG )((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
1613	GET_REG64(pReg, s_aCpumRegs[i].enmWHvReg);
1614	}
1615	}
1616	}
1617
1618	if (fWhat & CPUMCTX_EXTRN_V0_V31)
1619	{
1620	/* SIMD/FP registers. */
1621	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
1622	{
1623	PCPUMCTXVREG pVReg = (PCPUMCTXVREG)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
1624	GET_REG128(pVReg, s_aCpumFpRegs[i].enmWHvReg);
1625	}
1626	}
1627
1628	if (fWhat & CPUMCTX_EXTRN_SYSREG_DEBUG)
1629	{
1630	for (uint32_t i = 0; i < pVM->nem.s.cBreakpoints; i++)
1631	{
1632	GET_SYSREG64(&pVCpu->cpum.GstCtx.aBp[i].Ctrl, (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgbcr0El1 + i));
1633	GET_SYSREG64(&pVCpu->cpum.GstCtx.aBp[i].Value, (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgbvr0El1 + i));
1634	}
1635
1636	for (uint32_t i = 0; i < pVM->nem.s.cWatchpoints; i++)
1637	{
1638	GET_SYSREG64(&pVCpu->cpum.GstCtx.aWp[i].Ctrl, (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgwcr0El1 + i));
1639	GET_SYSREG64(&pVCpu->cpum.GstCtx.aWp[i].Value, (WHV_REGISTER_NAME)((uint32_t)WHvArm64RegisterDbgwvr0El1 + i));
1640	}
1641
1642	GET_SYSREG64(&pVCpu->cpum.GstCtx.Mdscr, WHvArm64RegisterMdscrEl1);
1643	}
1644
1645	if (fWhat & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS)
1646	{
1647	/* PAuth registers. */
1648	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
1649	{
1650	CPUMCTXSYSREG pReg = (CPUMCTXSYSREG )((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
1651	GET_SYSREG64(pReg, s_aCpumPAuthKeyRegs[i].enmWHvReg);
1652	}
1653	}
1654
1655	if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_SPSR \| CPUMCTX_EXTRN_ELR \| CPUMCTX_EXTRN_SP \| CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_MISC))
1656	!= (CPUMCTX_EXTRN_SPSR \| CPUMCTX_EXTRN_ELR \| CPUMCTX_EXTRN_SP \| CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_MISC))
1657	{
1658	/* System registers. */
1659	for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
1660	{
1661	if (!(s_aCpumSysRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
1662	{
1663	CPUMCTXSYSREG pReg = (CPUMCTXSYSREG )((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
1664	GET_SYSREG64(pReg, s_aCpumSysRegs[i].enmWHvReg);
1665	}
1666	}
1667	}
1668
1669	if (fWhat & CPUMCTX_EXTRN_PSTATE)
1670	GET_REG64_RAW(&pVCpu->cpum.GstCtx.fPState, WHvArm64RegisterPstate);
1671
1672	/* Almost done, just update extrn flags. */
1673	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1674	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1675	pVCpu->cpum.GstCtx.fExtrn = 0;
1676
1677	return VINF_SUCCESS;
1678	}
1679
1680
1681	/**
1682	* Interface for importing state on demand (used by IEM).
1683	*
1684	* @returns VBox status code.
1685	* @param pVCpu The cross context CPU structure.
1686	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1687	*/
1688	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1689	{
1690	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
1691	return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
1692	}
1693
1694
1695	/**
1696	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
1697	*
1698	* @returns VBox status code.
1699	* @param pVCpu The cross context CPU structure.
1700	* @param pcTicks Where to return the CPU tick count.
1701	* @param puAux Where to return the TSC_AUX register value.
1702	*/
1703	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
1704	{
1705	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
1706
1707	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1708	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1709	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1710
1711	/* Ensure time for the partition is suspended - it will be resumed as soon as a vCPU starts executing. */
1712	HRESULT hrc = WHvSuspendPartitionTime(pVM->nem.s.hPartition);
1713	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1714	("WHvSuspendPartitionTime(%p) -> %Rhrc (Last=%#x/%u)\n",
1715	pVM->nem.s.hPartition, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1716	, VERR_NEM_GET_REGISTERS_FAILED);
1717
1718	/* Call the offical API. */
1719	WHV_REGISTER_NAME enmName = WHvArm64RegisterCntvctEl0;
1720	WHV_REGISTER_VALUE Value = { { {0, 0} } };
1721	hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, &enmName, 1, &Value);
1722	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1723	("WHvGetVirtualProcessorRegisters(%p, %u,{CNTVCT_EL0},1,) -> %Rhrc (Last=%#x/%u)\n",
1724	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1725	, VERR_NEM_GET_REGISTERS_FAILED);
1726	*pcTicks = Value.Reg64;
1727	LogFlow(("NEMHCQueryCpuTick: %#RX64 (host: %#RX64)\n", *pcTicks, ASMReadTSC()));
1728	if (puAux)
1729	*puAux =0;
1730
1731	return VINF_SUCCESS;
1732	}
1733
1734
1735	/**
1736	* Resumes CPU clock (TSC) on all virtual CPUs.
1737	*
1738	* This is called by TM when the VM is started, restored, resumed or similar.
1739	*
1740	* @returns VBox status code.
1741	* @param pVM The cross context VM structure.
1742	* @param pVCpu The cross context CPU structure of the calling EMT.
1743	* @param uPausedTscValue The TSC value at the time of pausing.
1744	*/
1745	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
1746	{
1747	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
1748	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
1749
1750	/*
1751	* Call the offical API to do the job.
1752	*/
1753	LogFlow(("NEMHCResumeCpuTickOnAll: %#RX64 (host: %#RX64)\n", uPausedTscValue, ASMReadTSC()));
1754
1755	/*
1756	* Now set the CNTVCT_EL0 register for each vCPU, Hyper-V will program the timer offset in
1757	* CNTVOFF_EL2 accordingly. ARM guarantees that CNTVCT_EL0 is synchronised across all CPUs,
1758	* as long as CNTVOFF_EL2 is the same everywhere. Lets just hope scheduling will not affect it
1759	* if the partition time is suspended.
1760	*/
1761	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1762	{
1763	WHV_REGISTER_NAME enmName = WHvArm64RegisterCntvctEl0;
1764	WHV_REGISTER_VALUE Value;
1765	Value.Reg64 = uPausedTscValue;
1766	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, idCpu, &enmName, 1, &Value);
1767	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1768	("WHvSetVirtualProcessorRegisters(%p, 0,{CNTVCT_EL0},1,%#RX64) -> %Rhrc (Last=%#x/%u)\n",
1769	pVM->nem.s.hPartition, idCpu, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1770	, VERR_NEM_SET_TSC);
1771
1772	/* Make sure the CNTV_CTL_EL0 and CNTV_CVAL_EL0 registers are up to date after resuming (saved state load). */
1773	PVMCPUCC pVCpuDst = pVM->apCpusR3[idCpu];
1774	pVCpuDst->nem.s.fSyncCntvRegs = true;
1775	}
1776
1777	HRESULT hrc = WHvResumePartitionTime(pVM->nem.s.hPartition);
1778	AssertLogRelMsgReturn(SUCCEEDED(hrc),
1779	("WHvResumePartitionTime(%p) -> %Rhrc (Last=%#x/%u)\n",
1780	pVM->nem.s.hPartition, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
1781	, VERR_NEM_SET_TSC);
1782
1783	return VINF_SUCCESS;
1784	}
1785
1786
1787	#ifdef LOG_ENABLED
1788	/**
1789	* Logs the current CPU state.
1790	*/
1791	static void nemR3WinLogState(PVMCC pVM, PVMCPUCC pVCpu)
1792	{
1793	if (LogIs3Enabled())
1794	{
1795	char szRegs[4096];
1796	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
1797	"x0=%016VR{x0} x1=%016VR{x1} x2=%016VR{x2} x3=%016VR{x3}\n"
1798	"x4=%016VR{x4} x5=%016VR{x5} x6=%016VR{x6} x7=%016VR{x7}\n"
1799	"x8=%016VR{x8} x9=%016VR{x9} x10=%016VR{x10} x11=%016VR{x11}\n"
1800	"x12=%016VR{x12} x13=%016VR{x13} x14=%016VR{x14} x15=%016VR{x15}\n"
1801	"x16=%016VR{x16} x17=%016VR{x17} x18=%016VR{x18} x19=%016VR{x19}\n"
1802	"x20=%016VR{x20} x21=%016VR{x21} x22=%016VR{x22} x23=%016VR{x23}\n"
1803	"x24=%016VR{x24} x25=%016VR{x25} x26=%016VR{x26} x27=%016VR{x27}\n"
1804	"x28=%016VR{x28} x29=%016VR{x29} x30=%016VR{x30}\n"
1805	"pc=%016VR{pc} pstate=%016VR{pstate}\n"
1806	"sp_el0=%016VR{sp_el0} sp_el1=%016VR{sp_el1} elr_el1=%016VR{elr_el1}\n"
1807	"sctlr_el1=%016VR{sctlr_el1} tcr_el1=%016VR{tcr_el1}\n"
1808	"ttbr0_el1=%016VR{ttbr0_el1} ttbr1_el1=%016VR{ttbr1_el1}\n"
1809	"vbar_el1=%016VR{vbar_el1}\n"
1810	);
1811	char szInstr[256]; RT_ZERO(szInstr);
1812	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
1813	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
1814	szInstr, sizeof(szInstr), NULL);
1815	Log3(("%s%s\n", szRegs, szInstr));
1816	}
1817	}
1818	#endif /* LOG_ENABLED */
1819
1820
1821	/**
1822	* Copies register state from the (common) exit context.
1823	*
1824	* ASSUMES no state copied yet.
1825	*
1826	* @param pVCpu The cross context per CPU structure.
1827	* @param pMsgHdr The common message header.
1828	*/
1829	DECLINLINE(void) nemR3WinCopyStateFromArmHeader(PVMCPUCC pVCpu, WHV_INTERCEPT_MESSAGE_HEADER const *pMsgHdr)
1830	{
1831	#ifdef LOG_ENABLED /* When state logging is enabled the state is synced completely upon VM exit. */
1832	if (!LogIs3Enabled())
1833	#endif
1834	Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_PSTATE))
1835	== (CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_PSTATE));
1836
1837	pVCpu->cpum.GstCtx.Pc.u64 = pMsgHdr->Pc;
1838	pVCpu->cpum.GstCtx.fPState = pMsgHdr->Cpsr;
1839
1840	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_PC \| CPUMCTX_EXTRN_PSTATE);
1841	}
1842
1843
1844	/**
1845	* Returns the byte size from the given access SAS value.
1846	*
1847	* @returns Number of bytes to transfer.
1848	* @param uSas The SAS value to convert.
1849	*/
1850	DECLINLINE(size_t) nemR3WinGetByteCountFromSas(uint8_t uSas)
1851	{
1852	switch (uSas)
1853	{
1854	case ARMV8_EC_ISS_DATA_ABRT_SAS_BYTE: return sizeof(uint8_t);
1855	case ARMV8_EC_ISS_DATA_ABRT_SAS_HALFWORD: return sizeof(uint16_t);
1856	case ARMV8_EC_ISS_DATA_ABRT_SAS_WORD: return sizeof(uint32_t);
1857	case ARMV8_EC_ISS_DATA_ABRT_SAS_DWORD: return sizeof(uint64_t);
1858	default:
1859	AssertReleaseFailed();
1860	}
1861
1862	return 0;
1863	}
1864
1865
1866	/**
1867	* Sets the given general purpose register to the given value.
1868	*
1869	* @param pVCpu The cross context virtual CPU structure of the
1870	* calling EMT.
1871	* @param uReg The register index.
1872	* @param f64BitReg Flag whether to operate on a 64-bit or 32-bit register.
1873	* @param fSignExtend Flag whether to sign extend the value.
1874	* @param u64Val The value.
1875	*/
1876	DECLINLINE(void) nemR3WinSetGReg(PVMCPU pVCpu, uint8_t uReg, bool f64BitReg, bool fSignExtend, uint64_t u64Val)
1877	{
1878	AssertReturnVoid(uReg < 31);
1879
1880	if (f64BitReg)
1881	pVCpu->cpum.GstCtx.aGRegs[uReg].x = fSignExtend ? (int64_t)u64Val : u64Val;
1882	else
1883	pVCpu->cpum.GstCtx.aGRegs[uReg].x = (uint64_t)(fSignExtend ? (int32_t)u64Val : (uint32_t)u64Val);
1884
1885	/* Mark the register as not extern anymore. */
1886	switch (uReg)
1887	{
1888	case 0:
1889	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X0;
1890	break;
1891	case 1:
1892	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X1;
1893	break;
1894	case 2:
1895	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X2;
1896	break;
1897	case 3:
1898	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X3;
1899	break;
1900	default:
1901	AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_X4_X28));
1902	/** @todo We need to import all missing registers in order to clear this flag (or just set it in HV from here). */
1903	}
1904	}
1905
1906
1907	/**
1908	* Gets the given general purpose register and returns the value.
1909	*
1910	* @returns Value from the given register.
1911	* @param pVCpu The cross context virtual CPU structure of the
1912	* calling EMT.
1913	* @param uReg The register index.
1914	*/
1915	DECLINLINE(uint64_t) nemR3WinGetGReg(PVMCPU pVCpu, uint8_t uReg)
1916	{
1917	AssertReturn(uReg <= ARMV8_A64_REG_XZR, 0);
1918
1919	if (uReg == ARMV8_A64_REG_XZR)
1920	return 0;
1921
1922	/** @todo Import the register if extern. */
1923	AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_GPRS_MASK));
1924
1925	return pVCpu->cpum.GstCtx.aGRegs[uReg].x;
1926	}
1927
1928
1929	/**
1930	* Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
1931	*
1932	* @returns Strict VBox status code.
1933	* @param pVM The cross context VM structure.
1934	* @param pVCpu The cross context per CPU structure.
1935	* @param pExit The VM exit information to handle.
1936	* @sa nemHCWinHandleMessageMemory
1937	*/
1938	NEM_TMPL_STATIC VBOXSTRICTRC
1939	nemR3WinHandleExitMemory(PVMCC pVM, PVMCPUCC pVCpu, MY_WHV_RUN_VP_EXIT_CONTEXT const *pExit)
1940	{
1941	uint64_t const uHostTsc = ASMReadTSC();
1942	Assert(pExit->MemoryAccess.Header.InterceptAccessType != 3);
1943
1944	/*
1945	* Emulate the memory access, either access handler or special memory.
1946	*/
1947	WHV_INTERCEPT_MESSAGE_HEADER const *pHdr = &pExit->MemoryAccess.Header;
1948	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
1949	pExit->MemoryAccess.Header.InterceptAccessType == WHvMemoryAccessWrite
1950	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1951	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1952	pHdr->Pc, uHostTsc);
1953	nemR3WinCopyStateFromArmHeader(pVCpu, &pExit->MemoryAccess.Header);
1954	RT_NOREF_PV(pExitRec);
1955	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
1956	AssertRCReturn(rc, rc);
1957
1958	#ifdef LOG_ENABLED
1959	uint8_t const cbInstr = pExit->MemoryAccess.InstructionByteCount;
1960	RTGCPTR const GCPtrVa = pExit->MemoryAccess.Gva;
1961	#endif
1962	RTGCPHYS const GCPhys = pExit->MemoryAccess.Gpa;
1963	uint64_t const uIss = pExit->MemoryAccess.Syndrome;
1964	bool fIsv = RT_BOOL(uIss & ARMV8_EC_ISS_DATA_ABRT_ISV);
1965	bool fL2Fault = RT_BOOL(uIss & ARMV8_EC_ISS_DATA_ABRT_S1PTW);
1966	bool fWrite = RT_BOOL(uIss & ARMV8_EC_ISS_DATA_ABRT_WNR);
1967	bool f64BitReg = RT_BOOL(uIss & ARMV8_EC_ISS_DATA_ABRT_SF);
1968	bool fSignExtend = RT_BOOL(uIss & ARMV8_EC_ISS_DATA_ABRT_SSE);
1969	uint8_t uReg = ARMV8_EC_ISS_DATA_ABRT_SRT_GET(uIss);
1970	uint8_t uAcc = ARMV8_EC_ISS_DATA_ABRT_SAS_GET(uIss);
1971	size_t cbAcc = nemR3WinGetByteCountFromSas(uAcc);
1972	LogFlowFunc(("fIsv=%RTbool fL2Fault=%RTbool fWrite=%RTbool f64BitReg=%RTbool fSignExtend=%RTbool uReg=%u uAcc=%u GCPtrDataAbrt=%RGv GCPhys=%RGp cbInstr=%u\n",
1973	fIsv, fL2Fault, fWrite, f64BitReg, fSignExtend, uReg, uAcc, GCPtrVa, GCPhys, cbInstr));
1974
1975	RT_NOREF(fL2Fault);
1976
1977	VBOXSTRICTRC rcStrict;
1978	if (fIsv)
1979	{
1980	EMHistoryAddExit(pVCpu,
1981	fWrite
1982	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1983	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1984	pVCpu->cpum.GstCtx.Pc.u64, ASMReadTSC());
1985
1986	uint64_t u64Val = 0;
1987	if (fWrite)
1988	{
1989	u64Val = nemR3WinGetGReg(pVCpu, uReg);
1990	rcStrict = PGMPhysWrite(pVM, GCPhys, &u64Val, cbAcc, PGMACCESSORIGIN_HM);
1991	Log4(("MmioExit/%u: %08RX64: WRITE %RGp LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
1992	pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc.u64, GCPhys, cbAcc, cbAcc,
1993	&u64Val, VBOXSTRICTRC_VAL(rcStrict) ));
1994	}
1995	else
1996	{
1997	rcStrict = PGMPhysRead(pVM, GCPhys, &u64Val, cbAcc, PGMACCESSORIGIN_HM);
1998	Log4(("MmioExit/%u: %08RX64: READ %RGp LB %u -> %.*Rhxs rcStrict=%Rrc\n",
1999	pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc.u64, GCPhys, cbAcc, cbAcc,
2000	&u64Val, VBOXSTRICTRC_VAL(rcStrict) ));
2001	if (rcStrict == VINF_SUCCESS)
2002	nemR3WinSetGReg(pVCpu, uReg, f64BitReg, fSignExtend, u64Val);
2003	}
2004	}
2005	else
2006	{
2007	/** @todo Our UEFI firmware accesses the flash region with the following instruction
2008	* when the NVRAM actually contains data:
2009	* ldrb w9, [x6, #-0x0001]!
2010	* This is too complicated for the hardware so the ISV bit is not set. Until there
2011	* is a proper IEM implementation we just handle this here for now to avoid annoying
2012	* users too much.
2013	*/
2014	/* The following ASSUMES that the vCPU state is completely synced. */
2015
2016	/* Read instruction. */
2017	RTGCPTR GCPtrPage = pVCpu->cpum.GstCtx.Pc.u64 & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2018	const void *pvPageR3 = NULL;
2019	PGMPAGEMAPLOCK PageMapLock;
2020
2021	rcStrict = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrPage, &pvPageR3, &PageMapLock);
2022	if (rcStrict == VINF_SUCCESS)
2023	{
2024	uint32_t u32Instr = (uint32_t )((uint8_t *)pvPageR3 + (pVCpu->cpum.GstCtx.Pc.u64 - GCPtrPage));
2025	PGMPhysReleasePageMappingLock(pVCpu->pVMR3, &PageMapLock);
2026
2027	DISSTATE Dis;
2028	rcStrict = DISInstrWithPrefetchedBytes((uintptr_t)pVCpu->cpum.GstCtx.Pc.u64, DISCPUMODE_ARMV8_A64, 0 /fFilter - none /,
2029	&u32Instr, sizeof(u32Instr), NULL, NULL, &Dis, NULL);
2030	if (rcStrict == VINF_SUCCESS)
2031	{
2032	if ( Dis.pCurInstr->uOpcode == OP_ARMV8_A64_LDRB
2033	&& Dis.aParams[0].armv8.enmType == kDisArmv8OpParmReg
2034	&& Dis.aParams[0].armv8.Op.Reg.enmRegType == kDisOpParamArmV8RegType_Gpr_32Bit
2035	&& Dis.aParams[1].armv8.enmType == kDisArmv8OpParmAddrInGpr
2036	&& Dis.aParams[1].armv8.Op.Reg.enmRegType == kDisOpParamArmV8RegType_Gpr_64Bit
2037	&& (Dis.aParams[1].fUse & DISUSE_PRE_INDEXED))
2038	{
2039	/* The fault address is already the final address. */
2040	uint8_t bVal = 0;
2041	rcStrict = PGMPhysRead(pVM, GCPhys, &bVal, 1, PGMACCESSORIGIN_HM);
2042	Log4(("MmioExit/%u: %08RX64: READ %#RGp LB %u -> %.*Rhxs rcStrict=%Rrc\n",
2043	pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc.u64, GCPhys, sizeof(bVal), sizeof(bVal),
2044	&bVal, VBOXSTRICTRC_VAL(rcStrict) ));
2045	if (rcStrict == VINF_SUCCESS)
2046	{
2047	nemR3WinSetGReg(pVCpu, Dis.aParams[0].armv8.Op.Reg.idReg, false /f64BitReg/, false /fSignExtend/, bVal);
2048	/* Update the indexed register. */
2049	pVCpu->cpum.GstCtx.aGRegs[Dis.aParams[1].armv8.Op.Reg.idReg].x += Dis.aParams[1].armv8.u.offBase;
2050	}
2051	}
2052	/*
2053	* Seeing the following with the Windows 11/ARM TPM driver:
2054	* %fffff800e5342888 48 25 45 29 ldp w8, w9, [x10, #+0x0028]
2055	*/
2056	else if ( Dis.pCurInstr->uOpcode == OP_ARMV8_A64_LDP
2057	&& Dis.aParams[0].armv8.enmType == kDisArmv8OpParmReg
2058	&& Dis.aParams[0].armv8.Op.Reg.enmRegType == kDisOpParamArmV8RegType_Gpr_32Bit
2059	&& Dis.aParams[1].armv8.enmType == kDisArmv8OpParmReg
2060	&& Dis.aParams[1].armv8.Op.Reg.enmRegType == kDisOpParamArmV8RegType_Gpr_32Bit
2061	&& Dis.aParams[2].armv8.enmType == kDisArmv8OpParmAddrInGpr
2062	&& Dis.aParams[2].armv8.Op.Reg.enmRegType == kDisOpParamArmV8RegType_Gpr_64Bit)
2063	{
2064	/** @todo This is tricky to handle if the first register read returns something else than VINF_SUCCESS... */
2065	/* The fault address is already the final address. */
2066	uint32_t u32Val1 = 0;
2067	uint32_t u32Val2 = 0;
2068	rcStrict = PGMPhysRead(pVM, GCPhys, &u32Val1, sizeof(u32Val1), PGMACCESSORIGIN_HM);
2069	if (rcStrict == VINF_SUCCESS)
2070	rcStrict = PGMPhysRead(pVM, GCPhys + sizeof(uint32_t), &u32Val2, sizeof(u32Val2), PGMACCESSORIGIN_HM);
2071	Log4(("MmioExit/%u: %08RX64: READ %#RGp LB %u -> %.Rhxs %.Rhxs rcStrict=%Rrc\n",
2072	pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc.u64, GCPhys, 2 * sizeof(uint32_t), sizeof(u32Val1),
2073	&u32Val1, sizeof(u32Val2), &u32Val2, VBOXSTRICTRC_VAL(rcStrict) ));
2074	if (rcStrict == VINF_SUCCESS)
2075	{
2076	nemR3WinSetGReg(pVCpu, Dis.aParams[0].armv8.Op.Reg.idReg, false /f64BitReg/, false /fSignExtend/, u32Val1);
2077	nemR3WinSetGReg(pVCpu, Dis.aParams[1].armv8.Op.Reg.idReg, false /f64BitReg/, false /fSignExtend/, u32Val2);
2078	}
2079	}
2080	else
2081	AssertFailedReturn(VERR_NOT_SUPPORTED);
2082	}
2083	}
2084	}
2085
2086	if (rcStrict == VINF_SUCCESS)
2087	pVCpu->cpum.GstCtx.Pc.u64 += sizeof(uint32_t); /** @todo Why is InstructionByteCount always 0? */
2088
2089	return rcStrict;
2090	}
2091
2092
2093	/**
2094	* Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
2095	*
2096	* @returns Strict VBox status code.
2097	* @param pVM The cross context VM structure.
2098	* @param pVCpu The cross context per CPU structure.
2099	* @param pExit The VM exit information to handle.
2100	* @sa nemHCWinHandleMessageMemory
2101	*/
2102	NEM_TMPL_STATIC VBOXSTRICTRC
2103	nemR3WinHandleExitHypercall(PVMCC pVM, PVMCPUCC pVCpu, MY_WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2104	{
2105	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2106
2107	/** @todo Raise exception to EL1 if PSCI not configured. */
2108	/** @todo Need a generic mechanism here to pass this to, GIM maybe?. */
2109	uint32_t uFunId = pExit->Hypercall.Immediate;
2110	bool fHvc64 = RT_BOOL(uFunId & ARM_SMCCC_FUNC_ID_64BIT); RT_NOREF(fHvc64);
2111	uint32_t uEntity = ARM_SMCCC_FUNC_ID_ENTITY_GET(uFunId);
2112	uint32_t uFunNum = ARM_SMCCC_FUNC_ID_NUM_GET(uFunId);
2113	if (uEntity == ARM_SMCCC_FUNC_ID_ENTITY_STD_SEC_SERVICE)
2114	{
2115	switch (uFunNum)
2116	{
2117	case ARM_PSCI_FUNC_ID_PSCI_VERSION:
2118	nemR3WinSetGReg(pVCpu, ARMV8_A64_REG_X0, false /f64BitReg/, false /fSignExtend/, ARM_PSCI_FUNC_ID_PSCI_VERSION_SET(1, 2));
2119	break;
2120	case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
2121	rcStrict = VMR3PowerOff(pVM->pUVM);
2122	break;
2123	case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
2124	case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
2125	{
2126	bool fHaltOnReset;
2127	int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
2128	if (RT_SUCCESS(rc) && fHaltOnReset)
2129	{
2130	Log(("nemHCLnxHandleExitHypercall: Halt On Reset!\n"));
2131	rcStrict = VINF_EM_HALT;
2132	}
2133	else
2134	{
2135	/** @todo pVM->pdm.s.fResetFlags = fFlags; */
2136	VM_FF_SET(pVM, VM_FF_RESET);
2137	rcStrict = VINF_EM_RESET;
2138	}
2139	break;
2140	}
2141	case ARM_PSCI_FUNC_ID_CPU_ON:
2142	{
2143	uint64_t u64TgtCpu = pExit->Hypercall.X[1];
2144	RTGCPHYS GCPhysExecAddr = pExit->Hypercall.X[2];
2145	uint64_t u64CtxId = pExit->Hypercall.X[3];
2146	VMMR3CpuOn(pVM, u64TgtCpu & 0xff, GCPhysExecAddr, u64CtxId);
2147	nemR3WinSetGReg(pVCpu, ARMV8_A64_REG_X0, true /f64BitReg/, false /fSignExtend/, ARM_PSCI_STS_SUCCESS);
2148	break;
2149	}
2150	case ARM_PSCI_FUNC_ID_PSCI_FEATURES:
2151	{
2152	uint32_t u32FunNum = (uint32_t)pExit->Hypercall.X[1];
2153	switch (u32FunNum)
2154	{
2155	case ARM_PSCI_FUNC_ID_PSCI_VERSION:
2156	case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
2157	case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
2158	case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
2159	case ARM_PSCI_FUNC_ID_CPU_ON:
2160	nemR3WinSetGReg(pVCpu, ARMV8_A64_REG_X0,
2161	false /f64BitReg/, false /fSignExtend/,
2162	(uint64_t)ARM_PSCI_STS_SUCCESS);
2163	break;
2164	default:
2165	nemR3WinSetGReg(pVCpu, ARMV8_A64_REG_X0,
2166	false /f64BitReg/, false /fSignExtend/,
2167	(uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
2168	}
2169	break;
2170	}
2171	default:
2172	nemR3WinSetGReg(pVCpu, ARMV8_A64_REG_X0, false /f64BitReg/, false /fSignExtend/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
2173	}
2174	}
2175	else
2176	nemR3WinSetGReg(pVCpu, ARMV8_A64_REG_X0, false /f64BitReg/, false /fSignExtend/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
2177
2178	/** @todo What to do if immediate is != 0? */
2179
2180	if (rcStrict == VINF_SUCCESS)
2181	pVCpu->cpum.GstCtx.Pc.u64 += sizeof(uint32_t);
2182
2183	return rcStrict;
2184	}
2185
2186
2187	/**
2188	* Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException).
2189	*
2190	* @returns Strict VBox status code.
2191	* @param pVM The cross context VM structure.
2192	* @param pVCpu The cross context per CPU structure.
2193	* @param pExit The VM exit information to handle.
2194	* @sa nemHCWinHandleMessageUnrecoverableException
2195	*/
2196	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVMCC pVM, PVMCPUCC pVCpu, MY_WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2197	{
2198	#if 0
2199	/*
2200	* Just copy the state we've got and handle it in the loop for now.
2201	*/
2202	nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
2203	Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
2204	pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
2205	RT_NOREF_PV(pVM);
2206	return VINF_EM_TRIPLE_FAULT;
2207	#else
2208	/*
2209	* Let IEM decide whether this is really it.
2210	*/
2211	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
2212	pExit->UnrecoverableException.Header.Pc, ASMReadTSC());
2213	nemR3WinCopyStateFromArmHeader(pVCpu, &pExit->UnrecoverableException.Header);
2214	AssertReleaseFailed();
2215	RT_NOREF_PV(pVM);
2216	return VINF_SUCCESS;
2217	#endif
2218	}
2219
2220
2221	/**
2222	* Handles VM exits.
2223	*
2224	* @returns Strict VBox status code.
2225	* @param pVM The cross context VM structure.
2226	* @param pVCpu The cross context per CPU structure.
2227	* @param pExit The VM exit information to handle.
2228	* @sa nemHCWinHandleMessage
2229	*/
2230	NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVMCC pVM, PVMCPUCC pVCpu, MY_WHV_RUN_VP_EXIT_CONTEXT const *pExit)
2231	{
2232	#ifdef LOG_ENABLED
2233	if (LogIs3Enabled())
2234	{
2235	int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, CPUMCTX_EXTRN_ALL);
2236	AssertRCReturn(rc, rc);
2237
2238	nemR3WinLogState(pVM, pVCpu);
2239	}
2240	#endif
2241
2242	switch (pExit->ExitReason)
2243	{
2244	case WHvRunVpExitReasonUnmappedGpa:
2245	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
2246	return nemR3WinHandleExitMemory(pVM, pVCpu, pExit);
2247
2248	case WHvRunVpExitReasonCanceled:
2249	Log4(("CanceledExit/%u\n", pVCpu->idCpu));
2250	return VINF_SUCCESS;
2251
2252	case WHvRunVpExitReasonHypercall:
2253	return nemR3WinHandleExitHypercall(pVM, pVCpu, pExit);
2254
2255	case 0x8001000c: /* WHvRunVpExitReasonArm64Reset */
2256	{
2257	if (pExit->Arm64Reset.ResetType == WHV_ARM64_RESET_CONTEXT_TYPE_POWER_OFF)
2258	return VMR3PowerOff(pVM->pUVM);
2259	else if (pExit->Arm64Reset.ResetType == WHV_ARM64_RESET_CONTEXT_TYPE_RESET)
2260	{
2261	VM_FF_SET(pVM, VM_FF_RESET);
2262	return VINF_EM_RESET;
2263	}
2264	else
2265	AssertLogRelFailedReturn(VERR_NEM_IPE_3);
2266	}
2267
2268	case WHvRunVpExitReasonUnrecoverableException:
2269	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
2270	return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit);
2271
2272	case WHvRunVpExitReasonUnsupportedFeature:
2273	case WHvRunVpExitReasonInvalidVpRegisterValue:
2274	LogRel(("Unimplemented exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
2275	AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n",
2276	pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3);
2277
2278	/* Undesired exits: */
2279	case WHvRunVpExitReasonNone:
2280	default:
2281	LogRel(("Unknown exit:\n%.Rhxd\n", (int)sizeof(pExit), pExit));
2282	AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3);
2283	}
2284	}
2285
2286
2287	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2288	{
2289	LogFlow(("NEM/%u: %08RX64 pstate=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc, pVCpu->cpum.GstCtx.fPState));
2290	#ifdef LOG_ENABLED
2291	if (LogIs3Enabled())
2292	nemR3WinLogState(pVM, pVCpu);
2293	#endif
2294
2295	/*
2296	* Try switch to NEM runloop state.
2297	*/
2298	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2299	{ /* likely */ }
2300	else
2301	{
2302	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2303	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2304	return VINF_SUCCESS;
2305	}
2306
2307	if (pVCpu->nem.s.fSyncCntvRegs)
2308	{
2309	static const WHV_REGISTER_NAME s_aNames[2] = { WHvArm64RegisterCntvCtlEl0, WHvArm64RegisterCntvCvalEl0 };
2310	WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)];
2311	aRegs[0].Reg64 = pVCpu->cpum.GstCtx.CntvCtlEl0;
2312	aRegs[1].Reg64 = pVCpu->cpum.GstCtx.CntvCValEl0;
2313
2314	HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs);
2315	AssertLogRelMsgReturn(SUCCEEDED(hrc),
2316	("WHvSetVirtualProcessorRegisters(%p, 0,{CNTV_CTL_EL0, CNTV_CVAL_EL0}, 2,) -> %Rhrc (Last=%#x/%u)\n",
2317	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
2318	, VERR_NEM_IPE_9);
2319	pVCpu->nem.s.fSyncCntvRegs = false;
2320	}
2321
2322
2323	/*
2324	* The run loop.
2325	*
2326	* Current approach to state updating to use the sledgehammer and sync
2327	* everything every time. This will be optimized later.
2328	*/
2329	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2330	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2331	for (unsigned iLoop = 0;; iLoop++)
2332	{
2333	/*
2334	* Poll timers and run for a bit.
2335	*
2336	* With the VID approach (ring-0 or ring-3) we can specify a timeout here,
2337	* so we take the time of the next timer event and uses that as a deadline.
2338	* The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
2339	*/
2340	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2341	* the whole polling job when timers have changed... */
2342	uint64_t offDeltaIgnored;
2343	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2344	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2345	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2346	{
2347	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
2348	{
2349	/* Ensure that Hyper-V has the whole state. */
2350	int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu);
2351	AssertRCReturn(rc2, rc2);
2352
2353	#ifdef LOG_ENABLED
2354	if (LogIsFlowEnabled())
2355	{
2356	static const WHV_REGISTER_NAME s_aNames[2] = { WHvArm64RegisterPc, WHvArm64RegisterPstate };
2357	WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)] = { { { {0, 0} } } };
2358	WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs);
2359	LogFlow(("NEM/%u: Entry @ %08RX64 pstate=%#RX64\n", pVCpu->idCpu, aRegs[0].Reg64, aRegs[1].Reg64));
2360	}
2361	#endif
2362
2363	MY_WHV_RUN_VP_EXIT_CONTEXT ExitReason = {0};
2364	TMNotifyStartOfExecution(pVM, pVCpu);
2365
2366	HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
2367
2368	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
2369	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
2370	#ifdef LOG_ENABLED
2371	if (LogIsFlowEnabled())
2372	{
2373	static const WHV_REGISTER_NAME s_aNames[2] = { WHvArm64RegisterPc, WHvArm64RegisterPstate };
2374	WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)] = { { { {0, 0} } } };
2375	WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs);
2376	LogFlow(("NEM/%u: Exit @ %08RX64 pstate=%#RX64 Reason=%#x\n",
2377	pVCpu->idCpu, aRegs[0].Reg64, aRegs[1].Reg64, ExitReason.ExitReason));
2378	}
2379	#endif
2380	if (SUCCEEDED(hrc))
2381	{
2382	/* Always sync the CNTV_CTL_EL0/CNTV_CVAL_EL0 registers, just like we do on macOS. */
2383	static const WHV_REGISTER_NAME s_aNames[2] = { WHvArm64RegisterCntvCtlEl0, WHvArm64RegisterCntvCvalEl0 };
2384	WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)] = { { { {0, 0} } } };
2385	hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs);
2386	AssertLogRelMsgReturn(SUCCEEDED(hrc),
2387	("WHvGetVirtualProcessorRegisters(%p, 0,{CNTV_CTL_EL0, CNTV_CVAL_EL0}, 2,) -> %Rhrc (Last=%#x/%u)\n",
2388	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
2389	, VERR_NEM_IPE_9);
2390
2391	pVCpu->cpum.GstCtx.CntvCtlEl0 = aRegs[0].Reg64;
2392	pVCpu->cpum.GstCtx.CntvCValEl0 = aRegs[1].Reg64;
2393
2394	/*
2395	* Deal with the message.
2396	*/
2397	rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason);
2398	if (rcStrict == VINF_SUCCESS)
2399	{ /* hopefully likely */ }
2400	else
2401	{
2402	LogFlow(("NEM/%u: breaking: nemR3WinHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2403	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2404	break;
2405	}
2406	}
2407	else
2408	AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n",
2409	pVCpu->idCpu, hrc, GetLastError()),
2410	VERR_NEM_IPE_0);
2411
2412	/*
2413	* If no relevant FFs are pending, loop.
2414	*/
2415	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2416	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2417	continue;
2418
2419	/** @todo Try handle pending flags, not just return to EM loops. Take care
2420	* not to set important RCs here unless we've handled a message. */
2421	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2422	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2423	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2424	}
2425	else
2426	{
2427	LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
2428	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
2429	}
2430	}
2431	else
2432	{
2433	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2434	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2435	}
2436	break;
2437	} /* the run loop */
2438
2439
2440	/*
2441	* If the CPU is running, make sure to stop it before we try sync back the
2442	* state and return to EM. We don't sync back the whole state if we can help it.
2443	*/
2444	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2445	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2446
2447	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
2448	{
2449	/* Try anticipate what we might need. */
2450	uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK;
2451	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2452	\|\| RT_FAILURE(rcStrict))
2453	fImport = CPUMCTX_EXTRN_ALL;
2454	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ \| VMCPU_FF_INTERRUPT_FIQ))
2455	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2456
2457	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2458	{
2459	int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport);
2460	if (RT_SUCCESS(rc2))
2461	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2462	else if (RT_SUCCESS(rcStrict))
2463	rcStrict = rc2;
2464	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2465	pVCpu->cpum.GstCtx.fExtrn = 0;
2466	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2467	}
2468	else
2469	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2470	}
2471	else
2472	{
2473	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2474	pVCpu->cpum.GstCtx.fExtrn = 0;
2475	}
2476
2477	#if 0
2478	UINT32 cbWritten;
2479	WHV_ARM64_LOCAL_INTERRUPT_CONTROLLER_STATE IntrState;
2480	HRESULT hrc = WHvGetVirtualProcessorState(pVM->nem.s.hPartition, pVCpu->idCpu, WHvVirtualProcessorStateTypeInterruptControllerState2,
2481	&IntrState, sizeof(IntrState), &cbWritten);
2482	AssertLogRelMsgReturn(SUCCEEDED(hrc),
2483	("WHvGetVirtualProcessorState(%p, %u,WHvVirtualProcessorStateTypeInterruptControllerState2,) -> %Rhrc (Last=%#x/%u)\n",
2484	pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
2485	, VERR_NEM_GET_REGISTERS_FAILED);
2486	LogFlowFunc(("IntrState: cbWritten=%u\n"));
2487	for (uint32_t i = 0; i < RT_ELEMENTS(IntrState.BankedInterruptState); i++)
2488	{
2489	WHV_ARM64_INTERRUPT_STATE *pState = &IntrState.BankedInterruptState[i];
2490	LogFlowFunc(("IntrState: Intr %u:\n"
2491	" Enabled=%RTbool\n"
2492	" EdgeTriggered=%RTbool\n"
2493	" Asserted=%RTbool\n"
2494	" SetPending=%RTbool\n"
2495	" Active=%RTbool\n"
2496	" Direct=%RTbool\n"
2497	" GicrIpriorityrConfigured=%u\n"
2498	" GicrIpriorityrActive=%u\n",
2499	i, pState->Enabled, pState->EdgeTriggered, pState->Asserted, pState->SetPending, pState->Active, pState->Direct,
2500	pState->GicrIpriorityrConfigured, pState->GicrIpriorityrActive));
2501	}
2502	#endif
2503
2504	LogFlow(("NEM/%u: %08RX64 pstate=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc.u64,
2505	pVCpu->cpum.GstCtx.fPState, VBOXSTRICTRC_VAL(rcStrict) ));
2506	return rcStrict;
2507	}
2508
2509
2510	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
2511	{
2512	Assert(VM_IS_NEM_ENABLED(pVM));
2513	RT_NOREF(pVM, pVCpu);
2514	return true;
2515	}
2516
2517
2518	VMMR3_INT_DECL(int) NEMR3Halt(PVM pVM, PVMCPU pVCpu)
2519	{
2520	Assert(EMGetState(pVCpu) == EMSTATE_WAIT_SIPI);
2521
2522	/*
2523	* Force the vCPU to get out of the SIPI state and into the normal runloop
2524	* as Hyper-V doesn't cause VM exits for PSCI calls so we wouldn't notice when
2525	* when the guest brings APs online.
2526	* Instead we force the EMT to run the vCPU through Hyper-V which manages the state.
2527	*/
2528	RT_NOREF(pVM);
2529	EMSetState(pVCpu, EMSTATE_HALTED);
2530	return VINF_EM_RESCHEDULE;
2531	}
2532
2533
2534	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
2535	{
2536	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
2537	return false;
2538	}
2539
2540
2541	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
2542	{
2543	Log8(("nemR3NativeNotifyFF: canceling %u\n", pVCpu->idCpu));
2544	if (pVM->nem.s.fCreatedEmts)
2545	{
2546	HRESULT hrc = WHvCancelRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, 0);
2547	AssertMsg(SUCCEEDED(hrc), ("WHvCancelRunVirtualProcessor -> hrc=%Rhrc\n", hrc));
2548	RT_NOREF_PV(hrc);
2549	}
2550	RT_NOREF_PV(fFlags);
2551	}
2552
2553
2554	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
2555	{
2556	RT_NOREF(pVM, fUseDebugLoop);
2557	return false;
2558	}
2559
2560
2561	DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
2562	{
2563	RT_NOREF(pVM, pVCpu, fUseDebugLoop);
2564	return false;
2565	}
2566
2567
2568	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
2569	{
2570	PGMPAGEMAPLOCK Lock;
2571	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
2572	if (RT_SUCCESS(rc))
2573	PGMPhysReleasePageMappingLock(pVM, &Lock);
2574	return rc;
2575	}
2576
2577
2578	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
2579	{
2580	PGMPAGEMAPLOCK Lock;
2581	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
2582	if (RT_SUCCESS(rc))
2583	PGMPhysReleasePageMappingLock(pVM, &Lock);
2584	return rc;
2585	}
2586
2587
2588	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
2589	uint8_t pu2State, uint32_t puNemRange)
2590	{
2591	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p pu2State=%p (%d) puNemRange=%p (%d)\n",
2592	GCPhys, cb, pvR3, pu2State, pu2State, puNemRange, *puNemRange));
2593
2594	*pu2State = UINT8_MAX;
2595	RT_NOREF(puNemRange);
2596
2597	if (pvR3)
2598	{
2599	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
2600	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvR3, GCPhys, cb,
2601	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagWrite \| WHvMapGpaRangeFlagExecute);
2602	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
2603	if (SUCCEEDED(hrc))
2604	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2605	else
2606	{
2607	LogRel(("NEMR3NotifyPhysRamRegister: GCPhys=%RGp LB %RGp pvR3=%p hrc=%Rhrc (%#x) Last=%#x/%u\n",
2608	GCPhys, cb, pvR3, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2609	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
2610	return VERR_NEM_MAP_PAGES_FAILED;
2611	}
2612	}
2613	return VINF_SUCCESS;
2614	}
2615
2616
2617	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
2618	{
2619	RT_NOREF(pVM);
2620	return g_pfnWHvQueryGpaRangeDirtyBitmap != NULL;
2621	}
2622
2623
2624	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
2625	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
2626	{
2627	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d) puNemRange=%p (%#x)\n",
2628	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, pu2State, puNemRange, puNemRange ? puNemRange : UINT32_MAX));
2629	RT_NOREF(puNemRange);
2630
2631	/*
2632	* Unmap the RAM we're replacing.
2633	*/
2634	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
2635	{
2636	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRange, a);
2637	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, cb);
2638	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRange, a);
2639	if (SUCCEEDED(hrc))
2640	{ /* likely */ }
2641	else if (pvMmio2)
2642	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> hrc=%Rhrc (%#x) Last=%#x/%u (ignored)\n",
2643	GCPhys, cb, fFlags, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2644	else
2645	{
2646	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> hrc=%Rhrc (%#x) Last=%#x/%u\n",
2647	GCPhys, cb, fFlags, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2648	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
2649	return VERR_NEM_UNMAP_PAGES_FAILED;
2650	}
2651	}
2652
2653	/*
2654	* Map MMIO2 if any.
2655	*/
2656	if (pvMmio2)
2657	{
2658	Assert(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2);
2659	WHV_MAP_GPA_RANGE_FLAGS fWHvFlags = WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagWrite \| WHvMapGpaRangeFlagExecute;
2660	if ((fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES) && g_pfnWHvQueryGpaRangeDirtyBitmap)
2661	fWHvFlags \|= WHvMapGpaRangeFlagTrackDirtyPages;
2662	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
2663	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvMmio2, GCPhys, cb, fWHvFlags);
2664	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
2665	if (SUCCEEDED(hrc))
2666	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2667	else
2668	{
2669	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x pvMmio2=%p fWHvFlags=%#x: Map -> hrc=%Rhrc (%#x) Last=%#x/%u\n",
2670	GCPhys, cb, fFlags, pvMmio2, fWHvFlags, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2671	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
2672	return VERR_NEM_MAP_PAGES_FAILED;
2673	}
2674	}
2675	else
2676	{
2677	Assert(!(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2));
2678	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2679	}
2680	RT_NOREF(pvRam);
2681	return VINF_SUCCESS;
2682	}
2683
2684
2685	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
2686	void pvRam, void pvMmio2, uint32_t *puNemRange)
2687	{
2688	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
2689	return VINF_SUCCESS;
2690	}
2691
2692
2693	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
2694	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
2695	{
2696	int rc = VINF_SUCCESS;
2697	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p uNemRange=%#x (%#x)\n",
2698	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
2699
2700	/*
2701	* Unmap the MMIO2 pages.
2702	*/
2703	/** @todo If we implement aliasing (MMIO2 page aliased into MMIO range),
2704	* we may have more stuff to unmap even in case of pure MMIO... */
2705	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
2706	{
2707	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRange, a);
2708	HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, cb);
2709	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRange, a);
2710	if (FAILED(hrc))
2711	{
2712	LogRel2(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> hrc=%Rhrc (%#x) Last=%#x/%u (ignored)\n",
2713	GCPhys, cb, fFlags, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2714	rc = VERR_NEM_UNMAP_PAGES_FAILED;
2715	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
2716	}
2717	}
2718
2719	/*
2720	* Restore the RAM we replaced.
2721	*/
2722	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
2723	{
2724	AssertPtr(pvRam);
2725	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
2726	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvRam, GCPhys, cb,
2727	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagWrite \| WHvMapGpaRangeFlagExecute);
2728	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
2729	if (SUCCEEDED(hrc))
2730	{ /* likely */ }
2731	else
2732	{
2733	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp pvMmio2=%p hrc=%Rhrc (%#x) Last=%#x/%u\n",
2734	GCPhys, cb, pvMmio2, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2735	rc = VERR_NEM_MAP_PAGES_FAILED;
2736	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
2737	}
2738	if (pu2State)
2739	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2740	}
2741	/* Mark the pages as unmapped if relevant. */
2742	else if (pu2State)
2743	*pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
2744
2745	RT_NOREF(pvMmio2, puNemRange);
2746	return rc;
2747	}
2748
2749
2750	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
2751	void *pvBitmap, size_t cbBitmap)
2752	{
2753	Assert(VM_IS_NEM_ENABLED(pVM));
2754	AssertReturn(g_pfnWHvQueryGpaRangeDirtyBitmap, VERR_INTERNAL_ERROR_2);
2755	Assert(cbBitmap == (uint32_t)cbBitmap);
2756	RT_NOREF(uNemRange);
2757
2758	/* This is being profiled by PGM, see /PGM/Mmio2QueryAndResetDirtyBitmap. */
2759	HRESULT hrc = WHvQueryGpaRangeDirtyBitmap(pVM->nem.s.hPartition, GCPhys, cb, (UINT64 *)pvBitmap, (uint32_t)cbBitmap);
2760	if (SUCCEEDED(hrc))
2761	return VINF_SUCCESS;
2762
2763	AssertLogRelMsgFailed(("GCPhys=%RGp LB %RGp pvBitmap=%p LB %#zx hrc=%Rhrc (%#x) Last=%#x/%u\n",
2764	GCPhys, cb, pvBitmap, cbBitmap, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2765	return VERR_NEM_QUERY_DIRTY_BITMAP_FAILED;
2766	}
2767
2768
2769	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
2770	uint8_t pu2State, uint32_t puNemRange)
2771	{
2772	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
2773
2774	Log5(("NEMR3NotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
2775	*pu2State = UINT8_MAX;
2776	*puNemRange = 0;
2777	return VINF_SUCCESS;
2778	}
2779
2780
2781	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
2782	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
2783	{
2784	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
2785	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
2786	*pu2State = UINT8_MAX;
2787
2788	/*
2789	* (Re-)map readonly.
2790	*/
2791	AssertPtrReturn(pvPages, VERR_INVALID_POINTER);
2792	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
2793	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPages, GCPhys, cb, WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute);
2794	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
2795	if (SUCCEEDED(hrc))
2796	*pu2State = NEM_WIN_PAGE_STATE_READABLE;
2797	else
2798	{
2799	LogRel(("nemR3NativeNotifyPhysRomRegisterEarly: GCPhys=%RGp LB %RGp pvPages=%p fFlags=%#x hrc=%Rhrc (%#x) Last=%#x/%u\n",
2800	GCPhys, cb, pvPages, fFlags, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
2801	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
2802	return VERR_NEM_MAP_PAGES_FAILED;
2803	}
2804	RT_NOREF(fFlags, puNemRange);
2805	return VINF_SUCCESS;
2806	}
2807
2808	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
2809	{
2810	Log(("nemR3NativeNotifySetA20: fEnabled=%RTbool\n", fEnabled));
2811	Assert(VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)));
2812	RT_NOREF(pVCpu, fEnabled);
2813	}
2814
2815
2816	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
2817	{
2818	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
2819	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
2820	}
2821
2822
2823	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
2824	RTR3PTR pvMemR3, uint8_t *pu2State)
2825	{
2826	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
2827	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
2828
2829	*pu2State = UINT8_MAX;
2830	if (pvMemR3)
2831	{
2832	STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
2833	HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvMemR3, GCPhys, cb,
2834	WHvMapGpaRangeFlagRead \| WHvMapGpaRangeFlagExecute \| WHvMapGpaRangeFlagWrite);
2835	STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
2836	if (SUCCEEDED(hrc))
2837	*pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
2838	else
2839	AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: WHvMapGpaRange(,%p,%RGp,%RGp,) -> %Rhrc\n",
2840	pvMemR3, GCPhys, cb, hrc));
2841	}
2842	RT_NOREF(enmKind);
2843	}
2844
2845
2846	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
2847	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
2848	{
2849	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
2850	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
2851	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
2852	}
2853
2854
2855	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
2856	PGMPAGETYPE enmType, uint8_t *pu2State)
2857	{
2858	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
2859	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
2860	RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
2861
2862	AssertFailed();
2863	return VINF_SUCCESS;
2864	}
2865
2866
2867	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
2868	PGMPAGETYPE enmType, uint8_t *pu2State)
2869	{
2870	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
2871	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
2872	RT_NOREF(pVM, GCPhys, HCPhys, pvR3, fPageProt, enmType, pu2State);
2873	}
2874
2875
2876	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
2877	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
2878	{
2879	Log5(("NEMHCNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
2880	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
2881	RT_NOREF(pVM, GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, pu2State);
2882
2883	AssertFailed();
2884	}
2885
2886
2887	/**
2888	* Returns features supported by the NEM backend.
2889	*
2890	* @returns Flags of features supported by the native NEM backend.
2891	* @param pVM The cross context VM structure.
2892	*/
2893	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
2894	{
2895	RT_NOREF(pVM);
2896	/** @todo Is NEM_FEAT_F_FULL_GST_EXEC always true? */
2897	return NEM_FEAT_F_NESTED_PAGING \| NEM_FEAT_F_FULL_GST_EXEC;
2898	}
2899
2900
2901	/** @page pg_nem_win_aarmv8 NEM/win - Native Execution Manager, Windows.
2902	*
2903	* Open questions:
2904	* - InstructionByteCount and InstructionBytes for unmapped GPA exit are zero...
2905	*/
2906

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-win-armv8.cpp@ 108763

Download in other formats: