VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux-armv8.cpp@ 108475

Last change on this file since 108475 was 108475, checked in by vboxsync, 8 weeks ago

VMM/NEMR3Native-linux-armv8.cpp: Allow building on hosts which don't ship a recent enough kvm.h, bugref:10391
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 62.6 KB
Line 
1/* $Id: NEMR3Native-linux-armv8.cpp 108475 2025-03-07 12:32:07Z vboxsync $ */
2/** @file
3 * NEM - Native execution manager, native ring-3 Linux backend arm64 version.
4 */
5
6/*
7 * Copyright (C) 2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_NEM
33#define VMCPU_INCL_CPUM_GST_CTX
34#include <VBox/vmm/nem.h>
35#include <VBox/vmm/iem.h>
36#include <VBox/vmm/em.h>
37#include <VBox/vmm/pdmgic.h>
38#include <VBox/vmm/pdm.h>
39#include <VBox/vmm/trpm.h>
40#include "NEMInternal.h"
41#include <VBox/vmm/vmcc.h>
42
43#include <iprt/alloca.h>
44#include <iprt/string.h>
45#include <iprt/system.h>
46#include <iprt/armv8.h>
47
48#include <iprt/formats/arm-psci.h>
49
50#include <errno.h>
51#include <unistd.h>
52#include <sys/ioctl.h>
53#include <sys/fcntl.h>
54#include <sys/mman.h>
55#include <linux/kvm.h>
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61
62/** Core register group. */
63#define KVM_ARM64_REG_CORE_GROUP UINT64_C(0x6030000000100000)
64/** System register group. */
65#define KVM_ARM64_REG_SYS_GROUP UINT64_C(0x6030000000130000)
66/** System register group. */
67#define KVM_ARM64_REG_SIMD_GROUP UINT64_C(0x6040000000100050)
68/** FP register group. */
69#define KVM_ARM64_REG_FP_GROUP UINT64_C(0x6020000000100000)
70
71#define KVM_ARM64_REG_CORE_CREATE(a_idReg) (KVM_ARM64_REG_CORE_GROUP | ((uint64_t)(a_idReg) & 0xffff))
72#define KVM_ARM64_REG_GPR(a_iGpr) KVM_ARM64_REG_CORE_CREATE((a_iGpr) << 1)
73#define KVM_ARM64_REG_SP_EL0 KVM_ARM64_REG_CORE_CREATE(0x3e)
74#define KVM_ARM64_REG_PC KVM_ARM64_REG_CORE_CREATE(0x40)
75#define KVM_ARM64_REG_PSTATE KVM_ARM64_REG_CORE_CREATE(0x42)
76#define KVM_ARM64_REG_SP_EL1 KVM_ARM64_REG_CORE_CREATE(0x44)
77#define KVM_ARM64_REG_ELR_EL1 KVM_ARM64_REG_CORE_CREATE(0x46)
78#define KVM_ARM64_REG_SPSR_EL1 KVM_ARM64_REG_CORE_CREATE(0x48)
79#define KVM_ARM64_REG_SPSR_ABT KVM_ARM64_REG_CORE_CREATE(0x4a)
80#define KVM_ARM64_REG_SPSR_UND KVM_ARM64_REG_CORE_CREATE(0x4c)
81#define KVM_ARM64_REG_SPSR_IRQ KVM_ARM64_REG_CORE_CREATE(0x4e)
82#define KVM_ARM64_REG_SPSR_FIQ KVM_ARM64_REG_CORE_CREATE(0x50)
83
84/** This maps to our IPRT representation of system register IDs, yay! */
85#define KVM_ARM64_REG_SYS_CREATE(a_idSysReg) (KVM_ARM64_REG_SYS_GROUP | ((uint64_t)(a_idSysReg) & 0xffff))
86
87#define KVM_ARM64_REG_SIMD_CREATE(a_iVecReg) (KVM_ARM64_REG_SIMD_GROUP | (((uint64_t)(a_iVecReg) << 2) & 0xffff))
88
89#define KVM_ARM64_REG_FP_CREATE(a_idReg) (KVM_ARM64_REG_FP_GROUP | ((uint64_t)(a_idReg) & 0xffff))
90#define KVM_ARM64_REG_FP_FPSR KVM_ARM64_REG_FP_CREATE(0xd4)
91#define KVM_ARM64_REG_FP_FPCR KVM_ARM64_REG_FP_CREATE(0xd5)
92
93/* In case the build host doesn't ship with a new enough kvm.h. */
94#ifndef KVM_ARM_VM_SMCCC_CTRL
95# define KVM_ARM_VM_SMCCC_CTRL 0
96# define KVM_ARM_VM_SMCCC_FILTER 0
97
98# define KVM_SMCCC_FILTER_HANDLE 0
99# define KVM_SMCCC_FILTER_DENY 1
100# define KVM_SMCCC_FILTER_FWD_TO_USER 2
101
102struct kvm_smccc_filter
103{
104 uint32_t base;
105 uint32_t nr_functions;
106 uint8_t action;
107 uint8_t pad[15];
108};
109#endif
110
111
112/*********************************************************************************************************************************
113* Structures and Typedefs *
114*********************************************************************************************************************************/
115
116
117/*********************************************************************************************************************************
118* Global Variables *
119*********************************************************************************************************************************/
120/** The general registers. */
121static const struct
122{
123 uint64_t idKvmReg;
124 uint32_t fCpumExtrn;
125 uint32_t offCpumCtx;
126} s_aCpumRegs[] =
127{
128#define CPUM_GREG_EMIT_X0_X3(a_Idx) { KVM_ARM64_REG_GPR(a_Idx), CPUMCTX_EXTRN_X ## a_Idx, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
129#define CPUM_GREG_EMIT_X4_X28(a_Idx) { KVM_ARM64_REG_GPR(a_Idx), CPUMCTX_EXTRN_X4_X28, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
130 CPUM_GREG_EMIT_X0_X3(0),
131 CPUM_GREG_EMIT_X0_X3(1),
132 CPUM_GREG_EMIT_X0_X3(2),
133 CPUM_GREG_EMIT_X0_X3(3),
134 CPUM_GREG_EMIT_X4_X28(4),
135 CPUM_GREG_EMIT_X4_X28(5),
136 CPUM_GREG_EMIT_X4_X28(6),
137 CPUM_GREG_EMIT_X4_X28(7),
138 CPUM_GREG_EMIT_X4_X28(8),
139 CPUM_GREG_EMIT_X4_X28(9),
140 CPUM_GREG_EMIT_X4_X28(10),
141 CPUM_GREG_EMIT_X4_X28(11),
142 CPUM_GREG_EMIT_X4_X28(12),
143 CPUM_GREG_EMIT_X4_X28(13),
144 CPUM_GREG_EMIT_X4_X28(14),
145 CPUM_GREG_EMIT_X4_X28(15),
146 CPUM_GREG_EMIT_X4_X28(16),
147 CPUM_GREG_EMIT_X4_X28(17),
148 CPUM_GREG_EMIT_X4_X28(18),
149 CPUM_GREG_EMIT_X4_X28(19),
150 CPUM_GREG_EMIT_X4_X28(20),
151 CPUM_GREG_EMIT_X4_X28(21),
152 CPUM_GREG_EMIT_X4_X28(22),
153 CPUM_GREG_EMIT_X4_X28(23),
154 CPUM_GREG_EMIT_X4_X28(24),
155 CPUM_GREG_EMIT_X4_X28(25),
156 CPUM_GREG_EMIT_X4_X28(26),
157 CPUM_GREG_EMIT_X4_X28(27),
158 CPUM_GREG_EMIT_X4_X28(28),
159 { KVM_ARM64_REG_GPR(29), CPUMCTX_EXTRN_FP, RT_UOFFSETOF(CPUMCTX, aGRegs[29].x) },
160 { KVM_ARM64_REG_GPR(30), CPUMCTX_EXTRN_LR, RT_UOFFSETOF(CPUMCTX, aGRegs[30].x) },
161 { KVM_ARM64_REG_PC, CPUMCTX_EXTRN_PC, RT_UOFFSETOF(CPUMCTX, Pc.u64) },
162#undef CPUM_GREG_EMIT_X0_X3
163#undef CPUM_GREG_EMIT_X4_X28
164};
165/** SIMD/FP registers. */
166static const struct
167{
168 uint64_t idKvmReg;
169 uint32_t offCpumCtx;
170} s_aCpumFpRegs[] =
171{
172#define CPUM_VREG_EMIT(a_Idx) { KVM_ARM64_REG_SIMD_CREATE(a_Idx), RT_UOFFSETOF(CPUMCTX, aVRegs[a_Idx].v) }
173 CPUM_VREG_EMIT(0),
174 CPUM_VREG_EMIT(1),
175 CPUM_VREG_EMIT(2),
176 CPUM_VREG_EMIT(3),
177 CPUM_VREG_EMIT(4),
178 CPUM_VREG_EMIT(5),
179 CPUM_VREG_EMIT(6),
180 CPUM_VREG_EMIT(7),
181 CPUM_VREG_EMIT(8),
182 CPUM_VREG_EMIT(9),
183 CPUM_VREG_EMIT(10),
184 CPUM_VREG_EMIT(11),
185 CPUM_VREG_EMIT(12),
186 CPUM_VREG_EMIT(13),
187 CPUM_VREG_EMIT(14),
188 CPUM_VREG_EMIT(15),
189 CPUM_VREG_EMIT(16),
190 CPUM_VREG_EMIT(17),
191 CPUM_VREG_EMIT(18),
192 CPUM_VREG_EMIT(19),
193 CPUM_VREG_EMIT(20),
194 CPUM_VREG_EMIT(21),
195 CPUM_VREG_EMIT(22),
196 CPUM_VREG_EMIT(23),
197 CPUM_VREG_EMIT(24),
198 CPUM_VREG_EMIT(25),
199 CPUM_VREG_EMIT(26),
200 CPUM_VREG_EMIT(27),
201 CPUM_VREG_EMIT(28),
202 CPUM_VREG_EMIT(29),
203 CPUM_VREG_EMIT(30),
204 CPUM_VREG_EMIT(31)
205#undef CPUM_VREG_EMIT
206};
207/** System registers. */
208static const struct
209{
210 uint64_t idKvmReg;
211 uint32_t fCpumExtrn;
212 uint32_t offCpumCtx;
213} s_aCpumSysRegs[] =
214{
215 { KVM_ARM64_REG_SP_EL0, CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[0].u64) },
216 { KVM_ARM64_REG_SP_EL1, CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[1].u64) },
217 { KVM_ARM64_REG_SPSR_EL1, CPUMCTX_EXTRN_SPSR, RT_UOFFSETOF(CPUMCTX, Spsr.u64) },
218 { KVM_ARM64_REG_ELR_EL1, CPUMCTX_EXTRN_ELR, RT_UOFFSETOF(CPUMCTX, Elr.u64) },
219 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_SCTRL_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Sctlr.u64) },
220 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TCR_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Tcr.u64) },
221 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TTBR0_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Ttbr0.u64) },
222 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TTBR1_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Ttbr1.u64) },
223 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_VBAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, VBar.u64) },
224 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AFSR0_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr0.u64) },
225 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AFSR1_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr1.u64) },
226 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AMAIR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Amair.u64) },
227 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CNTKCTL_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, CntKCtl.u64) },
228 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CONTEXTIDR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, ContextIdr.u64) },
229 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CPACR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Cpacr.u64) },
230 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CSSELR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Csselr.u64) },
231 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ESR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Esr.u64) },
232 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_FAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Far.u64) },
233 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MAIR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Mair.u64) },
234 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_PAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Par.u64) },
235 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDRRO_EL0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, TpIdrRoEl0.u64) },
236 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDR_EL0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[0].u64) },
237 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[1].u64) },
238 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MDCCINT_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, MDccInt.u64) }
239};
240/** Debug system registers. */
241static const struct
242{
243 uint64_t idKvmReg;
244 uint32_t offCpumCtx;
245} s_aCpumDbgRegs[] =
246{
247#define CPUM_DBGREG_EMIT(a_BorW, a_Idx) \
248 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_DBG ## a_BorW ## CRn_EL1(a_Idx)), RT_UOFFSETOF(CPUMCTX, a ## a_BorW ## p[a_Idx].Ctrl.u64) }, \
249 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_DBG ## a_BorW ## VRn_EL1(a_Idx)), RT_UOFFSETOF(CPUMCTX, a ## a_BorW ## p[a_Idx].Value.u64) }
250 /* Breakpoint registers. */
251 CPUM_DBGREG_EMIT(B, 0),
252 CPUM_DBGREG_EMIT(B, 1),
253 CPUM_DBGREG_EMIT(B, 2),
254 CPUM_DBGREG_EMIT(B, 3),
255 CPUM_DBGREG_EMIT(B, 4),
256 CPUM_DBGREG_EMIT(B, 5),
257 CPUM_DBGREG_EMIT(B, 6),
258 CPUM_DBGREG_EMIT(B, 7),
259 CPUM_DBGREG_EMIT(B, 8),
260 CPUM_DBGREG_EMIT(B, 9),
261 CPUM_DBGREG_EMIT(B, 10),
262 CPUM_DBGREG_EMIT(B, 11),
263 CPUM_DBGREG_EMIT(B, 12),
264 CPUM_DBGREG_EMIT(B, 13),
265 CPUM_DBGREG_EMIT(B, 14),
266 CPUM_DBGREG_EMIT(B, 15),
267 /* Watchpoint registers. */
268 CPUM_DBGREG_EMIT(W, 0),
269 CPUM_DBGREG_EMIT(W, 1),
270 CPUM_DBGREG_EMIT(W, 2),
271 CPUM_DBGREG_EMIT(W, 3),
272 CPUM_DBGREG_EMIT(W, 4),
273 CPUM_DBGREG_EMIT(W, 5),
274 CPUM_DBGREG_EMIT(W, 6),
275 CPUM_DBGREG_EMIT(W, 7),
276 CPUM_DBGREG_EMIT(W, 8),
277 CPUM_DBGREG_EMIT(W, 9),
278 CPUM_DBGREG_EMIT(W, 10),
279 CPUM_DBGREG_EMIT(W, 11),
280 CPUM_DBGREG_EMIT(W, 12),
281 CPUM_DBGREG_EMIT(W, 13),
282 CPUM_DBGREG_EMIT(W, 14),
283 CPUM_DBGREG_EMIT(W, 15),
284 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MDSCR_EL1), RT_UOFFSETOF(CPUMCTX, Mdscr.u64) }
285#undef CPUM_DBGREG_EMIT
286};
287/** PAuth key system registers. */
288static const struct
289{
290 uint64_t idKvmReg;
291 uint32_t offCpumCtx;
292} s_aCpumPAuthKeyRegs[] =
293{
294 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apda.Low.u64) },
295 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apda.High.u64) },
296 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDBKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apdb.Low.u64) },
297 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDBKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apdb.High.u64) },
298 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APGAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apga.Low.u64) },
299 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APGAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apga.High.u64) },
300 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apia.Low.u64) },
301 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apia.High.u64) },
302 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIBKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apib.Low.u64) },
303 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIBKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apib.High.u64) }
304};
305/** ID registers. */
306static const struct
307{
308 uint64_t idKvmReg;
309 uint32_t offIdStruct;
310} s_aIdRegs[] =
311{
312 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64DFR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Dfr0El1) },
313 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64DFR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Dfr1El1) },
314 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64ISAR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Isar0El1) },
315 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64ISAR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Isar1El1) },
316 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Mmfr0El1) },
317 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Mmfr1El1) },
318 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR2_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Mmfr2El1) },
319 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64PFR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Pfr0El1) },
320 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64PFR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Pfr1El1) }
321};
322
323
324
325/* Forward declarations of things called by the template. */
326static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo);
327
328
329/* Instantiate the common bits we share with the x86 KVM backend. */
330#include "NEMR3NativeTemplate-linux.cpp.h"
331
332
333/**
334 * Queries and logs the supported register list from KVM.
335 *
336 * @returns VBox status code.
337 * @param fdVCpu The file descriptor number of vCPU 0.
338 */
339static int nemR3LnxLogRegList(int fdVCpu)
340{
341 struct KVM_REG_LIST
342 {
343 uint64_t cRegs;
344 uint64_t aRegs[1024];
345 } RegList; RT_ZERO(RegList);
346
347 RegList.cRegs = RT_ELEMENTS(RegList.aRegs);
348 int rcLnx = ioctl(fdVCpu, KVM_GET_REG_LIST, &RegList);
349 if (rcLnx != 0)
350 return RTErrConvertFromErrno(errno);
351
352 LogRel(("NEM: KVM vCPU registers:\n"));
353
354 for (uint32_t i = 0; i < RegList.cRegs; i++)
355 LogRel(("NEM: %36s: %#RX64\n", "Unknown" /** @todo */, RegList.aRegs[i]));
356
357 return VINF_SUCCESS;
358}
359
360
361/**
362 * Sets the given attribute in KVM to the given value.
363 *
364 * @returns VBox status code.
365 * @param pVM The VM instance.
366 * @param u32Grp The device attribute group being set.
367 * @param u32Attr The actual attribute inside the group being set.
368 * @param pvAttrVal Where the attribute value to set.
369 * @param pszAttribute Attribute description for logging.
370 * @param pErrInfo Optional error information.
371 */
372static int nemR3LnxSetAttribute(PVM pVM, uint32_t u32Grp, uint32_t u32Attr, const void *pvAttrVal, const char *pszAttribute,
373 PRTERRINFO pErrInfo)
374{
375 struct kvm_device_attr DevAttr;
376
377 DevAttr.flags = 0;
378 DevAttr.group = u32Grp;
379 DevAttr.attr = u32Attr;
380 DevAttr.addr = (uintptr_t)pvAttrVal;
381 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_HAS_DEVICE_ATTR, &DevAttr);
382 if (rcLnx < 0)
383 return RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno),
384 N_("KVM error: KVM doesn't support setting the attribute \"%s\" (%d)"),
385 pszAttribute, errno);
386
387 rcLnx = ioctl(pVM->nem.s.fdVm, KVM_SET_DEVICE_ATTR, &DevAttr);
388 if (rcLnx < 0)
389 return RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno),
390 N_("KVM error: Setting the attribute \"%s\" for KVM failed (%d)"),
391 pszAttribute, errno);
392
393 return VINF_SUCCESS;
394}
395
396
397DECL_FORCE_INLINE(int) nemR3LnxKvmSetQueryReg(PVMCPUCC pVCpu, bool fQuery, uint64_t idKvmReg, const void *pv)
398{
399 struct kvm_one_reg Reg;
400 Reg.id = idKvmReg;
401 Reg.addr = (uintptr_t)pv;
402
403 /*
404 * Who thought that this API was a good idea? Supporting to query/set just one register
405 * at a time is horribly inefficient.
406 */
407 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, fQuery ? KVM_GET_ONE_REG : KVM_SET_ONE_REG, &Reg);
408 if (!rcLnx)
409 return 0;
410
411 return RTErrConvertFromErrno(-rcLnx);
412}
413
414DECLINLINE(int) nemR3LnxKvmQueryRegU64(PVMCPUCC pVCpu, uint64_t idKvmReg, uint64_t *pu64)
415{
416 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pu64);
417}
418
419
420DECLINLINE(int) nemR3LnxKvmQueryRegU32(PVMCPUCC pVCpu, uint64_t idKvmReg, uint32_t *pu32)
421{
422 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pu32);
423}
424
425
426DECLINLINE(int) nemR3LnxKvmQueryRegPV(PVMCPUCC pVCpu, uint64_t idKvmReg, void *pv)
427{
428 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pv);
429}
430
431
432DECLINLINE(int) nemR3LnxKvmSetRegU64(PVMCPUCC pVCpu, uint64_t idKvmReg, const uint64_t *pu64)
433{
434 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pu64);
435}
436
437
438DECLINLINE(int) nemR3LnxKvmSetRegU32(PVMCPUCC pVCpu, uint64_t idKvmReg, const uint32_t *pu32)
439{
440 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pu32);
441}
442
443
444DECLINLINE(int) nemR3LnxKvmSetRegPV(PVMCPUCC pVCpu, uint64_t idKvmReg, const void *pv)
445{
446 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pv);
447}
448
449
450/**
451 * Does the early setup of a KVM VM.
452 *
453 * @returns VBox status code.
454 * @param pVM The cross context VM structure.
455 * @param pErrInfo Where to always return error info.
456 */
457static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
458{
459 AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
460
461 /*
462 * Create the VCpus.
463 */
464 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
465 {
466 PVMCPU pVCpu = pVM->apCpusR3[idCpu];
467
468 /* Create it. */
469 pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
470 if (pVCpu->nem.s.fdVCpu < 0)
471 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
472
473 /* Map the KVM_RUN area. */
474 pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ | PROT_WRITE, MAP_SHARED,
475 pVCpu->nem.s.fdVCpu, 0 /*offset*/);
476 if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
477 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "mmap failed for VCpu #%u: %d", idCpu, errno);
478
479 /* Initialize the vCPU. */
480 struct kvm_vcpu_init VCpuInit; RT_ZERO(VCpuInit);
481 VCpuInit.target = KVM_ARM_TARGET_GENERIC_V8;
482 /** @todo Enable features. */
483 if (ioctl(pVCpu->nem.s.fdVCpu, KVM_ARM_VCPU_INIT, &VCpuInit) != 0)
484 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_ARM_VCPU_INIT failed for VCpu #%u: %d", idCpu, errno);
485
486#if 0
487 uint32_t fFeatures = 0; /** @todo SVE */
488 if (ioctl(pVCpu->nem.s.fdVCpu, KVM_ARM_VCPU_FINALIZE, &fFeatures) != 0)
489 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_ARM_VCPU_FINALIZE failed for VCpu #%u: %d", idCpu, errno);
490#endif
491
492 if (idCpu == 0)
493 {
494 /* Query the supported register list and log it. */
495 int rc = nemR3LnxLogRegList(pVCpu->nem.s.fdVCpu);
496 if (RT_FAILURE(rc))
497 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "Querying the supported register list failed with %Rrc", rc);
498
499 /* Need to query the ID registers and populate CPUM. */
500 CPUMARMV8IDREGS IdRegs; RT_ZERO(IdRegs);
501 for (uint32_t i = 0; i < RT_ELEMENTS(s_aIdRegs); i++)
502 {
503 uint64_t *pu64 = (uint64_t *)((uint8_t *)&IdRegs + s_aIdRegs[i].offIdStruct);
504 rc = nemR3LnxKvmQueryRegU64(pVCpu, s_aIdRegs[i].idKvmReg, pu64);
505 if (RT_FAILURE(rc))
506 return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
507 "Querying register %#x failed: %Rrc", s_aIdRegs[i].idKvmReg, rc);
508 }
509
510 rc = CPUMR3PopulateFeaturesByIdRegisters(pVM, &IdRegs);
511 if (RT_FAILURE(rc))
512 return rc;
513 }
514 }
515
516 /*
517 * Setup the SMCCC filter to get exits for PSCI related
518 * guest calls (to support SMP, power off and reset).
519 */
520 struct kvm_smccc_filter SmcccPsciFilter; RT_ZERO(SmcccPsciFilter);
521 SmcccPsciFilter.base = ARM_PSCI_FUNC_ID_CREATE_FAST_64(ARM_PSCI_FUNC_ID_PSCI_VERSION);
522 SmcccPsciFilter.nr_functions = ARM_PSCI_FUNC_ID_CREATE_FAST_64(ARM_PSCI_FUNC_ID_SYSTEM_RESET) - SmcccPsciFilter.base + 1;
523 SmcccPsciFilter.action = KVM_SMCCC_FILTER_FWD_TO_USER;
524 int rc = nemR3LnxSetAttribute(pVM, KVM_ARM_VM_SMCCC_CTRL, KVM_ARM_VM_SMCCC_FILTER, &SmcccPsciFilter,
525 "KVM_ARM_VM_SMCCC_FILTER", pErrInfo);
526 if (RT_FAILURE(rc))
527 return rc;
528
529 SmcccPsciFilter.base = ARM_PSCI_FUNC_ID_CREATE_FAST_32(ARM_PSCI_FUNC_ID_PSCI_VERSION);
530 SmcccPsciFilter.nr_functions = ARM_PSCI_FUNC_ID_CREATE_FAST_32(ARM_PSCI_FUNC_ID_SYSTEM_RESET) - SmcccPsciFilter.base + 1;
531 SmcccPsciFilter.action = KVM_SMCCC_FILTER_FWD_TO_USER;
532 rc = nemR3LnxSetAttribute(pVM, KVM_ARM_VM_SMCCC_CTRL, KVM_ARM_VM_SMCCC_FILTER, &SmcccPsciFilter,
533 "KVM_ARM_VM_SMCCC_FILTER", pErrInfo);
534 if (RT_FAILURE(rc))
535 return rc;
536
537 return VINF_SUCCESS;
538}
539
540
541int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
542{
543 /*
544 * Make RTThreadPoke work again (disabled for avoiding unnecessary
545 * critical section issues in ring-0).
546 */
547 if (enmWhat == VMINITCOMPLETED_RING3)
548 VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
549
550 return VINF_SUCCESS;
551}
552
553
554/*********************************************************************************************************************************
555* CPU State *
556*********************************************************************************************************************************/
557
558/**
559 * Sets the given general purpose register to the given value.
560 *
561 * @param pVCpu The cross context virtual CPU structure of the
562 * calling EMT.
563 * @param uReg The register index.
564 * @param f64BitReg Flag whether to operate on a 64-bit or 32-bit register.
565 * @param fSignExtend Flag whether to sign extend the value.
566 * @param u64Val The value.
567 */
568DECLINLINE(void) nemR3LnxSetGReg(PVMCPU pVCpu, uint8_t uReg, bool f64BitReg, bool fSignExtend, uint64_t u64Val)
569{
570 AssertReturnVoid(uReg < 31);
571
572 if (f64BitReg)
573 pVCpu->cpum.GstCtx.aGRegs[uReg].x = fSignExtend ? (int64_t)u64Val : u64Val;
574 else
575 pVCpu->cpum.GstCtx.aGRegs[uReg].x = (uint64_t)(fSignExtend ? (int32_t)u64Val : (uint32_t)u64Val);
576
577 /* Mark the register as not extern anymore. */
578 switch (uReg)
579 {
580 case 0:
581 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X0;
582 break;
583 case 1:
584 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X1;
585 break;
586 case 2:
587 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X2;
588 break;
589 case 3:
590 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X3;
591 break;
592 default:
593 AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_X4_X28));
594 /** @todo We need to import all missing registers in order to clear this flag (or just set it in HV from here). */
595 }
596}
597
598
599/**
600 * Gets the given general purpose register and returns the value.
601 *
602 * @returns Value from the given register.
603 * @param pVCpu The cross context virtual CPU structure of the
604 * calling EMT.
605 * @param uReg The register index.
606 */
607DECLINLINE(uint64_t) nemR3LnxGetGReg(PVMCPU pVCpu, uint8_t uReg)
608{
609 AssertReturn(uReg <= ARMV8_A64_REG_XZR, 0);
610
611 if (uReg == ARMV8_A64_REG_XZR)
612 return 0;
613
614 /** @todo Import the register if extern. */
615 //AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_GPRS_MASK));
616
617 return pVCpu->cpum.GstCtx.aGRegs[uReg].x;
618}
619
620/**
621 * Worker that imports selected state from KVM.
622 */
623static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx)
624{
625 fWhat &= pVCpu->cpum.GstCtx.fExtrn;
626 if (!fWhat)
627 return VINF_SUCCESS;
628
629#if 0
630 hv_return_t hrc = hv_vcpu_get_sys_reg(pVCpu->nem.s.hVCpu, HV_SYS_REG_CNTV_CTL_EL0, &pVCpu->cpum.GstCtx.CntvCtlEl0);
631 if (hrc == HV_SUCCESS)
632 hrc = hv_vcpu_get_sys_reg(pVCpu->nem.s.hVCpu, HV_SYS_REG_CNTV_CVAL_EL0, &pVCpu->cpum.GstCtx.CntvCValEl0);
633#endif
634
635 int rc = VINF_SUCCESS;
636 if (fWhat & (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
637 {
638 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
639 {
640 if (s_aCpumRegs[i].fCpumExtrn & fWhat)
641 {
642 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
643 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumRegs[i].idKvmReg, pu64);
644 }
645 }
646 }
647
648 if ( rc == VINF_SUCCESS
649 && (fWhat & CPUMCTX_EXTRN_FPCR))
650 {
651 uint32_t u32Tmp;
652 rc |= nemR3LnxKvmQueryRegU32(pVCpu, KVM_ARM64_REG_FP_FPCR, &u32Tmp);
653 if (rc == VINF_SUCCESS)
654 pVCpu->cpum.GstCtx.fpcr = u32Tmp;
655 }
656
657 if ( rc == VINF_SUCCESS
658 && (fWhat & CPUMCTX_EXTRN_FPSR))
659 {
660 uint32_t u32Tmp;
661 rc |= nemR3LnxKvmQueryRegU32(pVCpu, KVM_ARM64_REG_FP_FPSR, &u32Tmp);
662 if (rc == VINF_SUCCESS)
663 pVCpu->cpum.GstCtx.fpsr = u32Tmp;
664 }
665
666 if ( rc == VINF_SUCCESS
667 && (fWhat & CPUMCTX_EXTRN_V0_V31))
668 {
669 /* SIMD/FP registers. */
670 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
671 {
672 void *pu128 = (void *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
673 rc |= nemR3LnxKvmQueryRegPV(pVCpu, s_aCpumFpRegs[i].idKvmReg, pu128);
674 }
675 }
676
677 if ( rc == VINF_SUCCESS
678 && (fWhat & CPUMCTX_EXTRN_SYSREG_DEBUG))
679 {
680 /* Debug registers. */
681 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumDbgRegs); i++)
682 {
683 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumDbgRegs[i].offCpumCtx);
684 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumDbgRegs[i].idKvmReg, pu64);
685 }
686 }
687
688 if ( rc == VINF_SUCCESS
689 && (fWhat & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS))
690 {
691 /* PAuth registers. */
692 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
693 {
694 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
695 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumPAuthKeyRegs[i].idKvmReg, pu64);
696 }
697 }
698
699 if ( rc == VINF_SUCCESS
700 && (fWhat & (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC)))
701 {
702 /* System registers. */
703 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
704 {
705 if (s_aCpumSysRegs[i].fCpumExtrn & fWhat)
706 {
707 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
708 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumSysRegs[i].idKvmReg, pu64);
709 }
710 }
711 }
712
713 if ( rc == VINF_SUCCESS
714 && (fWhat & CPUMCTX_EXTRN_PSTATE))
715 {
716 uint64_t u64Tmp;
717 rc |= nemR3LnxKvmQueryRegU64(pVCpu, KVM_ARM64_REG_PSTATE, &u64Tmp);
718 if (rc == VINF_SUCCESS)
719 pVCpu->cpum.GstCtx.fPState = (uint32_t)u64Tmp;
720
721 }
722
723 /*
724 * Update the external mask.
725 */
726 pCtx->fExtrn &= ~fWhat;
727 pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
728 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
729 pVCpu->cpum.GstCtx.fExtrn = 0;
730
731 return VINF_SUCCESS;
732}
733
734
735/**
736 * Interface for importing state on demand (used by IEM).
737 *
738 * @returns VBox status code.
739 * @param pVCpu The cross context CPU structure.
740 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
741 */
742VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
743{
744 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
745 return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx);
746}
747
748
749/**
750 * Exports state to KVM.
751 */
752static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
753{
754 uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
755 Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL); RT_NOREF(fExtrn);
756
757 RT_NOREF(pVM);
758 int rc = VINF_SUCCESS;
759 if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
760 != (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
761 {
762 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
763 {
764 if (!(s_aCpumRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
765 {
766 const uint64_t *pu64 = (const uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
767 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumRegs[i].idKvmReg, pu64);
768 }
769 }
770 }
771
772 if ( rc == VINF_SUCCESS
773 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_FPCR))
774 {
775 uint32_t u32Tmp = pVCpu->cpum.GstCtx.fpcr;
776 rc |= nemR3LnxKvmSetRegU32(pVCpu, KVM_ARM64_REG_FP_FPCR, &u32Tmp);
777 }
778
779 if ( rc == VINF_SUCCESS
780 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_FPSR))
781 {
782 uint32_t u32Tmp = pVCpu->cpum.GstCtx.fpsr;
783 rc |= nemR3LnxKvmSetRegU32(pVCpu, KVM_ARM64_REG_FP_FPSR, &u32Tmp);
784 }
785
786 if ( rc == VINF_SUCCESS
787 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_V0_V31))
788 {
789 /* SIMD/FP registers. */
790 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
791 {
792 void *pu128 = (void *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
793 rc |= nemR3LnxKvmSetRegPV(pVCpu, s_aCpumFpRegs[i].idKvmReg, pu128);
794 }
795 }
796
797 if ( rc == VINF_SUCCESS
798 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SYSREG_DEBUG))
799 {
800 /* Debug registers. */
801 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumDbgRegs); i++)
802 {
803 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumDbgRegs[i].offCpumCtx);
804 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumDbgRegs[i].idKvmReg, pu64);
805 }
806 }
807
808 if ( rc == VINF_SUCCESS
809 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS))
810 {
811 /* PAuth registers. */
812 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
813 {
814 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
815 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumPAuthKeyRegs[i].idKvmReg, pu64);
816 }
817 }
818
819 if ( rc == VINF_SUCCESS
820 && (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC))
821 != (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC))
822 {
823 /* System registers. */
824 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
825 {
826 if (!(s_aCpumSysRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
827 {
828 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
829 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumSysRegs[i].idKvmReg, pu64);
830 }
831 }
832 }
833
834 if ( rc == VINF_SUCCESS
835 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_PSTATE))
836 {
837 uint64_t u64Tmp = pVCpu->cpum.GstCtx.fPState;
838 rc = nemR3LnxKvmSetRegU64(pVCpu, KVM_ARM64_REG_PSTATE, &u64Tmp);
839 }
840
841 /*
842 * KVM now owns all the state.
843 */
844 pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM | CPUMCTX_EXTRN_ALL;
845 return VINF_SUCCESS;
846}
847
848
849/**
850 * Query the CPU tick counter and optionally the TSC_AUX MSR value.
851 *
852 * @returns VBox status code.
853 * @param pVCpu The cross context CPU structure.
854 * @param pcTicks Where to return the CPU tick count.
855 * @param puAux Where to return the TSC_AUX register value.
856 */
857VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
858{
859 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
860 // KVM_GET_CLOCK?
861 RT_NOREF(pVCpu, pcTicks, puAux);
862 return VINF_SUCCESS;
863}
864
865
866/**
867 * Resumes CPU clock (TSC) on all virtual CPUs.
868 *
869 * This is called by TM when the VM is started, restored, resumed or similar.
870 *
871 * @returns VBox status code.
872 * @param pVM The cross context VM structure.
873 * @param pVCpu The cross context CPU structure of the calling EMT.
874 * @param uPausedTscValue The TSC value at the time of pausing.
875 */
876VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
877{
878 // KVM_SET_CLOCK?
879 RT_NOREF(pVM, pVCpu, uPausedTscValue);
880 return VINF_SUCCESS;
881}
882
883
884VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
885{
886 RT_NOREF(pVM);
887 return NEM_FEAT_F_NESTED_PAGING
888 | NEM_FEAT_F_FULL_GST_EXEC;
889}
890
891
892
893/*********************************************************************************************************************************
894* Execution *
895*********************************************************************************************************************************/
896
897
898VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
899{
900 RT_NOREF(pVM, pVCpu);
901 Assert(VM_IS_NEM_ENABLED(pVM));
902 return true;
903}
904
905
906bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
907{
908 NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
909 return false;
910}
911
912
913void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
914{
915 int rc = RTThreadPoke(pVCpu->hThread);
916 LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
917 AssertRC(rc);
918 RT_NOREF(pVM, fFlags);
919}
920
921
922DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
923{
924 RT_NOREF(pVM, fUseDebugLoop);
925 return false;
926}
927
928
929DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
930{
931 RT_NOREF(pVM, pVCpu, fUseDebugLoop);
932 return false;
933}
934
935
936DECL_FORCE_INLINE(int) nemR3LnxKvmUpdateIntrState(PVM pVM, PVMCPU pVCpu, bool fIrq, bool fAsserted)
937{
938 struct kvm_irq_level IrqLvl;
939
940 LogFlowFunc(("pVM=%p pVCpu=%p fIrq=%RTbool fAsserted=%RTbool\n",
941 pVM, pVCpu, fIrq, fAsserted));
942
943 IrqLvl.irq = ((uint32_t)KVM_ARM_IRQ_TYPE_CPU << 24) /* Directly drives CPU interrupt lines. */
944 | (pVCpu->idCpu & 0xff) << 16
945 | (fIrq ? 0 : 1);
946 IrqLvl.level = fAsserted ? 1 : 0;
947 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_IRQ_LINE, &IrqLvl);
948 AssertReturn(rcLnx == 0, VERR_NEM_IPE_9);
949
950 return VINF_SUCCESS;
951}
952
953
954/**
955 * Deals with pending interrupt FFs prior to executing guest code.
956 */
957static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu)
958{
959 LogFlowFunc(("pVCpu=%p{.idCpu=%u} fIrq=%RTbool fFiq=%RTbool\n",
960 pVCpu, pVCpu->idCpu,
961 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ),
962 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_FIQ)));
963
964 bool fIrq = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ);
965 bool fFiq = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_FIQ);
966
967 /* Update the pending interrupt state. */
968 if (fIrq != pVCpu->nem.s.fIrqLastSeen)
969 {
970 int rc = nemR3LnxKvmUpdateIntrState(pVM, pVCpu, true /*fIrq*/, fIrq);
971 AssertRCReturn(rc, VERR_NEM_IPE_9);
972 pVCpu->nem.s.fIrqLastSeen = fIrq;
973 }
974
975 if (fFiq != pVCpu->nem.s.fIrqLastSeen)
976 {
977 int rc = nemR3LnxKvmUpdateIntrState(pVM, pVCpu, false /*fIrq*/, fFiq);
978 AssertRCReturn(rc, VERR_NEM_IPE_9);
979 pVCpu->nem.s.fFiqLastSeen = fFiq;
980 }
981
982 return VINF_SUCCESS;
983}
984
985
986#if 0
987/**
988 * Handles KVM_EXIT_INTERNAL_ERROR.
989 */
990static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
991{
992 Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
993 pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
994
995 /*
996 * Deal with each suberror, returning if we don't want IEM to handle it.
997 */
998 switch (pRun->internal.suberror)
999 {
1000 case KVM_INTERNAL_ERROR_EMULATION:
1001 {
1002 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
1003 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1004 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
1005 break;
1006 }
1007
1008 case KVM_INTERNAL_ERROR_SIMUL_EX:
1009 case KVM_INTERNAL_ERROR_DELIVERY_EV:
1010 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
1011 default:
1012 {
1013 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
1014 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1015 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
1016 const char *pszName;
1017 switch (pRun->internal.suberror)
1018 {
1019 case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
1020 case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
1021 case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
1022 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
1023 default: pszName = "unknown"; break;
1024 }
1025 LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
1026 pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
1027 return VERR_NEM_IPE_0;
1028 }
1029 }
1030
1031 /*
1032 * Execute instruction in IEM and try get on with it.
1033 */
1034 Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
1035 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
1036 VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
1037 IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_INHIBIT_INT
1038 | CPUMCTX_EXTRN_INHIBIT_NMI,
1039 &pVCpu->cpum.GstCtx, pRun);
1040 if (RT_SUCCESS(rcStrict))
1041 rcStrict = IEMExecOne(pVCpu);
1042 return rcStrict;
1043}
1044#endif
1045
1046
1047/**
1048 * Handles KVM_EXIT_MMIO.
1049 */
1050static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1051{
1052 /*
1053 * Input validation.
1054 */
1055 Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
1056 Assert(pRun->mmio.is_write <= 1);
1057
1058#if 0
1059 /*
1060 * We cannot easily act on the exit history here, because the MMIO port
1061 * exit is stateful and the instruction will be completed in the next
1062 * KVM_RUN call. There seems no way to circumvent this.
1063 */
1064 EMHistoryAddExit(pVCpu,
1065 pRun->mmio.is_write
1066 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1067 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1068 pRun->s.regs.regs.pc, ASMReadTSC());
1069#else
1070 RT_NOREF(pVCpu);
1071#endif
1072
1073 /*
1074 * Do the requested job.
1075 */
1076 VBOXSTRICTRC rcStrict;
1077 if (pRun->mmio.is_write)
1078 {
1079 rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1080 Log4(("MmioExit/%u:WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
1081 pVCpu->idCpu,
1082 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1083 }
1084 else
1085 {
1086 rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1087 Log4(("MmioExit/%u: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
1088 pVCpu->idCpu,
1089 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1090 }
1091 return rcStrict;
1092}
1093
1094
1095/**
1096 * Handles KVM_EXIT_HYPERCALL.
1097 */
1098static VBOXSTRICTRC nemHCLnxHandleExitHypercall(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1099{
1100#if 0
1101 /*
1102 * We cannot easily act on the exit history here, because the MMIO port
1103 * exit is stateful and the instruction will be completed in the next
1104 * KVM_RUN call. There seems no way to circumvent this.
1105 */
1106 EMHistoryAddExit(pVCpu,
1107 pRun->mmio.is_write
1108 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1109 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1110 pRun->s.regs.regs.pc, ASMReadTSC());
1111#else
1112 RT_NOREF(pVCpu);
1113#endif
1114
1115 /*
1116 * Do the requested job.
1117 */
1118 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1119
1120 /** @todo Raise exception to EL1 if PSCI not configured. */
1121 /** @todo Need a generic mechanism here to pass this to, GIM maybe?. */
1122 uint32_t uFunId = pRun->hypercall.nr;
1123 bool fHvc64 = RT_BOOL(uFunId & ARM_SMCCC_FUNC_ID_64BIT); RT_NOREF(fHvc64);
1124 uint32_t uEntity = ARM_SMCCC_FUNC_ID_ENTITY_GET(uFunId);
1125 uint32_t uFunNum = ARM_SMCCC_FUNC_ID_NUM_GET(uFunId);
1126 if (uEntity == ARM_SMCCC_FUNC_ID_ENTITY_STD_SEC_SERVICE)
1127 {
1128 rcStrict = nemHCLnxImportState(pVCpu, CPUMCTX_EXTRN_X0 | CPUMCTX_EXTRN_X1 | CPUMCTX_EXTRN_X2 | CPUMCTX_EXTRN_X3,
1129 &pVCpu->cpum.GstCtx);
1130 if (rcStrict != VINF_SUCCESS)
1131 return rcStrict;
1132
1133 switch (uFunNum)
1134 {
1135 case ARM_PSCI_FUNC_ID_PSCI_VERSION:
1136 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, ARM_PSCI_FUNC_ID_PSCI_VERSION_SET(1, 2));
1137 break;
1138 case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
1139 rcStrict = VMR3PowerOff(pVM->pUVM);
1140 break;
1141 case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
1142 case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
1143 {
1144 bool fHaltOnReset;
1145 int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
1146 if (RT_SUCCESS(rc) && fHaltOnReset)
1147 {
1148 Log(("nemHCLnxHandleExitHypercall: Halt On Reset!\n"));
1149 rcStrict = VINF_EM_HALT;
1150 }
1151 else
1152 {
1153 /** @todo pVM->pdm.s.fResetFlags = fFlags; */
1154 VM_FF_SET(pVM, VM_FF_RESET);
1155 rcStrict = VINF_EM_RESET;
1156 }
1157 break;
1158 }
1159 case ARM_PSCI_FUNC_ID_CPU_ON:
1160 {
1161 uint64_t u64TgtCpu = nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X1);
1162 RTGCPHYS GCPhysExecAddr = nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X2);
1163 uint64_t u64CtxId = nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X3);
1164 VMMR3CpuOn(pVM, u64TgtCpu & 0xff, GCPhysExecAddr, u64CtxId);
1165 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, true /*f64BitReg*/, false /*fSignExtend*/, ARM_PSCI_STS_SUCCESS);
1166 break;
1167 }
1168 case ARM_PSCI_FUNC_ID_PSCI_FEATURES:
1169 {
1170 uint32_t u32FunNum = (uint32_t)nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X1);
1171 switch (u32FunNum)
1172 {
1173 case ARM_PSCI_FUNC_ID_PSCI_VERSION:
1174 case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
1175 case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
1176 case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
1177 case ARM_PSCI_FUNC_ID_CPU_ON:
1178 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0,
1179 false /*f64BitReg*/, false /*fSignExtend*/,
1180 (uint64_t)ARM_PSCI_STS_SUCCESS);
1181 break;
1182 default:
1183 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0,
1184 false /*f64BitReg*/, false /*fSignExtend*/,
1185 (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1186 }
1187 break;
1188 }
1189 default:
1190 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1191 }
1192 }
1193 else
1194 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1195
1196
1197 return rcStrict;
1198}
1199
1200
1201static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun, bool *pfStatefulExit)
1202{
1203 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
1204
1205 if (pVCpu->nem.s.fIrqDeviceLvls != pRun->s.regs.device_irq_level)
1206 {
1207 uint64_t fChanged = pVCpu->nem.s.fIrqDeviceLvls ^ pRun->s.regs.device_irq_level;
1208
1209 if (fChanged & KVM_ARM_DEV_EL1_VTIMER)
1210 {
1211 TMCpuSetVTimerNextActivation(pVCpu, UINT64_MAX);
1212 PDMGicSetPpi(pVCpu, pVM->nem.s.u32GicPpiVTimer, RT_BOOL(pRun->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER));
1213 }
1214
1215 if (fChanged & KVM_ARM_DEV_EL1_PTIMER)
1216 {
1217 //TMCpuSetVTimerNextActivation(pVCpu, UINT64_MAX);
1218 PDMGicSetPpi(pVCpu, pVM->nem.s.u32GicPpiVTimer, RT_BOOL(pRun->s.regs.device_irq_level & KVM_ARM_DEV_EL1_PTIMER));
1219 }
1220
1221 pVCpu->nem.s.fIrqDeviceLvls = pRun->s.regs.device_irq_level;
1222 }
1223
1224 switch (pRun->exit_reason)
1225 {
1226 case KVM_EXIT_EXCEPTION:
1227 AssertFailed();
1228 break;
1229
1230 case KVM_EXIT_MMIO:
1231 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
1232 *pfStatefulExit = true;
1233 return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
1234
1235 case KVM_EXIT_INTR: /* EINTR */
1236 //EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
1237 // pRun->s.regs.regs.pc, ASMReadTSC());
1238 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
1239 Log5(("Intr/%u\n", pVCpu->idCpu));
1240 return VINF_SUCCESS;
1241
1242 case KVM_EXIT_HYPERCALL:
1243 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
1244 return nemHCLnxHandleExitHypercall(pVM, pVCpu, pRun);
1245
1246#if 0
1247 case KVM_EXIT_DEBUG:
1248 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
1249 AssertFailed();
1250 break;
1251
1252 case KVM_EXIT_SYSTEM_EVENT:
1253 AssertFailed();
1254 break;
1255
1256 case KVM_EXIT_DIRTY_RING_FULL:
1257 AssertFailed();
1258 break;
1259 case KVM_EXIT_AP_RESET_HOLD:
1260 AssertFailed();
1261 break;
1262
1263
1264 case KVM_EXIT_SHUTDOWN:
1265 AssertFailed();
1266 break;
1267
1268 case KVM_EXIT_FAIL_ENTRY:
1269 LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
1270 pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
1271 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
1272 pRun->s.regs.regs.pc, ASMReadTSC());
1273 return VERR_NEM_IPE_1;
1274
1275 case KVM_EXIT_INTERNAL_ERROR:
1276 /* we're counting sub-reasons inside the function. */
1277 return nemR3LnxHandleInternalError(pVCpu, pRun);
1278#endif
1279
1280 /*
1281 * Foreign and unknowns.
1282 */
1283#if 0
1284 case KVM_EXIT_IO:
1285 AssertLogRelMsgFailedReturn(("KVM_EXIT_IO on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1286 case KVM_EXIT_NMI:
1287 AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1288 case KVM_EXIT_EPR:
1289 AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1290 case KVM_EXIT_WATCHDOG:
1291 AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1292 case KVM_EXIT_ARM_NISV:
1293 AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1294 case KVM_EXIT_S390_STSI:
1295 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1296 case KVM_EXIT_S390_TSCH:
1297 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1298 case KVM_EXIT_OSI:
1299 AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1300 case KVM_EXIT_PAPR_HCALL:
1301 AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1302 case KVM_EXIT_S390_UCONTROL:
1303 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1304 case KVM_EXIT_DCR:
1305 AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1306 case KVM_EXIT_S390_SIEIC:
1307 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1308 case KVM_EXIT_S390_RESET:
1309 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1310 case KVM_EXIT_UNKNOWN:
1311 AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1312 case KVM_EXIT_XEN:
1313 AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1314#endif
1315 default:
1316 AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u!\n", pRun->exit_reason, pVCpu->idCpu), VERR_NEM_IPE_1);
1317 }
1318 RT_NOREF(pVM, pVCpu);
1319 return VERR_NOT_IMPLEMENTED;
1320}
1321
1322
1323VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
1324{
1325 /*
1326 * Try switch to NEM runloop state.
1327 */
1328 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
1329 { /* likely */ }
1330 else
1331 {
1332 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1333 LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
1334 return VINF_SUCCESS;
1335 }
1336
1337 /*
1338 * The run loop.
1339 */
1340 struct kvm_run * const pRun = pVCpu->nem.s.pRun;
1341 const bool fSingleStepping = DBGFIsStepping(pVCpu);
1342 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1343 bool fStatefulExit = false; /* For MMIO and IO exits. */
1344 for (unsigned iLoop = 0;; iLoop++)
1345 {
1346 /*
1347 * Sync the interrupt state.
1348 */
1349 rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu);
1350 if (rcStrict == VINF_SUCCESS)
1351 { /* likely */ }
1352 else
1353 {
1354 LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1355 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1356 break;
1357 }
1358
1359 /*
1360 * Ensure KVM has the whole state.
1361 */
1362 if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
1363 {
1364 int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx);
1365 AssertRCReturn(rc2, rc2);
1366 }
1367
1368 /*
1369 * Poll timers and run for a bit.
1370 *
1371 * With the VID approach (ring-0 or ring-3) we can specify a timeout here,
1372 * so we take the time of the next timer event and uses that as a deadline.
1373 * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
1374 */
1375 /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
1376 * the whole polling job when timers have changed... */
1377 uint64_t offDeltaIgnored;
1378 uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
1379 if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
1380 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1381 {
1382 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
1383 {
1384 //LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
1385 // pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1386 // !!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
1387 // pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
1388 TMNotifyStartOfExecution(pVM, pVCpu);
1389
1390 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1391
1392 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
1393 TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
1394
1395#if 0 //def LOG_ENABLED
1396 if (LogIsFlowEnabled())
1397 {
1398 struct kvm_mp_state MpState = {UINT32_MAX};
1399 ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
1400 LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
1401 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
1402 pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
1403 pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
1404 }
1405#endif
1406 fStatefulExit = false;
1407 if (RT_LIKELY(rcLnx == 0 || errno == EINTR))
1408 {
1409 /*
1410 * Deal with the exit.
1411 */
1412 rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1413 if (rcStrict == VINF_SUCCESS)
1414 { /* hopefully likely */ }
1415 else
1416 {
1417 LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1418 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1419 break;
1420 }
1421 }
1422 else
1423 {
1424 int rc2 = RTErrConvertFromErrno(errno);
1425 AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
1426 }
1427
1428 /*
1429 * If no relevant FFs are pending, loop.
1430 */
1431 if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
1432 && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
1433 { /* likely */ }
1434 else
1435 {
1436
1437 /** @todo Try handle pending flags, not just return to EM loops. Take care
1438 * not to set important RCs here unless we've handled an exit. */
1439 LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
1440 pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
1441 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
1442 break;
1443 }
1444 }
1445 else
1446 {
1447 LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
1448 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
1449 break;
1450 }
1451 }
1452 else
1453 {
1454 LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
1455 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
1456 break;
1457 }
1458 } /* the run loop */
1459
1460
1461 /*
1462 * If the last exit was stateful, commit the state we provided before
1463 * returning to the EM loop so we have a consistent state and can safely
1464 * be rescheduled and whatnot. This may require us to make multiple runs
1465 * for larger MMIO and I/O operations. Sigh^3.
1466 *
1467 * Note! There is no 'ing way to reset the kernel side completion callback
1468 * for these stateful i/o exits. Very annoying interface.
1469 */
1470 /** @todo check how this works with string I/O and string MMIO. */
1471 if (fStatefulExit && RT_SUCCESS(rcStrict))
1472 {
1473 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
1474 uint32_t const uOrgExit = pRun->exit_reason;
1475 for (uint32_t i = 0; ; i++)
1476 {
1477 pRun->immediate_exit = 1;
1478 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1479 Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
1480 if (rcLnx == -1 && errno == EINTR)
1481 {
1482 switch (i)
1483 {
1484 case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
1485 case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
1486 case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
1487 default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
1488 }
1489 break;
1490 }
1491 AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
1492 ("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
1493 rcStrict = VERR_NEM_IPE_6);
1494 VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1495 if (rcStrict2 == VINF_SUCCESS || rcStrict2 == rcStrict)
1496 { /* likely */ }
1497 else if (RT_FAILURE(rcStrict2))
1498 {
1499 rcStrict = rcStrict2;
1500 break;
1501 }
1502 else
1503 {
1504 AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
1505 ("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
1506 rcStrict = VERR_NEM_IPE_7);
1507 rcStrict = rcStrict2;
1508 }
1509 }
1510 pRun->immediate_exit = 0;
1511 }
1512
1513 /*
1514 * If the CPU is running, make sure to stop it before we try sync back the
1515 * state and return to EM. We don't sync back the whole state if we can help it.
1516 */
1517 if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
1518 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1519
1520 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
1521 {
1522 /* Try anticipate what we might need. */
1523 uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK /*?*/;
1524 if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
1525 || RT_FAILURE(rcStrict))
1526 fImport = CPUMCTX_EXTRN_ALL;
1527 else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ | VMCPU_FF_INTERRUPT_FIQ))
1528 fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK;
1529
1530 if (pVCpu->cpum.GstCtx.fExtrn & fImport)
1531 {
1532 int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx);
1533 if (RT_SUCCESS(rc2))
1534 pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
1535 else if (RT_SUCCESS(rcStrict))
1536 rcStrict = rc2;
1537 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1538 pVCpu->cpum.GstCtx.fExtrn = 0;
1539 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
1540 }
1541 else
1542 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1543 }
1544 else
1545 {
1546 pVCpu->cpum.GstCtx.fExtrn = 0;
1547 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1548 }
1549
1550 LogFlow(("NEM/%u: %08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc, VBOXSTRICTRC_VAL(rcStrict) ));
1551 return rcStrict;
1552}
1553
1554
1555/** @page pg_nem_linux_armv8 NEM/linux - Native Execution Manager, Linux.
1556 *
1557 * This is using KVM.
1558 *
1559 */
1560
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette