VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux-armv8.cpp

Last change on this file was 108843, checked in by vboxsync, 4 weeks ago

VMM/PGM,NEM: Some early page table management infrastructure for ARMv8, bugref:10388
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 64.1 KB
Line 
1/* $Id: NEMR3Native-linux-armv8.cpp 108843 2025-04-04 08:36:32Z vboxsync $ */
2/** @file
3 * NEM - Native execution manager, native ring-3 Linux backend arm64 version.
4 */
5
6/*
7 * Copyright (C) 2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_NEM
33#define VMCPU_INCL_CPUM_GST_CTX
34#include <VBox/vmm/nem.h>
35#include <VBox/vmm/iem.h>
36#include <VBox/vmm/em.h>
37#include <VBox/vmm/pdmgic.h>
38#include <VBox/vmm/pdm.h>
39#include <VBox/vmm/trpm.h>
40#include "NEMInternal.h"
41#include <VBox/vmm/vmcc.h>
42
43#include <iprt/alloca.h>
44#include <iprt/string.h>
45#include <iprt/system.h>
46#include <iprt/armv8.h>
47
48#include <iprt/formats/arm-psci.h>
49
50#include <errno.h>
51#include <unistd.h>
52#include <sys/ioctl.h>
53#include <sys/fcntl.h>
54#include <sys/mman.h>
55#include <linux/kvm.h>
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61
62/** Core register group. */
63#define KVM_ARM64_REG_CORE_GROUP UINT64_C(0x6030000000100000)
64/** System register group. */
65#define KVM_ARM64_REG_SYS_GROUP UINT64_C(0x6030000000130000)
66/** System register group. */
67#define KVM_ARM64_REG_SIMD_GROUP UINT64_C(0x6040000000100050)
68/** FP register group. */
69#define KVM_ARM64_REG_FP_GROUP UINT64_C(0x6020000000100000)
70
71#define KVM_ARM64_REG_CORE_CREATE(a_idReg) (KVM_ARM64_REG_CORE_GROUP | ((uint64_t)(a_idReg) & 0xffff))
72#define KVM_ARM64_REG_GPR(a_iGpr) KVM_ARM64_REG_CORE_CREATE((a_iGpr) << 1)
73#define KVM_ARM64_REG_SP_EL0 KVM_ARM64_REG_CORE_CREATE(0x3e)
74#define KVM_ARM64_REG_PC KVM_ARM64_REG_CORE_CREATE(0x40)
75#define KVM_ARM64_REG_PSTATE KVM_ARM64_REG_CORE_CREATE(0x42)
76#define KVM_ARM64_REG_SP_EL1 KVM_ARM64_REG_CORE_CREATE(0x44)
77#define KVM_ARM64_REG_ELR_EL1 KVM_ARM64_REG_CORE_CREATE(0x46)
78#define KVM_ARM64_REG_SPSR_EL1 KVM_ARM64_REG_CORE_CREATE(0x48)
79#define KVM_ARM64_REG_SPSR_ABT KVM_ARM64_REG_CORE_CREATE(0x4a)
80#define KVM_ARM64_REG_SPSR_UND KVM_ARM64_REG_CORE_CREATE(0x4c)
81#define KVM_ARM64_REG_SPSR_IRQ KVM_ARM64_REG_CORE_CREATE(0x4e)
82#define KVM_ARM64_REG_SPSR_FIQ KVM_ARM64_REG_CORE_CREATE(0x50)
83
84/** This maps to our IPRT representation of system register IDs, yay! */
85#define KVM_ARM64_REG_SYS_CREATE(a_idSysReg) (KVM_ARM64_REG_SYS_GROUP | ((uint64_t)(a_idSysReg) & 0xffff))
86
87#define KVM_ARM64_REG_SIMD_CREATE(a_iVecReg) (KVM_ARM64_REG_SIMD_GROUP | (((uint64_t)(a_iVecReg) << 2) & 0xffff))
88
89#define KVM_ARM64_REG_FP_CREATE(a_idReg) (KVM_ARM64_REG_FP_GROUP | ((uint64_t)(a_idReg) & 0xffff))
90#define KVM_ARM64_REG_FP_FPSR KVM_ARM64_REG_FP_CREATE(0xd4)
91#define KVM_ARM64_REG_FP_FPCR KVM_ARM64_REG_FP_CREATE(0xd5)
92
93/* In case the build host doesn't ship with a new enough kvm.h. */
94#ifndef KVM_ARM_VM_SMCCC_CTRL
95# define KVM_ARM_VM_SMCCC_CTRL 0
96# define KVM_ARM_VM_SMCCC_FILTER 0
97
98# define KVM_SMCCC_FILTER_HANDLE 0
99# define KVM_SMCCC_FILTER_DENY 1
100# define KVM_SMCCC_FILTER_FWD_TO_USER 2
101
102struct kvm_smccc_filter
103{
104 uint32_t base;
105 uint32_t nr_functions;
106 uint8_t action;
107 uint8_t pad[15];
108};
109#endif
110
111
112/*********************************************************************************************************************************
113* Structures and Typedefs *
114*********************************************************************************************************************************/
115
116
117/*********************************************************************************************************************************
118* Global Variables *
119*********************************************************************************************************************************/
120/** The general registers. */
121static const struct
122{
123 uint64_t idKvmReg;
124 uint32_t fCpumExtrn;
125 uint32_t offCpumCtx;
126} s_aCpumRegs[] =
127{
128#define CPUM_GREG_EMIT_X0_X3(a_Idx) { KVM_ARM64_REG_GPR(a_Idx), CPUMCTX_EXTRN_X ## a_Idx, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
129#define CPUM_GREG_EMIT_X4_X28(a_Idx) { KVM_ARM64_REG_GPR(a_Idx), CPUMCTX_EXTRN_X4_X28, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
130 CPUM_GREG_EMIT_X0_X3(0),
131 CPUM_GREG_EMIT_X0_X3(1),
132 CPUM_GREG_EMIT_X0_X3(2),
133 CPUM_GREG_EMIT_X0_X3(3),
134 CPUM_GREG_EMIT_X4_X28(4),
135 CPUM_GREG_EMIT_X4_X28(5),
136 CPUM_GREG_EMIT_X4_X28(6),
137 CPUM_GREG_EMIT_X4_X28(7),
138 CPUM_GREG_EMIT_X4_X28(8),
139 CPUM_GREG_EMIT_X4_X28(9),
140 CPUM_GREG_EMIT_X4_X28(10),
141 CPUM_GREG_EMIT_X4_X28(11),
142 CPUM_GREG_EMIT_X4_X28(12),
143 CPUM_GREG_EMIT_X4_X28(13),
144 CPUM_GREG_EMIT_X4_X28(14),
145 CPUM_GREG_EMIT_X4_X28(15),
146 CPUM_GREG_EMIT_X4_X28(16),
147 CPUM_GREG_EMIT_X4_X28(17),
148 CPUM_GREG_EMIT_X4_X28(18),
149 CPUM_GREG_EMIT_X4_X28(19),
150 CPUM_GREG_EMIT_X4_X28(20),
151 CPUM_GREG_EMIT_X4_X28(21),
152 CPUM_GREG_EMIT_X4_X28(22),
153 CPUM_GREG_EMIT_X4_X28(23),
154 CPUM_GREG_EMIT_X4_X28(24),
155 CPUM_GREG_EMIT_X4_X28(25),
156 CPUM_GREG_EMIT_X4_X28(26),
157 CPUM_GREG_EMIT_X4_X28(27),
158 CPUM_GREG_EMIT_X4_X28(28),
159 { KVM_ARM64_REG_GPR(29), CPUMCTX_EXTRN_FP, RT_UOFFSETOF(CPUMCTX, aGRegs[29].x) },
160 { KVM_ARM64_REG_GPR(30), CPUMCTX_EXTRN_LR, RT_UOFFSETOF(CPUMCTX, aGRegs[30].x) },
161 { KVM_ARM64_REG_PC, CPUMCTX_EXTRN_PC, RT_UOFFSETOF(CPUMCTX, Pc.u64) },
162#undef CPUM_GREG_EMIT_X0_X3
163#undef CPUM_GREG_EMIT_X4_X28
164};
165/** SIMD/FP registers. */
166static const struct
167{
168 uint64_t idKvmReg;
169 uint32_t offCpumCtx;
170} s_aCpumFpRegs[] =
171{
172#define CPUM_VREG_EMIT(a_Idx) { KVM_ARM64_REG_SIMD_CREATE(a_Idx), RT_UOFFSETOF(CPUMCTX, aVRegs[a_Idx].v) }
173 CPUM_VREG_EMIT(0),
174 CPUM_VREG_EMIT(1),
175 CPUM_VREG_EMIT(2),
176 CPUM_VREG_EMIT(3),
177 CPUM_VREG_EMIT(4),
178 CPUM_VREG_EMIT(5),
179 CPUM_VREG_EMIT(6),
180 CPUM_VREG_EMIT(7),
181 CPUM_VREG_EMIT(8),
182 CPUM_VREG_EMIT(9),
183 CPUM_VREG_EMIT(10),
184 CPUM_VREG_EMIT(11),
185 CPUM_VREG_EMIT(12),
186 CPUM_VREG_EMIT(13),
187 CPUM_VREG_EMIT(14),
188 CPUM_VREG_EMIT(15),
189 CPUM_VREG_EMIT(16),
190 CPUM_VREG_EMIT(17),
191 CPUM_VREG_EMIT(18),
192 CPUM_VREG_EMIT(19),
193 CPUM_VREG_EMIT(20),
194 CPUM_VREG_EMIT(21),
195 CPUM_VREG_EMIT(22),
196 CPUM_VREG_EMIT(23),
197 CPUM_VREG_EMIT(24),
198 CPUM_VREG_EMIT(25),
199 CPUM_VREG_EMIT(26),
200 CPUM_VREG_EMIT(27),
201 CPUM_VREG_EMIT(28),
202 CPUM_VREG_EMIT(29),
203 CPUM_VREG_EMIT(30),
204 CPUM_VREG_EMIT(31)
205#undef CPUM_VREG_EMIT
206};
207/** System registers. */
208static const struct
209{
210 uint64_t idKvmReg;
211 uint32_t fCpumExtrn;
212 uint32_t offCpumCtx;
213} s_aCpumSysRegs[] =
214{
215 { KVM_ARM64_REG_SP_EL0, CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[0].u64) },
216 { KVM_ARM64_REG_SP_EL1, CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[1].u64) },
217 { KVM_ARM64_REG_SPSR_EL1, CPUMCTX_EXTRN_SPSR, RT_UOFFSETOF(CPUMCTX, Spsr.u64) },
218 { KVM_ARM64_REG_ELR_EL1, CPUMCTX_EXTRN_ELR, RT_UOFFSETOF(CPUMCTX, Elr.u64) },
219 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_VBAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, VBar.u64) },
220 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AFSR0_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr0.u64) },
221 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AFSR1_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr1.u64) },
222 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AMAIR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Amair.u64) },
223 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CNTKCTL_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, CntKCtl.u64) },
224 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CONTEXTIDR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, ContextIdr.u64) },
225 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CPACR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Cpacr.u64) },
226 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CSSELR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Csselr.u64) },
227 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ESR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Esr.u64) },
228 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_FAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Far.u64) },
229 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MAIR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Mair.u64) },
230 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_PAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Par.u64) },
231 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDRRO_EL0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, TpIdrRoEl0.u64) },
232 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDR_EL0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[0].u64) },
233 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[1].u64) },
234 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MDCCINT_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, MDccInt.u64) }
235};
236/** Paging related system registers (CPUMCTX_EXTRN_SCTLR_TCR_TTBR). */
237static const struct
238{
239 uint64_t idKvmReg;
240 uint32_t offCpumCtx;
241} s_aCpumSysRegsPg[] =
242{
243 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_SCTRL_EL1), RT_UOFFSETOF(CPUMCTX, Sctlr.u64) },
244 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TCR_EL1), RT_UOFFSETOF(CPUMCTX, Tcr.u64) },
245 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TTBR0_EL1), RT_UOFFSETOF(CPUMCTX, Ttbr0.u64) },
246 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TTBR1_EL1), RT_UOFFSETOF(CPUMCTX, Ttbr1.u64) }
247};
248/** Debug system registers. */
249static const struct
250{
251 uint64_t idKvmReg;
252 uint32_t offCpumCtx;
253} s_aCpumDbgRegs[] =
254{
255#define CPUM_DBGREG_EMIT(a_BorW, a_Idx) \
256 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_DBG ## a_BorW ## CRn_EL1(a_Idx)), RT_UOFFSETOF(CPUMCTX, a ## a_BorW ## p[a_Idx].Ctrl.u64) }, \
257 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_DBG ## a_BorW ## VRn_EL1(a_Idx)), RT_UOFFSETOF(CPUMCTX, a ## a_BorW ## p[a_Idx].Value.u64) }
258 /* Breakpoint registers. */
259 CPUM_DBGREG_EMIT(B, 0),
260 CPUM_DBGREG_EMIT(B, 1),
261 CPUM_DBGREG_EMIT(B, 2),
262 CPUM_DBGREG_EMIT(B, 3),
263 CPUM_DBGREG_EMIT(B, 4),
264 CPUM_DBGREG_EMIT(B, 5),
265 CPUM_DBGREG_EMIT(B, 6),
266 CPUM_DBGREG_EMIT(B, 7),
267 CPUM_DBGREG_EMIT(B, 8),
268 CPUM_DBGREG_EMIT(B, 9),
269 CPUM_DBGREG_EMIT(B, 10),
270 CPUM_DBGREG_EMIT(B, 11),
271 CPUM_DBGREG_EMIT(B, 12),
272 CPUM_DBGREG_EMIT(B, 13),
273 CPUM_DBGREG_EMIT(B, 14),
274 CPUM_DBGREG_EMIT(B, 15),
275 /* Watchpoint registers. */
276 CPUM_DBGREG_EMIT(W, 0),
277 CPUM_DBGREG_EMIT(W, 1),
278 CPUM_DBGREG_EMIT(W, 2),
279 CPUM_DBGREG_EMIT(W, 3),
280 CPUM_DBGREG_EMIT(W, 4),
281 CPUM_DBGREG_EMIT(W, 5),
282 CPUM_DBGREG_EMIT(W, 6),
283 CPUM_DBGREG_EMIT(W, 7),
284 CPUM_DBGREG_EMIT(W, 8),
285 CPUM_DBGREG_EMIT(W, 9),
286 CPUM_DBGREG_EMIT(W, 10),
287 CPUM_DBGREG_EMIT(W, 11),
288 CPUM_DBGREG_EMIT(W, 12),
289 CPUM_DBGREG_EMIT(W, 13),
290 CPUM_DBGREG_EMIT(W, 14),
291 CPUM_DBGREG_EMIT(W, 15),
292 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MDSCR_EL1), RT_UOFFSETOF(CPUMCTX, Mdscr.u64) }
293#undef CPUM_DBGREG_EMIT
294};
295/** PAuth key system registers. */
296static const struct
297{
298 uint64_t idKvmReg;
299 uint32_t offCpumCtx;
300} s_aCpumPAuthKeyRegs[] =
301{
302 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apda.Low.u64) },
303 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apda.High.u64) },
304 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDBKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apdb.Low.u64) },
305 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDBKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apdb.High.u64) },
306 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APGAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apga.Low.u64) },
307 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APGAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apga.High.u64) },
308 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apia.Low.u64) },
309 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apia.High.u64) },
310 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIBKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apib.Low.u64) },
311 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIBKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apib.High.u64) }
312};
313/** ID registers. */
314static const struct
315{
316 uint64_t idKvmReg;
317 uint32_t offIdStruct;
318} s_aIdRegs[] =
319{
320 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64DFR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Dfr0El1) },
321 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64DFR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Dfr1El1) },
322 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64ISAR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Isar0El1) },
323 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64ISAR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Isar1El1) },
324 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Mmfr0El1) },
325 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Mmfr1El1) },
326 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR2_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Mmfr2El1) },
327 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64PFR0_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Pfr0El1) },
328 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64PFR1_EL1), RT_UOFFSETOF(CPUMARMV8IDREGS, u64RegIdAa64Pfr1El1) }
329};
330
331
332
333/* Forward declarations of things called by the template. */
334static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo);
335
336
337/* Instantiate the common bits we share with the x86 KVM backend. */
338#include "NEMR3NativeTemplate-linux.cpp.h"
339
340
341/**
342 * Queries and logs the supported register list from KVM.
343 *
344 * @returns VBox status code.
345 * @param fdVCpu The file descriptor number of vCPU 0.
346 */
347static int nemR3LnxLogRegList(int fdVCpu)
348{
349 struct KVM_REG_LIST
350 {
351 uint64_t cRegs;
352 uint64_t aRegs[1024];
353 } RegList; RT_ZERO(RegList);
354
355 RegList.cRegs = RT_ELEMENTS(RegList.aRegs);
356 int rcLnx = ioctl(fdVCpu, KVM_GET_REG_LIST, &RegList);
357 if (rcLnx != 0)
358 return RTErrConvertFromErrno(errno);
359
360 LogRel(("NEM: KVM vCPU registers:\n"));
361
362 for (uint32_t i = 0; i < RegList.cRegs; i++)
363 LogRel(("NEM: %36s: %#RX64\n", "Unknown" /** @todo */, RegList.aRegs[i]));
364
365 return VINF_SUCCESS;
366}
367
368
369/**
370 * Sets the given attribute in KVM to the given value.
371 *
372 * @returns VBox status code.
373 * @param pVM The VM instance.
374 * @param u32Grp The device attribute group being set.
375 * @param u32Attr The actual attribute inside the group being set.
376 * @param pvAttrVal Where the attribute value to set.
377 * @param pszAttribute Attribute description for logging.
378 * @param pErrInfo Optional error information.
379 */
380static int nemR3LnxSetAttribute(PVM pVM, uint32_t u32Grp, uint32_t u32Attr, const void *pvAttrVal, const char *pszAttribute,
381 PRTERRINFO pErrInfo)
382{
383 struct kvm_device_attr DevAttr;
384
385 DevAttr.flags = 0;
386 DevAttr.group = u32Grp;
387 DevAttr.attr = u32Attr;
388 DevAttr.addr = (uintptr_t)pvAttrVal;
389 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_HAS_DEVICE_ATTR, &DevAttr);
390 if (rcLnx < 0)
391 return RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno),
392 N_("KVM error: KVM doesn't support setting the attribute \"%s\" (%d)"),
393 pszAttribute, errno);
394
395 rcLnx = ioctl(pVM->nem.s.fdVm, KVM_SET_DEVICE_ATTR, &DevAttr);
396 if (rcLnx < 0)
397 return RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno),
398 N_("KVM error: Setting the attribute \"%s\" for KVM failed (%d)"),
399 pszAttribute, errno);
400
401 return VINF_SUCCESS;
402}
403
404
405DECL_FORCE_INLINE(int) nemR3LnxKvmSetQueryReg(PVMCPUCC pVCpu, bool fQuery, uint64_t idKvmReg, const void *pv)
406{
407 struct kvm_one_reg Reg;
408 Reg.id = idKvmReg;
409 Reg.addr = (uintptr_t)pv;
410
411 /*
412 * Who thought that this API was a good idea? Supporting to query/set just one register
413 * at a time is horribly inefficient.
414 */
415 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, fQuery ? KVM_GET_ONE_REG : KVM_SET_ONE_REG, &Reg);
416 if (!rcLnx)
417 return 0;
418
419 return RTErrConvertFromErrno(-rcLnx);
420}
421
422DECLINLINE(int) nemR3LnxKvmQueryRegU64(PVMCPUCC pVCpu, uint64_t idKvmReg, uint64_t *pu64)
423{
424 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pu64);
425}
426
427
428DECLINLINE(int) nemR3LnxKvmQueryRegU32(PVMCPUCC pVCpu, uint64_t idKvmReg, uint32_t *pu32)
429{
430 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pu32);
431}
432
433
434DECLINLINE(int) nemR3LnxKvmQueryRegPV(PVMCPUCC pVCpu, uint64_t idKvmReg, void *pv)
435{
436 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pv);
437}
438
439
440DECLINLINE(int) nemR3LnxKvmSetRegU64(PVMCPUCC pVCpu, uint64_t idKvmReg, const uint64_t *pu64)
441{
442 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pu64);
443}
444
445
446DECLINLINE(int) nemR3LnxKvmSetRegU32(PVMCPUCC pVCpu, uint64_t idKvmReg, const uint32_t *pu32)
447{
448 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pu32);
449}
450
451
452DECLINLINE(int) nemR3LnxKvmSetRegPV(PVMCPUCC pVCpu, uint64_t idKvmReg, const void *pv)
453{
454 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pv);
455}
456
457
458/**
459 * Does the early setup of a KVM VM.
460 *
461 * @returns VBox status code.
462 * @param pVM The cross context VM structure.
463 * @param pErrInfo Where to always return error info.
464 */
465static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
466{
467 AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
468
469 /*
470 * Create the VCpus.
471 */
472 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
473 {
474 PVMCPU pVCpu = pVM->apCpusR3[idCpu];
475
476 /* Create it. */
477 pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
478 if (pVCpu->nem.s.fdVCpu < 0)
479 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
480
481 /* Map the KVM_RUN area. */
482 pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ | PROT_WRITE, MAP_SHARED,
483 pVCpu->nem.s.fdVCpu, 0 /*offset*/);
484 if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
485 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "mmap failed for VCpu #%u: %d", idCpu, errno);
486
487 /* Initialize the vCPU. */
488 struct kvm_vcpu_init VCpuInit; RT_ZERO(VCpuInit);
489 VCpuInit.target = KVM_ARM_TARGET_GENERIC_V8;
490 /** @todo Enable features. */
491 if (ioctl(pVCpu->nem.s.fdVCpu, KVM_ARM_VCPU_INIT, &VCpuInit) != 0)
492 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_ARM_VCPU_INIT failed for VCpu #%u: %d", idCpu, errno);
493
494#if 0
495 uint32_t fFeatures = 0; /** @todo SVE */
496 if (ioctl(pVCpu->nem.s.fdVCpu, KVM_ARM_VCPU_FINALIZE, &fFeatures) != 0)
497 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_ARM_VCPU_FINALIZE failed for VCpu #%u: %d", idCpu, errno);
498#endif
499
500 if (idCpu == 0)
501 {
502 /* Query the supported register list and log it. */
503 int rc = nemR3LnxLogRegList(pVCpu->nem.s.fdVCpu);
504 if (RT_FAILURE(rc))
505 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "Querying the supported register list failed with %Rrc", rc);
506
507 /* Need to query the ID registers and populate CPUM. */
508 CPUMARMV8IDREGS IdRegs; RT_ZERO(IdRegs);
509 for (uint32_t i = 0; i < RT_ELEMENTS(s_aIdRegs); i++)
510 {
511 uint64_t *pu64 = (uint64_t *)((uint8_t *)&IdRegs + s_aIdRegs[i].offIdStruct);
512 rc = nemR3LnxKvmQueryRegU64(pVCpu, s_aIdRegs[i].idKvmReg, pu64);
513 if (RT_FAILURE(rc))
514 return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
515 "Querying register %#x failed: %Rrc", s_aIdRegs[i].idKvmReg, rc);
516 }
517
518 rc = CPUMR3PopulateFeaturesByIdRegisters(pVM, &IdRegs);
519 if (RT_FAILURE(rc))
520 return rc;
521 }
522 }
523
524 /*
525 * Setup the SMCCC filter to get exits for PSCI related
526 * guest calls (to support SMP, power off and reset).
527 */
528 struct kvm_smccc_filter SmcccPsciFilter; RT_ZERO(SmcccPsciFilter);
529 SmcccPsciFilter.base = ARM_PSCI_FUNC_ID_CREATE_FAST_64(ARM_PSCI_FUNC_ID_PSCI_VERSION);
530 SmcccPsciFilter.nr_functions = ARM_PSCI_FUNC_ID_CREATE_FAST_64(ARM_PSCI_FUNC_ID_SYSTEM_RESET) - SmcccPsciFilter.base + 1;
531 SmcccPsciFilter.action = KVM_SMCCC_FILTER_FWD_TO_USER;
532 int rc = nemR3LnxSetAttribute(pVM, KVM_ARM_VM_SMCCC_CTRL, KVM_ARM_VM_SMCCC_FILTER, &SmcccPsciFilter,
533 "KVM_ARM_VM_SMCCC_FILTER", pErrInfo);
534 if (RT_FAILURE(rc))
535 return rc;
536
537 SmcccPsciFilter.base = ARM_PSCI_FUNC_ID_CREATE_FAST_32(ARM_PSCI_FUNC_ID_PSCI_VERSION);
538 SmcccPsciFilter.nr_functions = ARM_PSCI_FUNC_ID_CREATE_FAST_32(ARM_PSCI_FUNC_ID_SYSTEM_RESET) - SmcccPsciFilter.base + 1;
539 SmcccPsciFilter.action = KVM_SMCCC_FILTER_FWD_TO_USER;
540 rc = nemR3LnxSetAttribute(pVM, KVM_ARM_VM_SMCCC_CTRL, KVM_ARM_VM_SMCCC_FILTER, &SmcccPsciFilter,
541 "KVM_ARM_VM_SMCCC_FILTER", pErrInfo);
542 if (RT_FAILURE(rc))
543 return rc;
544
545 return VINF_SUCCESS;
546}
547
548
549int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
550{
551 /*
552 * Make RTThreadPoke work again (disabled for avoiding unnecessary
553 * critical section issues in ring-0).
554 */
555 if (enmWhat == VMINITCOMPLETED_RING3)
556 VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
557
558 return VINF_SUCCESS;
559}
560
561
562/*********************************************************************************************************************************
563* CPU State *
564*********************************************************************************************************************************/
565
566/**
567 * Sets the given general purpose register to the given value.
568 *
569 * @param pVCpu The cross context virtual CPU structure of the
570 * calling EMT.
571 * @param uReg The register index.
572 * @param f64BitReg Flag whether to operate on a 64-bit or 32-bit register.
573 * @param fSignExtend Flag whether to sign extend the value.
574 * @param u64Val The value.
575 */
576DECLINLINE(void) nemR3LnxSetGReg(PVMCPU pVCpu, uint8_t uReg, bool f64BitReg, bool fSignExtend, uint64_t u64Val)
577{
578 AssertReturnVoid(uReg < 31);
579
580 if (f64BitReg)
581 pVCpu->cpum.GstCtx.aGRegs[uReg].x = fSignExtend ? (int64_t)u64Val : u64Val;
582 else
583 pVCpu->cpum.GstCtx.aGRegs[uReg].x = (uint64_t)(fSignExtend ? (int32_t)u64Val : (uint32_t)u64Val);
584
585 /* Mark the register as not extern anymore. */
586 switch (uReg)
587 {
588 case 0:
589 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X0;
590 break;
591 case 1:
592 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X1;
593 break;
594 case 2:
595 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X2;
596 break;
597 case 3:
598 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X3;
599 break;
600 default:
601 AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_X4_X28));
602 /** @todo We need to import all missing registers in order to clear this flag (or just set it in HV from here). */
603 }
604}
605
606
607/**
608 * Gets the given general purpose register and returns the value.
609 *
610 * @returns Value from the given register.
611 * @param pVCpu The cross context virtual CPU structure of the
612 * calling EMT.
613 * @param uReg The register index.
614 */
615DECLINLINE(uint64_t) nemR3LnxGetGReg(PVMCPU pVCpu, uint8_t uReg)
616{
617 AssertReturn(uReg <= ARMV8_A64_REG_XZR, 0);
618
619 if (uReg == ARMV8_A64_REG_XZR)
620 return 0;
621
622 /** @todo Import the register if extern. */
623 //AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_GPRS_MASK));
624
625 return pVCpu->cpum.GstCtx.aGRegs[uReg].x;
626}
627
628/**
629 * Worker that imports selected state from KVM.
630 */
631static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx)
632{
633 fWhat &= pVCpu->cpum.GstCtx.fExtrn;
634 if (!fWhat)
635 return VINF_SUCCESS;
636
637#if 0
638 hv_return_t hrc = hv_vcpu_get_sys_reg(pVCpu->nem.s.hVCpu, HV_SYS_REG_CNTV_CTL_EL0, &pVCpu->cpum.GstCtx.CntvCtlEl0);
639 if (hrc == HV_SUCCESS)
640 hrc = hv_vcpu_get_sys_reg(pVCpu->nem.s.hVCpu, HV_SYS_REG_CNTV_CVAL_EL0, &pVCpu->cpum.GstCtx.CntvCValEl0);
641#endif
642
643 int rc = VINF_SUCCESS;
644 if (fWhat & (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
645 {
646 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
647 {
648 if (s_aCpumRegs[i].fCpumExtrn & fWhat)
649 {
650 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
651 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumRegs[i].idKvmReg, pu64);
652 }
653 }
654 }
655
656 if ( rc == VINF_SUCCESS
657 && (fWhat & CPUMCTX_EXTRN_FPCR))
658 {
659 uint32_t u32Tmp;
660 rc |= nemR3LnxKvmQueryRegU32(pVCpu, KVM_ARM64_REG_FP_FPCR, &u32Tmp);
661 if (rc == VINF_SUCCESS)
662 pVCpu->cpum.GstCtx.fpcr = u32Tmp;
663 }
664
665 if ( rc == VINF_SUCCESS
666 && (fWhat & CPUMCTX_EXTRN_FPSR))
667 {
668 uint32_t u32Tmp;
669 rc |= nemR3LnxKvmQueryRegU32(pVCpu, KVM_ARM64_REG_FP_FPSR, &u32Tmp);
670 if (rc == VINF_SUCCESS)
671 pVCpu->cpum.GstCtx.fpsr = u32Tmp;
672 }
673
674 if ( rc == VINF_SUCCESS
675 && (fWhat & CPUMCTX_EXTRN_V0_V31))
676 {
677 /* SIMD/FP registers. */
678 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
679 {
680 void *pu128 = (void *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
681 rc |= nemR3LnxKvmQueryRegPV(pVCpu, s_aCpumFpRegs[i].idKvmReg, pu128);
682 }
683 }
684
685 if ( rc == VINF_SUCCESS
686 && (fWhat & CPUMCTX_EXTRN_SYSREG_DEBUG))
687 {
688 /* Debug registers. */
689 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumDbgRegs); i++)
690 {
691 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumDbgRegs[i].offCpumCtx);
692 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumDbgRegs[i].idKvmReg, pu64);
693 }
694 }
695
696 if ( rc == VINF_SUCCESS
697 && (fWhat & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS))
698 {
699 /* PAuth registers. */
700 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
701 {
702 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
703 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumPAuthKeyRegs[i].idKvmReg, pu64);
704 }
705 }
706
707 if ( rc == VINF_SUCCESS
708 && (fWhat & (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SYSREG_MISC)))
709 {
710 /* System registers. */
711 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
712 {
713 if (s_aCpumSysRegs[i].fCpumExtrn & fWhat)
714 {
715 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
716 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumSysRegs[i].idKvmReg, pu64);
717 }
718 }
719 }
720
721 /* The paging related system registers need to be treated differently as they might invoke a PGM mode change. */
722 bool fPgModeChange = false;
723 uint64_t u64RegSctlrEl1;
724 uint64_t u64RegTcrEl1;
725 if ( rc == VINF_SUCCESS
726 && (fWhat & CPUMCTX_EXTRN_SCTLR_TCR_TTBR))
727 {
728 rc |= nemR3LnxKvmQueryRegU64(pVCpu, ARMV8_AARCH64_SYSREG_SCTRL_EL1, &u64RegSctlrEl1);
729 rc |= nemR3LnxKvmQueryRegU64(pVCpu, ARMV8_AARCH64_SYSREG_TCR_EL1, &u64RegTcrEl1);
730 rc |= nemR3LnxKvmQueryRegU64(pVCpu, ARMV8_AARCH64_SYSREG_TTBR0_EL1, &pVCpu->cpum.GstCtx.Ttbr0.u64);
731 rc |= nemR3LnxKvmQueryRegU64(pVCpu, ARMV8_AARCH64_SYSREG_TTBR1_EL1, &pVCpu->cpum.GstCtx.Ttbr1.u64);
732 if ( rc == VINF_SUCCESS
733 && ( u64RegSctlrEl1 != pVCpu->cpum.GstCtx.Sctlr.u64
734 || u64RegTcrEl1 != pVCpu->cpum.GstCtx.Tcr.u64))
735 {
736 pVCpu->cpum.GstCtx.Sctlr.u64 = u64RegSctlrEl1;
737 pVCpu->cpum.GstCtx.Tcr.u64 = u64RegTcrEl1;
738 fPgModeChange = true;
739 }
740 }
741
742 if ( rc == VINF_SUCCESS
743 && (fWhat & CPUMCTX_EXTRN_PSTATE))
744 {
745 uint64_t u64Tmp;
746 rc |= nemR3LnxKvmQueryRegU64(pVCpu, KVM_ARM64_REG_PSTATE, &u64Tmp);
747 if (rc == VINF_SUCCESS)
748 pVCpu->cpum.GstCtx.fPState = (uint32_t)u64Tmp;
749
750 }
751
752 if (fPgModeChange)
753 {
754 rc = PGMChangeMode(pVCpu, 1 /*bEl*/, u64RegSctlrEl1, u64RegTcrEl1);
755 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
756 }
757
758 /*
759 * Update the external mask.
760 */
761 pCtx->fExtrn &= ~fWhat;
762 pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
763 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
764 pVCpu->cpum.GstCtx.fExtrn = 0;
765
766 return VINF_SUCCESS;
767}
768
769
770/**
771 * Interface for importing state on demand (used by IEM).
772 *
773 * @returns VBox status code.
774 * @param pVCpu The cross context CPU structure.
775 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
776 */
777VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
778{
779 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
780 return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx);
781}
782
783
784/**
785 * Exports state to KVM.
786 */
787static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
788{
789 uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
790 Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL); RT_NOREF(fExtrn);
791
792 RT_NOREF(pVM);
793 int rc = VINF_SUCCESS;
794 if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
795 != (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
796 {
797 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
798 {
799 if (!(s_aCpumRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
800 {
801 const uint64_t *pu64 = (const uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
802 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumRegs[i].idKvmReg, pu64);
803 }
804 }
805 }
806
807 if ( rc == VINF_SUCCESS
808 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_FPCR))
809 {
810 uint32_t u32Tmp = pVCpu->cpum.GstCtx.fpcr;
811 rc |= nemR3LnxKvmSetRegU32(pVCpu, KVM_ARM64_REG_FP_FPCR, &u32Tmp);
812 }
813
814 if ( rc == VINF_SUCCESS
815 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_FPSR))
816 {
817 uint32_t u32Tmp = pVCpu->cpum.GstCtx.fpsr;
818 rc |= nemR3LnxKvmSetRegU32(pVCpu, KVM_ARM64_REG_FP_FPSR, &u32Tmp);
819 }
820
821 if ( rc == VINF_SUCCESS
822 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_V0_V31))
823 {
824 /* SIMD/FP registers. */
825 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
826 {
827 void *pu128 = (void *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
828 rc |= nemR3LnxKvmSetRegPV(pVCpu, s_aCpumFpRegs[i].idKvmReg, pu128);
829 }
830 }
831
832 if ( rc == VINF_SUCCESS
833 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SYSREG_DEBUG))
834 {
835 /* Debug registers. */
836 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumDbgRegs); i++)
837 {
838 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumDbgRegs[i].offCpumCtx);
839 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumDbgRegs[i].idKvmReg, pu64);
840 }
841 }
842
843 if ( rc == VINF_SUCCESS
844 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS))
845 {
846 /* PAuth registers. */
847 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
848 {
849 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
850 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumPAuthKeyRegs[i].idKvmReg, pu64);
851 }
852 }
853
854 if ( rc == VINF_SUCCESS
855 && (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC))
856 != (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC))
857 {
858 /* System registers. */
859 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
860 {
861 if (!(s_aCpumSysRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
862 {
863 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
864 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumSysRegs[i].idKvmReg, pu64);
865 }
866 }
867 }
868
869 if ( rc == VINF_SUCCESS
870 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SCTLR_TCR_TTBR))
871 {
872 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegsPg); i++)
873 {
874 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegsPg[i].offCpumCtx);
875 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumSysRegsPg[i].idKvmReg, pu64);
876 }
877 }
878
879 if ( rc == VINF_SUCCESS
880 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_PSTATE))
881 {
882 uint64_t u64Tmp = pVCpu->cpum.GstCtx.fPState;
883 rc = nemR3LnxKvmSetRegU64(pVCpu, KVM_ARM64_REG_PSTATE, &u64Tmp);
884 }
885
886 /*
887 * KVM now owns all the state.
888 */
889 pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM | CPUMCTX_EXTRN_ALL;
890 return VINF_SUCCESS;
891}
892
893
894/**
895 * Query the CPU tick counter and optionally the TSC_AUX MSR value.
896 *
897 * @returns VBox status code.
898 * @param pVCpu The cross context CPU structure.
899 * @param pcTicks Where to return the CPU tick count.
900 * @param puAux Where to return the TSC_AUX register value.
901 */
902VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
903{
904 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
905 // KVM_GET_CLOCK?
906 RT_NOREF(pVCpu, pcTicks, puAux);
907 return VINF_SUCCESS;
908}
909
910
911/**
912 * Resumes CPU clock (TSC) on all virtual CPUs.
913 *
914 * This is called by TM when the VM is started, restored, resumed or similar.
915 *
916 * @returns VBox status code.
917 * @param pVM The cross context VM structure.
918 * @param pVCpu The cross context CPU structure of the calling EMT.
919 * @param uPausedTscValue The TSC value at the time of pausing.
920 */
921VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
922{
923 // KVM_SET_CLOCK?
924 RT_NOREF(pVM, pVCpu, uPausedTscValue);
925 return VINF_SUCCESS;
926}
927
928
929VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
930{
931 RT_NOREF(pVM);
932 return NEM_FEAT_F_NESTED_PAGING
933 | NEM_FEAT_F_FULL_GST_EXEC;
934}
935
936
937
938/*********************************************************************************************************************************
939* Execution *
940*********************************************************************************************************************************/
941
942
943VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
944{
945 RT_NOREF(pVM, pVCpu);
946 Assert(VM_IS_NEM_ENABLED(pVM));
947 return true;
948}
949
950
951bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
952{
953 NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
954 return false;
955}
956
957
958void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
959{
960 int rc = RTThreadPoke(pVCpu->hThread);
961 LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
962 AssertRC(rc);
963 RT_NOREF(pVM, fFlags);
964}
965
966
967DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
968{
969 RT_NOREF(pVM, fUseDebugLoop);
970 return false;
971}
972
973
974DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
975{
976 RT_NOREF(pVM, pVCpu, fUseDebugLoop);
977 return false;
978}
979
980
981DECL_FORCE_INLINE(int) nemR3LnxKvmUpdateIntrState(PVM pVM, PVMCPU pVCpu, bool fIrq, bool fAsserted)
982{
983 struct kvm_irq_level IrqLvl;
984
985 LogFlowFunc(("pVM=%p pVCpu=%p fIrq=%RTbool fAsserted=%RTbool\n",
986 pVM, pVCpu, fIrq, fAsserted));
987
988 IrqLvl.irq = ((uint32_t)KVM_ARM_IRQ_TYPE_CPU << 24) /* Directly drives CPU interrupt lines. */
989 | (pVCpu->idCpu & 0xff) << 16
990 | (fIrq ? 0 : 1);
991 IrqLvl.level = fAsserted ? 1 : 0;
992 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_IRQ_LINE, &IrqLvl);
993 AssertReturn(rcLnx == 0, VERR_NEM_IPE_9);
994
995 return VINF_SUCCESS;
996}
997
998
999/**
1000 * Deals with pending interrupt FFs prior to executing guest code.
1001 */
1002static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu)
1003{
1004 LogFlowFunc(("pVCpu=%p{.idCpu=%u} fIrq=%RTbool fFiq=%RTbool\n",
1005 pVCpu, pVCpu->idCpu,
1006 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ),
1007 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_FIQ)));
1008
1009 bool fIrq = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ);
1010 bool fFiq = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_FIQ);
1011
1012 /* Update the pending interrupt state. */
1013 if (fIrq != pVCpu->nem.s.fIrqLastSeen)
1014 {
1015 int rc = nemR3LnxKvmUpdateIntrState(pVM, pVCpu, true /*fIrq*/, fIrq);
1016 AssertRCReturn(rc, VERR_NEM_IPE_9);
1017 pVCpu->nem.s.fIrqLastSeen = fIrq;
1018 }
1019
1020 if (fFiq != pVCpu->nem.s.fFiqLastSeen)
1021 {
1022 int rc = nemR3LnxKvmUpdateIntrState(pVM, pVCpu, false /*fIrq*/, fFiq);
1023 AssertRCReturn(rc, VERR_NEM_IPE_9);
1024 pVCpu->nem.s.fFiqLastSeen = fFiq;
1025 }
1026
1027 return VINF_SUCCESS;
1028}
1029
1030
1031#if 0
1032/**
1033 * Handles KVM_EXIT_INTERNAL_ERROR.
1034 */
1035static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
1036{
1037 Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
1038 pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
1039
1040 /*
1041 * Deal with each suberror, returning if we don't want IEM to handle it.
1042 */
1043 switch (pRun->internal.suberror)
1044 {
1045 case KVM_INTERNAL_ERROR_EMULATION:
1046 {
1047 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
1048 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1049 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
1050 break;
1051 }
1052
1053 case KVM_INTERNAL_ERROR_SIMUL_EX:
1054 case KVM_INTERNAL_ERROR_DELIVERY_EV:
1055 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
1056 default:
1057 {
1058 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
1059 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1060 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
1061 const char *pszName;
1062 switch (pRun->internal.suberror)
1063 {
1064 case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
1065 case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
1066 case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
1067 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
1068 default: pszName = "unknown"; break;
1069 }
1070 LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
1071 pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
1072 return VERR_NEM_IPE_0;
1073 }
1074 }
1075
1076 /*
1077 * Execute instruction in IEM and try get on with it.
1078 */
1079 Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
1080 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
1081 VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
1082 IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_INHIBIT_INT
1083 | CPUMCTX_EXTRN_INHIBIT_NMI,
1084 &pVCpu->cpum.GstCtx, pRun);
1085 if (RT_SUCCESS(rcStrict))
1086 rcStrict = IEMExecOne(pVCpu);
1087 return rcStrict;
1088}
1089#endif
1090
1091
1092/**
1093 * Handles KVM_EXIT_MMIO.
1094 */
1095static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1096{
1097 /*
1098 * Input validation.
1099 */
1100 Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
1101 Assert(pRun->mmio.is_write <= 1);
1102
1103#if 0
1104 /*
1105 * We cannot easily act on the exit history here, because the MMIO port
1106 * exit is stateful and the instruction will be completed in the next
1107 * KVM_RUN call. There seems no way to circumvent this.
1108 */
1109 EMHistoryAddExit(pVCpu,
1110 pRun->mmio.is_write
1111 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1112 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1113 pRun->s.regs.regs.pc, ASMReadTSC());
1114#else
1115 RT_NOREF(pVCpu);
1116#endif
1117
1118 /*
1119 * Do the requested job.
1120 */
1121 VBOXSTRICTRC rcStrict;
1122 if (pRun->mmio.is_write)
1123 {
1124 rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1125 Log4(("MmioExit/%u:WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
1126 pVCpu->idCpu,
1127 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1128 }
1129 else
1130 {
1131 rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1132 Log4(("MmioExit/%u: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
1133 pVCpu->idCpu,
1134 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1135 }
1136 return rcStrict;
1137}
1138
1139
1140/**
1141 * Handles KVM_EXIT_HYPERCALL.
1142 */
1143static VBOXSTRICTRC nemHCLnxHandleExitHypercall(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1144{
1145#if 0
1146 /*
1147 * We cannot easily act on the exit history here, because the MMIO port
1148 * exit is stateful and the instruction will be completed in the next
1149 * KVM_RUN call. There seems no way to circumvent this.
1150 */
1151 EMHistoryAddExit(pVCpu,
1152 pRun->mmio.is_write
1153 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1154 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1155 pRun->s.regs.regs.pc, ASMReadTSC());
1156#else
1157 RT_NOREF(pVCpu);
1158#endif
1159
1160 /*
1161 * Do the requested job.
1162 */
1163 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1164
1165 /** @todo Raise exception to EL1 if PSCI not configured. */
1166 /** @todo Need a generic mechanism here to pass this to, GIM maybe?. */
1167 uint32_t uFunId = pRun->hypercall.nr;
1168 bool fHvc64 = RT_BOOL(uFunId & ARM_SMCCC_FUNC_ID_64BIT); RT_NOREF(fHvc64);
1169 uint32_t uEntity = ARM_SMCCC_FUNC_ID_ENTITY_GET(uFunId);
1170 uint32_t uFunNum = ARM_SMCCC_FUNC_ID_NUM_GET(uFunId);
1171 if (uEntity == ARM_SMCCC_FUNC_ID_ENTITY_STD_SEC_SERVICE)
1172 {
1173 rcStrict = nemHCLnxImportState(pVCpu, CPUMCTX_EXTRN_X0 | CPUMCTX_EXTRN_X1 | CPUMCTX_EXTRN_X2 | CPUMCTX_EXTRN_X3,
1174 &pVCpu->cpum.GstCtx);
1175 if (rcStrict != VINF_SUCCESS)
1176 return rcStrict;
1177
1178 switch (uFunNum)
1179 {
1180 case ARM_PSCI_FUNC_ID_PSCI_VERSION:
1181 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, ARM_PSCI_FUNC_ID_PSCI_VERSION_SET(1, 2));
1182 break;
1183 case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
1184 rcStrict = VMR3PowerOff(pVM->pUVM);
1185 break;
1186 case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
1187 case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
1188 {
1189 bool fHaltOnReset;
1190 int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
1191 if (RT_SUCCESS(rc) && fHaltOnReset)
1192 {
1193 Log(("nemHCLnxHandleExitHypercall: Halt On Reset!\n"));
1194 rcStrict = VINF_EM_HALT;
1195 }
1196 else
1197 {
1198 /** @todo pVM->pdm.s.fResetFlags = fFlags; */
1199 VM_FF_SET(pVM, VM_FF_RESET);
1200 rcStrict = VINF_EM_RESET;
1201 }
1202 break;
1203 }
1204 case ARM_PSCI_FUNC_ID_CPU_ON:
1205 {
1206 uint64_t u64TgtCpu = nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X1);
1207 RTGCPHYS GCPhysExecAddr = nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X2);
1208 uint64_t u64CtxId = nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X3);
1209 VMMR3CpuOn(pVM, u64TgtCpu & 0xff, GCPhysExecAddr, u64CtxId);
1210 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, true /*f64BitReg*/, false /*fSignExtend*/, ARM_PSCI_STS_SUCCESS);
1211 break;
1212 }
1213 case ARM_PSCI_FUNC_ID_PSCI_FEATURES:
1214 {
1215 uint32_t u32FunNum = (uint32_t)nemR3LnxGetGReg(pVCpu, ARMV8_A64_REG_X1);
1216 switch (u32FunNum)
1217 {
1218 case ARM_PSCI_FUNC_ID_PSCI_VERSION:
1219 case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
1220 case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
1221 case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
1222 case ARM_PSCI_FUNC_ID_CPU_ON:
1223 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0,
1224 false /*f64BitReg*/, false /*fSignExtend*/,
1225 (uint64_t)ARM_PSCI_STS_SUCCESS);
1226 break;
1227 default:
1228 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0,
1229 false /*f64BitReg*/, false /*fSignExtend*/,
1230 (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1231 }
1232 break;
1233 }
1234 default:
1235 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1236 }
1237 }
1238 else
1239 nemR3LnxSetGReg(pVCpu, ARMV8_A64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1240
1241
1242 return rcStrict;
1243}
1244
1245
1246static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun, bool *pfStatefulExit)
1247{
1248 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
1249
1250 if (pVCpu->nem.s.fIrqDeviceLvls != pRun->s.regs.device_irq_level)
1251 {
1252 uint64_t fChanged = pVCpu->nem.s.fIrqDeviceLvls ^ pRun->s.regs.device_irq_level;
1253
1254 if (fChanged & KVM_ARM_DEV_EL1_VTIMER)
1255 {
1256 TMCpuSetVTimerNextActivation(pVCpu, UINT64_MAX);
1257 PDMGicSetPpi(pVCpu, pVM->nem.s.u32GicPpiVTimer, RT_BOOL(pRun->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER));
1258 }
1259
1260 if (fChanged & KVM_ARM_DEV_EL1_PTIMER)
1261 {
1262 //TMCpuSetVTimerNextActivation(pVCpu, UINT64_MAX);
1263 PDMGicSetPpi(pVCpu, pVM->nem.s.u32GicPpiVTimer, RT_BOOL(pRun->s.regs.device_irq_level & KVM_ARM_DEV_EL1_PTIMER));
1264 }
1265
1266 pVCpu->nem.s.fIrqDeviceLvls = pRun->s.regs.device_irq_level;
1267 }
1268
1269 switch (pRun->exit_reason)
1270 {
1271 case KVM_EXIT_EXCEPTION:
1272 AssertFailed();
1273 break;
1274
1275 case KVM_EXIT_MMIO:
1276 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
1277 *pfStatefulExit = true;
1278 return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
1279
1280 case KVM_EXIT_INTR: /* EINTR */
1281 //EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
1282 // pRun->s.regs.regs.pc, ASMReadTSC());
1283 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
1284 Log5(("Intr/%u\n", pVCpu->idCpu));
1285 return VINF_SUCCESS;
1286
1287 case KVM_EXIT_HYPERCALL:
1288 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
1289 return nemHCLnxHandleExitHypercall(pVM, pVCpu, pRun);
1290
1291#if 0
1292 case KVM_EXIT_DEBUG:
1293 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
1294 AssertFailed();
1295 break;
1296
1297 case KVM_EXIT_SYSTEM_EVENT:
1298 AssertFailed();
1299 break;
1300
1301 case KVM_EXIT_DIRTY_RING_FULL:
1302 AssertFailed();
1303 break;
1304 case KVM_EXIT_AP_RESET_HOLD:
1305 AssertFailed();
1306 break;
1307
1308
1309 case KVM_EXIT_SHUTDOWN:
1310 AssertFailed();
1311 break;
1312
1313 case KVM_EXIT_FAIL_ENTRY:
1314 LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
1315 pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
1316 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
1317 pRun->s.regs.regs.pc, ASMReadTSC());
1318 return VERR_NEM_IPE_1;
1319
1320 case KVM_EXIT_INTERNAL_ERROR:
1321 /* we're counting sub-reasons inside the function. */
1322 return nemR3LnxHandleInternalError(pVCpu, pRun);
1323#endif
1324
1325 /*
1326 * Foreign and unknowns.
1327 */
1328#if 0
1329 case KVM_EXIT_IO:
1330 AssertLogRelMsgFailedReturn(("KVM_EXIT_IO on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1331 case KVM_EXIT_NMI:
1332 AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1333 case KVM_EXIT_EPR:
1334 AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1335 case KVM_EXIT_WATCHDOG:
1336 AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1337 case KVM_EXIT_ARM_NISV:
1338 AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1339 case KVM_EXIT_S390_STSI:
1340 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1341 case KVM_EXIT_S390_TSCH:
1342 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1343 case KVM_EXIT_OSI:
1344 AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1345 case KVM_EXIT_PAPR_HCALL:
1346 AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1347 case KVM_EXIT_S390_UCONTROL:
1348 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1349 case KVM_EXIT_DCR:
1350 AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1351 case KVM_EXIT_S390_SIEIC:
1352 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1353 case KVM_EXIT_S390_RESET:
1354 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1355 case KVM_EXIT_UNKNOWN:
1356 AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1357 case KVM_EXIT_XEN:
1358 AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1359#endif
1360 default:
1361 AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u!\n", pRun->exit_reason, pVCpu->idCpu), VERR_NEM_IPE_1);
1362 }
1363 RT_NOREF(pVM, pVCpu);
1364 return VERR_NOT_IMPLEMENTED;
1365}
1366
1367
1368VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
1369{
1370 /*
1371 * Try switch to NEM runloop state.
1372 */
1373 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
1374 { /* likely */ }
1375 else
1376 {
1377 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1378 LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
1379 return VINF_SUCCESS;
1380 }
1381
1382 /*
1383 * The run loop.
1384 */
1385 struct kvm_run * const pRun = pVCpu->nem.s.pRun;
1386 const bool fSingleStepping = DBGFIsStepping(pVCpu);
1387 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1388 bool fStatefulExit = false; /* For MMIO and IO exits. */
1389 for (unsigned iLoop = 0;; iLoop++)
1390 {
1391 /*
1392 * Sync the interrupt state.
1393 */
1394 rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu);
1395 if (rcStrict == VINF_SUCCESS)
1396 { /* likely */ }
1397 else
1398 {
1399 LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1400 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1401 break;
1402 }
1403
1404 /*
1405 * Ensure KVM has the whole state.
1406 */
1407 if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
1408 {
1409 int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx);
1410 AssertRCReturn(rc2, rc2);
1411 }
1412
1413 /*
1414 * Poll timers and run for a bit.
1415 *
1416 * With the VID approach (ring-0 or ring-3) we can specify a timeout here,
1417 * so we take the time of the next timer event and uses that as a deadline.
1418 * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
1419 */
1420 /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
1421 * the whole polling job when timers have changed... */
1422 uint64_t offDeltaIgnored;
1423 uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
1424 if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
1425 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1426 {
1427 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
1428 {
1429 //LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
1430 // pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1431 // !!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
1432 // pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
1433 TMNotifyStartOfExecution(pVM, pVCpu);
1434
1435 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1436
1437 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
1438 TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
1439
1440#if 0 //def LOG_ENABLED
1441 if (LogIsFlowEnabled())
1442 {
1443 struct kvm_mp_state MpState = {UINT32_MAX};
1444 ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
1445 LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
1446 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
1447 pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
1448 pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
1449 }
1450#endif
1451 fStatefulExit = false;
1452 if (RT_LIKELY(rcLnx == 0 || errno == EINTR))
1453 {
1454 /*
1455 * Deal with the exit.
1456 */
1457 rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1458 if (rcStrict == VINF_SUCCESS)
1459 { /* hopefully likely */ }
1460 else
1461 {
1462 LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1463 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1464 break;
1465 }
1466 }
1467 else
1468 {
1469 int rc2 = RTErrConvertFromErrno(errno);
1470 AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
1471 }
1472
1473 /*
1474 * If no relevant FFs are pending, loop.
1475 */
1476 if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
1477 && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
1478 { /* likely */ }
1479 else
1480 {
1481
1482 /** @todo Try handle pending flags, not just return to EM loops. Take care
1483 * not to set important RCs here unless we've handled an exit. */
1484 LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
1485 pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
1486 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
1487 break;
1488 }
1489 }
1490 else
1491 {
1492 LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
1493 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
1494 break;
1495 }
1496 }
1497 else
1498 {
1499 LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
1500 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
1501 break;
1502 }
1503 } /* the run loop */
1504
1505
1506 /*
1507 * If the last exit was stateful, commit the state we provided before
1508 * returning to the EM loop so we have a consistent state and can safely
1509 * be rescheduled and whatnot. This may require us to make multiple runs
1510 * for larger MMIO and I/O operations. Sigh^3.
1511 *
1512 * Note! There is no 'ing way to reset the kernel side completion callback
1513 * for these stateful i/o exits. Very annoying interface.
1514 */
1515 /** @todo check how this works with string I/O and string MMIO. */
1516 if (fStatefulExit && RT_SUCCESS(rcStrict))
1517 {
1518 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
1519 uint32_t const uOrgExit = pRun->exit_reason;
1520 for (uint32_t i = 0; ; i++)
1521 {
1522 pRun->immediate_exit = 1;
1523 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1524 Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
1525 if (rcLnx == -1 && errno == EINTR)
1526 {
1527 switch (i)
1528 {
1529 case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
1530 case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
1531 case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
1532 default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
1533 }
1534 break;
1535 }
1536 AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
1537 ("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
1538 rcStrict = VERR_NEM_IPE_6);
1539 VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1540 if (rcStrict2 == VINF_SUCCESS || rcStrict2 == rcStrict)
1541 { /* likely */ }
1542 else if (RT_FAILURE(rcStrict2))
1543 {
1544 rcStrict = rcStrict2;
1545 break;
1546 }
1547 else
1548 {
1549 AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
1550 ("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
1551 rcStrict = VERR_NEM_IPE_7);
1552 rcStrict = rcStrict2;
1553 }
1554 }
1555 pRun->immediate_exit = 0;
1556 }
1557
1558 /*
1559 * If the CPU is running, make sure to stop it before we try sync back the
1560 * state and return to EM. We don't sync back the whole state if we can help it.
1561 */
1562 if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
1563 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1564
1565 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
1566 {
1567 /* Try anticipate what we might need. */
1568 uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK /*?*/;
1569 if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
1570 || RT_FAILURE(rcStrict))
1571 fImport = CPUMCTX_EXTRN_ALL;
1572 else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ | VMCPU_FF_INTERRUPT_FIQ))
1573 fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK;
1574
1575 if (pVCpu->cpum.GstCtx.fExtrn & fImport)
1576 {
1577 int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx);
1578 if (RT_SUCCESS(rc2))
1579 pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
1580 else if (RT_SUCCESS(rcStrict))
1581 rcStrict = rc2;
1582 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1583 pVCpu->cpum.GstCtx.fExtrn = 0;
1584 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
1585 }
1586 else
1587 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1588 }
1589 else
1590 {
1591 pVCpu->cpum.GstCtx.fExtrn = 0;
1592 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1593 }
1594
1595 LogFlow(("NEM/%u: %08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc, VBOXSTRICTRC_VAL(rcStrict) ));
1596 return rcStrict;
1597}
1598
1599
1600/** @page pg_nem_linux_armv8 NEM/linux - Native Execution Manager, Linux.
1601 *
1602 * This is using KVM.
1603 *
1604 */
1605
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette