Context Navigation

← Previous Revision
Next Revision →
Blame
Revision Log

VirtioCore.cpp

Last change on this file was 104432, checked in by vboxsync, 3 weeks ago
Virtio/Core: Fixes and descriptor table printing for our debugger's 'info virtionet' bugref:10572 jiraref:VBP-575
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 137.8 KB

Line
1	/* $Id: VirtioCore.cpp 104432 2024-04-25 14:04:47Z vboxsync $ */
2
3	/** @file
4	* VirtioCore - Virtio Core (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
5	*/
6
7	/*
8	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
9	*
10	* This file is part of VirtualBox base platform packages, as
11	* available from https://www.virtualbox.org.
12	*
13	* This program is free software; you can redistribute it and/or
14	* modify it under the terms of the GNU General Public License
15	* as published by the Free Software Foundation, in version 3 of the
16	* License.
17	*
18	* This program is distributed in the hope that it will be useful, but
19	* WITHOUT ANY WARRANTY; without even the implied warranty of
20	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21	* General Public License for more details.
22	*
23	* You should have received a copy of the GNU General Public License
24	* along with this program; if not, see <https://www.gnu.org/licenses>.
25	*
26	* SPDX-License-Identifier: GPL-3.0-only
27	*/
28
29
30	/*********************************************************************************************************************************
31	* Header Files *
32	*********************************************************************************************************************************/
33	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
34
35	#include <iprt/assert.h>
36	#include <iprt/uuid.h>
37	#include <iprt/mem.h>
38	#include <iprt/sg.h>
39	#include <iprt/assert.h>
40	#include <iprt/string.h>
41	#include <iprt/param.h>
42	#include <iprt/types.h>
43	#include <VBox/log.h>
44	#include <VBox/msi.h>
45	#include <iprt/types.h>
46	#include <VBox/AssertGuest.h>
47	#include <VBox/vmm/pdmdev.h>
48	#include "VirtioCore.h"
49
50
51	/*********************************************************************************************************************************
52	* Defined Constants And Macros *
53	*********************************************************************************************************************************/
54
55	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
56	#define VIRTQNAME(a_pVirtio, a_uVirtq) ((a_pVirtio)->aVirtqueues[(a_uVirtq)].szName)
57
58	#define IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq) \
59	(virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq) == 0)
60
61	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
62	#define WAS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
63
64	/**
65	* These defines are used to track guest virtio-net driver writing driver features accepted flags
66	* in two 32-bit operations (in arbitrary order), and one bit dedicated to ensured 'features complete'
67	* is handled once.
68	*/
69	#define DRIVER_FEATURES_0_WRITTEN 1 /*< fDriverFeatures[0] written by guest virtio-net /
70	#define DRIVER_FEATURES_1_WRITTEN 2 /*< fDriverFeatures[1] written by guest virtio-net /
71	#define DRIVER_FEATURES_0_AND_1_WRITTEN 3 /*< Both 32-bit parts of fDriverFeatures[] written /
72	#define DRIVER_FEATURES_COMPLETE_HANDLED 4 /*< Features negotiation complete handler called /
73
74	/**
75	* This macro returns true if the @a a_offAccess and access length (@a
76	* a_cbAccess) are within the range of the mapped capability struct described by
77	* @a a_LocCapData.
78	*
79	* @param[in] a_offAccess Input: The offset into the MMIO bar of the access.
80	* @param[in] a_cbAccess Input: The access size.
81	* @param[out] a_offsetIntoCap Output: uint32_t variable to return the intra-capability offset into.
82	* @param[in] a_LocCapData Input: The capability location info.
83	*/
84	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offsetIntoCap, a_LocCapData) \
85	( ((a_offsetIntoCap) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
86	&& (a_offsetIntoCap) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
87
88
89	/*********************************************************************************************************************************
90	* Structures and Typedefs *
91	*********************************************************************************************************************************/
92
93	/** @name virtq related flags
94	* @{ */
95	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
96	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
97	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
98
99	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
100	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
101	/** @} */
102
103	/**
104	* virtq-related structs
105	* (struct names follow VirtIO 1.0 spec, field names use VBox styled naming, w/respective spec'd name in comments)
106	*/
107	typedef struct virtq_desc
108	{
109	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
110	uint32_t cb; /*< len Buffer length /
111	uint16_t fFlags; /*< flags Buffer specific flags /
112	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
113	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
114
115	typedef struct virtq_avail
116	{
117	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
118	uint16_t uIdx; /*< idx Index of next free ring slot /
119	RT_FLEXIBLE_ARRAY_EXTENSION
120	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
121	//uint16_t uUsedEventIdx; /*< used_event (if VIRTQ_USED_F_EVENT_IDX) /
122	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
123
124	typedef struct virtq_used_elem
125	{
126	uint32_t uDescIdx; /*< idx Start of used desc chain /
127	uint32_t cbElem; /*< len Total len of used desc chain /
128	} VIRTQ_USED_ELEM_T;
129
130	typedef struct virt_used
131	{
132	uint16_t fFlags; /*< flags used ring host-to-guest flags /
133	uint16_t uIdx; /*< idx Index of next ring slot /
134	RT_FLEXIBLE_ARRAY_EXTENSION
135	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
136	//uint16_t uAvailEventIdx; /*< avail_event if (VIRTQ_USED_F_EVENT_IDX) /
137	} VIRTQ_USED_T, *PVIRTQ_USED_T;
138
139	DECLHIDDEN(const char *) virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
140	{
141	switch (enmState)
142	{
143	case kvirtIoVmStateChangedReset: return "VM RESET";
144	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
145	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
146	case kvirtIoVmStateChangedResume: return "VM RESUME";
147	default: return "<BAD ENUM>";
148	}
149	}
150
151	/* Internal Functions */
152
153	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq);
154	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
155
156	#ifdef IN_RING3
157	# ifdef LOG_ENABLED
158	DECLINLINE(uint16_t) virtioCoreR3CountPendingBufs(uint16_t uRingIdx, uint16_t uShadowIdx, uint16_t uQueueSize)
159	{
160	if (uShadowIdx == uRingIdx)
161	return 0;
162	else
163	if (uShadowIdx > uRingIdx)
164	return uShadowIdx - uRingIdx;
165	return uQueueSize - (uRingIdx - uShadowIdx);
166	}
167	# endif
168	#endif
169	/** @name Internal queue operations
170	* @{ */
171
172	/**
173	* Accessor for virtq descriptor
174	*/
175	#ifdef IN_RING3
176	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
177	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
178	{
179	/*
180	* Shut up assertion for legacy virtio-net driver in FreeBSD up to 12.3 (see virtioCoreR3VirtqUsedBufPut()
181	* for more information).
182	*/
183	AssertMsg( IS_DRIVER_OK(pVirtio)
184	\|\| ( pVirtio->fLegacyDriver
185	&& pVirtq->GCPhysVirtqDesc),
186	("Called with guest driver not ready\n"));
187	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
188
189	virtioCoreGCPhysRead(pVirtio, pDevIns,
190	pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * (idxDesc % cVirtqItems),
191	pDesc, sizeof(VIRTQ_DESC_T));
192	}
193	#endif
194
195	/**
196	* Accessors for virtq avail ring
197	*/
198	#ifdef IN_RING3
199	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t availIdx)
200	{
201	uint16_t uDescIdx;
202
203	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
204	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
205	virtioCoreGCPhysRead(pVirtio, pDevIns,
206	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cVirtqItems]),
207	&uDescIdx, sizeof(uDescIdx));
208	return uDescIdx;
209	}
210
211	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
212	{
213	uint16_t uUsedEventIdx;
214	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
215	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
216	virtioCoreGCPhysRead(pVirtio, pDevIns,
217	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]),
218	&uUsedEventIdx, sizeof(uUsedEventIdx));
219	return uUsedEventIdx;
220	}
221	#endif
222
223	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
224	{
225	uint16_t uIdx = 0;
226	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
227	virtioCoreGCPhysRead(pVirtio, pDevIns,
228	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
229	&uIdx, sizeof(uIdx));
230	return uIdx;
231	}
232
233	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
234	{
235	uint16_t fFlags = 0;
236	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
237	virtioCoreGCPhysRead(pVirtio, pDevIns,
238	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
239	&fFlags, sizeof(fFlags));
240	return fFlags;
241	}
242
243	/** @} */
244
245	/** @name Accessors for virtq used ring
246	* @{
247	*/
248
249	#ifdef IN_RING3
250	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
251	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
252	{
253	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
254	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
255	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
256	virtioCoreGCPhysWrite(pVirtio, pDevIns,
257	pVirtq->GCPhysVirtqUsed
258	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cVirtqItems]),
259	&elem, sizeof(elem));
260	}
261
262	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t fFlags)
263	{
264	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
265	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
266	virtioCoreGCPhysWrite(pVirtio, pDevIns,
267	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
268	&fFlags, sizeof(fFlags));
269	}
270	#endif
271
272	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t uIdx)
273	{
274	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
275	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
276	virtioCoreGCPhysWrite(pVirtio, pDevIns,
277	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
278	&uIdx, sizeof(uIdx));
279	}
280
281	#ifdef IN_RING3
282	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
283	{
284	uint16_t uIdx = 0;
285	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
286	virtioCoreGCPhysRead(pVirtio, pDevIns,
287	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
288	&uIdx, sizeof(uIdx));
289	return uIdx;
290	}
291
292	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
293	{
294	uint16_t fFlags = 0;
295	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
296	virtioCoreGCPhysRead(pVirtio, pDevIns,
297	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
298	&fFlags, sizeof(fFlags));
299	return fFlags;
300	}
301
302	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t uAvailEventIdx)
303	{
304	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
305	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
306	virtioCoreGCPhysWrite(pVirtio, pDevIns,
307	pVirtq->GCPhysVirtqUsed
308	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
309	&uAvailEventIdx, sizeof(uAvailEventIdx));
310	}
311	#endif
312	/** @} */
313
314
315	DECLINLINE(uint16_t) virtioCoreVirtqAvailCnt(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
316	{
317	uint16_t uIdxActual = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
318	uint16_t uIdxShadow = pVirtq->uAvailIdxShadow;
319	uint16_t uIdxDelta;
320
321	if (uIdxActual < uIdxShadow)
322	uIdxDelta = (uIdxActual + pVirtq->uQueueSize) - uIdxShadow;
323	else
324	uIdxDelta = uIdxActual - uIdxShadow;
325
326	return uIdxDelta;
327	}
328	/**
329	* Get count of new (e.g. pending) elements in available ring.
330	*
331	* @param pDevIns The device instance.
332	* @param pVirtio Pointer to the shared virtio state.
333	* @param uVirtq Virtq number
334	*
335	* @returns how many entries have been added to ring as a delta of the consumer's
336	* avail index and the queue's guest-side current avail index.
337	*/
338	DECLHIDDEN(uint16_t) virtioCoreVirtqAvailBufCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
339	{
340	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues), ("uVirtq out of range"), 0);
341	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
342
343	if (!IS_DRIVER_OK(pVirtio))
344	{
345	LogRelFunc(("Driver not ready\n"));
346	return 0;
347	}
348	if (!pVirtio->fLegacyDriver && !pVirtq->uEnable)
349	{
350	LogRelFunc(("virtq: %s not enabled\n", VIRTQNAME(pVirtio, uVirtq)));
351	return 0;
352	}
353	return virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq);
354	}
355
356	#ifdef IN_RING3
357
358	static void virtioCoreR3FeatureDump(VIRTIOCORE pVirtio, PCDBGFINFOHLP pHlp, const VIRTIO_FEATURES_LIST s_aFeatures, int cFeatures, int fBanner)
359	{
360	#define MAXLINE 80
361	/* Display as a single buf to prevent interceding log messages */
362	uint16_t cbBuf = cFeatures * 132;
363	char pszBuf = (char )RTMemAllocZ(cbBuf);
364	Assert(pszBuf);
365	char *cp = pszBuf;
366	for (int i = 0; i < cFeatures; ++i)
367	{
368	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
369	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
370	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
371	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
372	}
373	if (pHlp) {
374	if (fBanner)
375	pHlp->pfnPrintf(pHlp, "VirtIO Features Configuration\n\n"
376	" Offered Accepted Feature Description\n"
377	" ------- -------- ------- -----------\n");
378	pHlp->pfnPrintf(pHlp, "%s\n", pszBuf);
379	}
380	#ifdef LOG_ENABLED
381	else
382	{
383	if (fBanner)
384	Log(("VirtIO Features Configuration\n\n"
385	" Offered Accepted Feature Description\n"
386	" ------- -------- ------- -----------\n"));
387	Log(("%s\n", pszBuf));
388	}
389	#endif
390	RTMemFree(pszBuf);
391	}
392
393	/** API Function: See header file*/
394	DECLHIDDEN(void) virtioCorePrintDeviceFeatures(VIRTIOCORE *pVirtio, PCDBGFINFOHLP pHlp,
395	const VIRTIO_FEATURES_LIST *s_aDevSpecificFeatures, int cFeatures) {
396	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aCoreFeatures, RT_ELEMENTS(s_aCoreFeatures), 1 /fBanner /);
397	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aDevSpecificFeatures, cFeatures, 0 /fBanner /);
398	}
399
400	#endif
401
402	#ifdef LOG_ENABLED
403
404	/** API Function: See header file */
405	DECLHIDDEN(void) virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
406	{
407	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
408	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
409	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
410	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
411	if (pszTitle)
412	{
413	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
414	ADJCURSOR(cbPrint);
415	}
416	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
417	{
418	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
419	ADJCURSOR(cbPrint);
420	for (uint8_t col = 0; col < 16; col++)
421	{
422	uint32_t idx = row * 16 + col;
423	if (idx >= cb)
424	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
425	else
426	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", pv[idx], (col + 1) % 8 ? "" : " ");
427	ADJCURSOR(cbPrint);
428	}
429	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
430	{
431	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.'));
432	ADJCURSOR(cbPrint);
433	}
434	*pszOut++ = '\n';
435	--cbRemain;
436	}
437	Log(("%s\n", pszBuf));
438	RTMemFree(pszBuf);
439	RT_NOREF2(uBase, pv);
440	#undef ADJCURSOR
441	}
442
443	/* API FUnction: See header file */
444	DECLHIDDEN(void) virtioCoreGCPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint16_t cb, uint32_t uBase, const char *pszTitle)
445	{
446	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
447	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
448	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
449	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
450	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
451	if (pszTitle)
452	{
453	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
454	ADJCURSOR(cbPrint);
455	}
456	for (uint16_t row = 0; row < (uint16_t)RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
457	{
458	uint8_t c;
459	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
460	ADJCURSOR(cbPrint);
461	for (uint8_t col = 0; col < 16; col++)
462	{
463	uint32_t idx = row * 16 + col;
464	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
465	if (idx >= cb)
466	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
467	else
468	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", c, (col + 1) % 8 ? "" : " ");
469	ADJCURSOR(cbPrint);
470	}
471	for (uint16_t idx = row * 16; idx < row * 16 + 16; idx++)
472	{
473	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
474	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.'));
475	ADJCURSOR(cbPrint);
476	}
477	*pszOut++ = '\n';
478	--cbRemain;
479	}
480	Log(("%s\n", pszBuf));
481	RTMemFree(pszBuf);
482	RT_NOREF(uBase);
483	#undef ADJCURSOR
484	}
485
486
487	/** API function: See header file */
488	DECLHIDDEN(void) virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
489	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
490	int fHasIndex, uint32_t idx)
491	{
492	if (LogIs6Enabled())
493	{
494	char szIdx[16];
495	if (fHasIndex)
496	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
497	else
498	szIdx[0] = '\0';
499
500	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
501	{
502	char szDepiction[64];
503	size_t cchDepiction;
504	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
505	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
506	pszMember, szIdx, uOffset, uOffset + cb - 1);
507	else
508	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
509
510	/* padding */
511	if (cchDepiction < 30)
512	szDepiction[cchDepiction++] = ' ';
513	while (cchDepiction < 30)
514	szDepiction[cchDepiction++] = '.';
515	szDepiction[cchDepiction] = '\0';
516
517	RTUINT64U uValue;
518	uValue.u = 0;
519	memcpy(uValue.au8, pv, cb);
520	Log6(("%-23s: Guest %s %s %#0*RX64\n",
521	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
522	}
523	else /* odd number or oversized access, ... log inline hex-dump style */
524	{
525	Log6(("%-23s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
526	pszFunc, fWrite ? "wrote" : "read ", pszMember,
527	szIdx, uOffset, uOffset + cb, cb, pv));
528	}
529	}
530	RT_NOREF2(fWrite, pszFunc);
531	}
532
533	/**
534	* Log MMIO-mapped Virtio fDeviceStatus register bitmask, naming the bits
535	*/
536	DECLINLINE(void) virtioCoreFormatDeviceStatus(uint8_t bStatus, char *pszBuf, size_t uSize)
537	{
538	# define ADJCURSOR(len) { cp += len; uSize -= len; sep = (char *)" \| "; }
539	memset(pszBuf, 0, uSize);
540	char cp = pszBuf, sep = (char *)"";
541	size_t len;
542	if (bStatus == 0)
543	RTStrPrintf(cp, uSize, "RESET");
544	else
545	{
546	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
547	{
548	len = RTStrPrintf(cp, uSize, "ACKNOWLEDGE");
549	ADJCURSOR(len);
550	}
551	if (bStatus & VIRTIO_STATUS_DRIVER)
552	{
553	len = RTStrPrintf(cp, uSize, "%sDRIVER", sep);
554	ADJCURSOR(len);
555	}
556	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
557	{
558	len = RTStrPrintf(cp, uSize, "%sFEATURES_OK", sep);
559	ADJCURSOR(len);
560	}
561	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
562	{
563	len = RTStrPrintf(cp, uSize, "%sDRIVER_OK", sep);
564	ADJCURSOR(len);
565	}
566	if (bStatus & VIRTIO_STATUS_FAILED)
567	{
568	len = RTStrPrintf(cp, uSize, "%sFAILED", sep);
569	ADJCURSOR(len);
570	}
571	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
572	RTStrPrintf(cp, uSize, "%sNEEDS_RESET", sep);
573	}
574	# undef ADJCURSOR
575	}
576
577	#endif /* LOG_ENABLED */
578
579	/** API function: See header file */
580	DECLHIDDEN(int) virtioCoreIsLegacyMode(PVIRTIOCORE pVirtio)
581	{
582	return pVirtio->fLegacyDriver;
583	}
584
585	#ifdef IN_RING3
586
587	DECLHIDDEN(int) virtioCoreR3VirtqAttach(PVIRTIOCORE pVirtio, uint16_t uVirtq, const char *pcszName)
588	{
589	LogFunc(("Attaching %s to VirtIO core\n", pcszName));
590	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
591	pVirtq->uVirtq = uVirtq;
592	pVirtq->fUsedRingEvent = false;
593	pVirtq->fAttached = true;
594	RTStrCopy(pVirtq->szName, sizeof(pVirtq->szName), pcszName);
595	return VINF_SUCCESS;
596	}
597
598	DECLHIDDEN(int) virtioCoreR3VirtqDetach(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
599	{
600	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
601	pVirtq->uVirtq = 0;
602	pVirtq->uAvailIdxShadow = 0;
603	pVirtq->uUsedIdxShadow = 0;
604	pVirtq->fUsedRingEvent = false;
605	pVirtq->fAttached = false;
606	memset(pVirtq->szName, 0, sizeof(pVirtq->szName));
607	return VINF_SUCCESS;
608	}
609
610	DECLHIDDEN(bool) virtioCoreR3VirtqIsAttached(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
611	{
612	return pVirtio->aVirtqueues[uVirtqNbr].fAttached;
613	}
614
615	DECLHIDDEN(bool) virtioCoreR3VirtqIsEnabled(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
616	{
617	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
618	return (bool)pVirtq->uEnable && pVirtq->GCPhysVirtqDesc;
619	}
620
621	DECLINLINE(void) virtioCoreR3DescInfo(PCDBGFINFOHLP pHlp, PVIRTQ_DESC_T pDesc, uint16_t iDesc, const char *cszTail)
622	{
623	if (pDesc->fFlags & VIRTQ_DESC_F_NEXT)
624	pHlp->pfnPrintf(pHlp, " [%4d]%c%c %5d bytes @ %p [%4d] %s\n",
625	iDesc, pDesc->fFlags & VIRTQ_DESC_F_INDIRECT ? 'I' : ' ',
626	pDesc->fFlags & VIRTQ_DESC_F_WRITE ? 'W' : 'R',
627	pDesc->cb, pDesc->GCPhysBuf, pDesc->uDescIdxNext, cszTail);
628	else
629	pHlp->pfnPrintf(pHlp, " [%4d]%c%c %5d bytes @ %p %s\n",
630	iDesc, pDesc->fFlags & VIRTQ_DESC_F_INDIRECT ? 'I' : ' ',
631	pDesc->fFlags & VIRTQ_DESC_F_WRITE ? 'W' : 'R',
632	pDesc->cb, pDesc->GCPhysBuf, cszTail);
633	}
634
635	/** API Fuunction: See header file */
636	DECLHIDDEN(void) virtioCoreR3VirtqInfo(PPDMDEVINS pDevIns, PCDBGFINFOHLP pHlp, const char *pszArgs, int uVirtq)
637	{
638	RT_NOREF(pszArgs);
639	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
640	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
641
642	/** @todo add ability to dump physical contents described by any descriptor (using existing VirtIO core API function) */
643	// bool fDump = pszArgs && (pszArgs == 'd' \|\| pszArgs == 'D'); /* "dump" (avail phys descriptor)"
644
645	uint16_t uAvailIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
646	uint16_t uAvailIdxShadow = pVirtq->uAvailIdxShadow;
647
648	uint16_t uUsedIdx = virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq);
649	uint16_t uUsedIdxShadow = pVirtq->uUsedIdxShadow;
650
651	uint16_t uAvailEventIdx = 0;
652	uint16_t uUsedEventIdx = 0;
653	bool fNotify = !!(pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX);
654	if (fNotify)
655	{
656	uUsedEventIdx = virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq);
657	/* There is no helper for reading AvailEvent since the device is not supposed to read it. */
658	virtioCoreGCPhysRead(pVirtio, pDevIns,
659	pVirtq->GCPhysVirtqUsed
660	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
661	&uAvailEventIdx, sizeof(uAvailEventIdx));
662	}
663
664	VIRTQBUF_T VirtqBuf;
665	PVIRTQBUF pVirtqBuf = &VirtqBuf;
666	RT_ZERO(VirtqBuf); /* Make sure pSgPhysSend and pSgPhysReturn are initialized. */
667	bool fEmpty = IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq);
668
669	LogFunc(("%s, empty = %s\n", pVirtq->szName, fEmpty ? "true" : "false"));
670
671	int cSendSegs = 0, cReturnSegs = 0;
672	if (!fEmpty)
673	{
674	virtioCoreR3VirtqAvailBufPeek(pDevIns, pVirtio, uVirtq, pVirtqBuf);
675	cSendSegs = pVirtqBuf->pSgPhysSend ? pVirtqBuf->pSgPhysSend->cSegs : 0;
676	cReturnSegs = pVirtqBuf->pSgPhysReturn ? pVirtqBuf->pSgPhysReturn->cSegs : 0;
677	}
678
679	bool fAvailNoInterrupt = virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT;
680	bool fUsedNoNotify = virtioReadUsedRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_USED_F_NO_NOTIFY;
681
682	pHlp->pfnPrintf(pHlp, " queue enabled: ........... %s\n", pVirtq->uEnable ? "true" : "false");
683	pHlp->pfnPrintf(pHlp, " size: .................... %d\n", pVirtq->uQueueSize);
684	pHlp->pfnPrintf(pHlp, " notify offset: ........... %d\n", pVirtq->uNotifyOffset);
685	if (pVirtio->fMsiSupport)
686	pHlp->pfnPrintf(pHlp, " MSIX vector: ....... %4.4x\n", pVirtq->uMsixVector);
687	pHlp->pfnPrintf(pHlp, "\n");
688	pHlp->pfnPrintf(pHlp, " avail ring (%d entries):\n", uAvailIdx - uAvailIdxShadow);
689	pHlp->pfnPrintf(pHlp, " index: ................ %d (%d)\n", pVirtq->uQueueSize ? uAvailIdx % pVirtq->uQueueSize : uAvailIdx, uAvailIdx);
690	pHlp->pfnPrintf(pHlp, " shadow: ............... %d (%d)\n", pVirtq->uQueueSize ? uAvailIdxShadow % pVirtq->uQueueSize : uAvailIdxShadow, uAvailIdxShadow);
691	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fAvailNoInterrupt ? "NO_INTERRUPT" : "");
692	pHlp->pfnPrintf(pHlp, "\n");
693	pHlp->pfnPrintf(pHlp, " used ring (%d entries):\n", uUsedIdxShadow - uUsedIdx);
694	pHlp->pfnPrintf(pHlp, " index: ................ %d (%d)\n", pVirtq->uQueueSize ? uUsedIdx % pVirtq->uQueueSize : uUsedIdx, uUsedIdx);
695	pHlp->pfnPrintf(pHlp, " shadow: ............... %d (%d)\n", pVirtq->uQueueSize ? uUsedIdxShadow % pVirtq->uQueueSize : uUsedIdxShadow, uUsedIdxShadow);
696	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fUsedNoNotify ? "NO_NOTIFY" : "");
697	pHlp->pfnPrintf(pHlp, "\n");
698	if (!fEmpty)
699	{
700	pHlp->pfnPrintf(pHlp, " desc chain:\n");
701	pHlp->pfnPrintf(pHlp, " head idx: ............. %d (%d)\n", pVirtq->uQueueSize ? uUsedIdx % pVirtq->uQueueSize : uUsedIdx, uUsedIdx);
702	pHlp->pfnPrintf(pHlp, " segs: ................. %d\n", cSendSegs + cReturnSegs);
703	pHlp->pfnPrintf(pHlp, " refCnt ................ %d\n", pVirtqBuf->cRefs);
704	pHlp->pfnPrintf(pHlp, "\n");
705	pHlp->pfnPrintf(pHlp, " host-to-guest (%d bytes):\n", pVirtqBuf->cbPhysSend);
706	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cSendSegs);
707	if (cSendSegs)
708	{
709	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysSend->idxSeg);
710	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysSend->cbSegLeft);
711	}
712	pHlp->pfnPrintf(pHlp, "\n");
713	pHlp->pfnPrintf(pHlp, " guest-to-host (%d bytes):\n", pVirtqBuf->cbPhysReturn);
714	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cReturnSegs);
715	if (cReturnSegs)
716	{
717	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysReturn->idxSeg);
718	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysReturn->cbSegLeft);
719	}
720	} else
721	pHlp->pfnPrintf(pHlp, " no desc chains available\n");
722	pHlp->pfnPrintf(pHlp, "\n");
723
724	/* Avoid handling zero-sized queues, there is nothing to show anyway. */
725	if (pVirtq->uQueueSize == 0)
726	return;
727
728	pHlp->pfnPrintf(pHlp, " desc table:\n");
729	/*
730	* Each line in the descriptor table output consists of two parts: a fixed part and a variable "tail".
731	* The fixed part shows the descriptor index, its writability, size, physical address, and optionally
732	* which descriptor is next the chain. The tail shows which elements of avail/used rings point to
733	* this descriptor.
734	*/
735	VIRTQ_DESC_T descTable[VIRTQ_SIZE];
736	char aszTails[VIRTQ_SIZE][32];
737	virtioCoreGCPhysRead(pVirtio, pDevIns, pVirtq->GCPhysVirtqDesc,
738	&descTable, sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize);
739	RT_BZERO(aszTails, sizeof(aszTails)); /* No tails by default */
740
741	/* Fill avail tail fields. */
742
743	/* The first available descriptor gets outer reverse angle brackets. */
744	char chOuterLeft = '>', chOuterRight = '<';
745	char chLeft = '[', chRight = ']';
746	/* Use 'not-equal' instead of 'less' because of uint16_t wrapping! */
747	for (uint16_t i = uAvailIdxShadow; i != uAvailIdx; i++)
748	{
749	/* The last descriptor gets inner curly braces, inner square brackets for the rest. */
750	if (i + 1 == uAvailIdx) { chLeft = '{'; chRight = '}'; }
751	uint16_t uDescIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, i);
752	/* Print an exclamation sign instead of outer right bracket if this descriptor triggers notification. */
753	RTStrPrintf(aszTails[uDescIdx], sizeof(aszTails[0]), "%c%c%4d%c%c ",
754	chOuterLeft, chLeft, i % pVirtq->uQueueSize, chRight,
755	fNotify ? ((i % pVirtq->uQueueSize) == (uAvailEventIdx % pVirtq->uQueueSize) ? '!' : chOuterRight) : chOuterRight);
756	chOuterLeft = chOuterRight = ' ';
757	}
758
759	/* Fill used tail fields, see comments in the similar loop above. */
760
761	chOuterLeft = '>'; chOuterRight = '<';
762	chLeft = '['; chRight = ']';
763	for (uint16_t i = uUsedIdx; i != uUsedIdxShadow; i++)
764	{
765	VIRTQ_USED_ELEM_T elem;
766	virtioCoreGCPhysRead(pVirtio, pDevIns,
767	pVirtq->GCPhysVirtqUsed
768	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[i % pVirtq->uQueueSize]),
769	&elem, sizeof(elem));
770	if (i + 1 == uUsedIdxShadow) { chLeft = '{'; chRight = '}'; }
771	char *szTail = aszTails[elem.uDescIdx % pVirtq->uQueueSize];
772	/* Add empty avail field if none is present, 9 spaces + terminating zero. */
773	if (*szTail == '\0')
774	RTStrCopy(szTail, 10, " ");
775	RTStrPrintf(szTail + 9, sizeof(aszTails[0]) - 9, " %c%c%4d%c%c %d bytes",
776	chOuterLeft, chLeft, i % pVirtq->uQueueSize, chRight,
777	fNotify ? ((i % pVirtq->uQueueSize) == (uUsedEventIdx % pVirtq->uQueueSize) ? '!' : chOuterRight) : chOuterRight,
778	elem.cbElem);
779	chOuterLeft = chOuterRight = ' ';
780	}
781
782	pHlp->pfnPrintf(pHlp, " index w/r size phys addr next @avail @used\n");
783	pHlp->pfnPrintf(pHlp, " ------ - ----------- ---------------- ------- -------- ------------------\n");
784	for (uint16_t i = 0; i < pVirtq->uQueueSize; i++)
785	virtioCoreR3DescInfo(pHlp, &descTable[i], i, aszTails[i]);
786	}
787
788
789	/** API Function: See header file */
790	DECLHIDDEN(PVIRTQBUF) virtioCoreR3VirtqBufAlloc(void)
791	{
792	PVIRTQBUF pVirtqBuf = (PVIRTQBUF)RTMemAllocZ(sizeof(VIRTQBUF_T));
793	AssertReturn(pVirtqBuf, NULL);
794	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
795	pVirtqBuf->cRefs = 1;
796	return pVirtqBuf;
797	}
798
799
800	/** API Function: See header file */
801	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRetain(PVIRTQBUF pVirtqBuf)
802	{
803	AssertReturn(pVirtqBuf, UINT32_MAX);
804	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, UINT32_MAX);
805	uint32_t cRefs = ASMAtomicIncU32(&pVirtqBuf->cRefs);
806	Assert(cRefs > 1);
807	Assert(cRefs < 16);
808	return cRefs;
809	}
810
811	/** API Function: See header file */
812	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRelease(PVIRTIOCORE pVirtio, PVIRTQBUF pVirtqBuf)
813	{
814	if (!pVirtqBuf)
815	return 0;
816	AssertReturn(pVirtqBuf, 0);
817	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, 0);
818	uint32_t cRefs = ASMAtomicDecU32(&pVirtqBuf->cRefs);
819	Assert(cRefs < 16);
820	if (cRefs == 0)
821	{
822	pVirtqBuf->u32Magic = ~VIRTQBUF_MAGIC;
823	RTMemFree(pVirtqBuf);
824	#ifdef VBOX_WITH_STATISTICS
825	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
826	#endif
827	}
828	RT_NOREF(pVirtio);
829	return cRefs;
830	}
831
832	/** API Function: See header file */
833	DECLHIDDEN(void) virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
834	{
835	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
836	}
837
838
839	/** API Function: See header file */
840	DECLHIDDEN(void) virtioCoreVirtqEnableNotify(PVIRTIOCORE pVirtio, uint16_t uVirtq, bool fEnable)
841	{
842	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
843	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
844
845	if (IS_DRIVER_OK(pVirtio))
846	{
847	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq);
848
849	if (fEnable)
850	fFlags &= ~VIRTQ_USED_F_NO_NOTIFY;
851	else
852	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
853
854	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq, fFlags);
855	}
856	}
857
858	/** API function: See Header file */
859	DECLHIDDEN(void) virtioCoreResetAll(PVIRTIOCORE pVirtio)
860	{
861	LogFunc(("\n"));
862	pVirtio->fDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
863	if (IS_DRIVER_OK(pVirtio))
864	{
865	if (!pVirtio->fLegacyDriver)
866	pVirtio->fGenUpdatePending = true;
867	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
868	}
869	}
870
871	/** API function: See Header file */
872	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufPeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PVIRTQBUF pVirtqBuf)
873	{
874	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, pVirtqBuf, false);
875	}
876
877
878	/** API function: See Header file */
879	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufNext(PVIRTIOCORE pVirtio, uint16_t uVirtq)
880	{
881	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
882	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
883
884	if (!pVirtio->fLegacyDriver)
885	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
886	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
887
888	if (IS_VIRTQ_EMPTY(pVirtio->pDevInsR3, pVirtio, pVirtq))
889	return VERR_NOT_AVAILABLE;
890
891	Log6Func(("%s avail shadow idx: %u\n", pVirtq->szName, pVirtq->uAvailIdxShadow));
892	pVirtq->uAvailIdxShadow++;
893
894	return VINF_SUCCESS;
895	}
896
897	/** API Function: See header file */
898	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
899	uint16_t uHeadIdx, PVIRTQBUF pVirtqBuf)
900	{
901	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues),
902	("uVirtq out of range"), VERR_INVALID_PARAMETER);
903
904	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
905
906	if (!pVirtio->fLegacyDriver)
907	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
908	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
909
910	uint16_t uDescIdx = uHeadIdx;
911
912	Log6Func(("%s DESC CHAIN: (head idx = %u)\n", pVirtio->aVirtqueues[uVirtq].szName, uHeadIdx));
913
914	/*
915	* Allocate and initialize the descriptor chain structure.
916	*/
917	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
918	pVirtqBuf->cRefs = 1;
919	pVirtqBuf->uHeadIdx = uHeadIdx;
920	pVirtqBuf->uVirtq = uVirtq;
921
922	/*
923	* Gather segments.
924	*/
925	VIRTQ_DESC_T desc;
926
927	uint32_t cbIn = 0;
928	uint32_t cbOut = 0;
929	uint32_t cSegsIn = 0;
930	uint32_t cSegsOut = 0;
931
932	PVIRTIOSGSEG paSegsIn = pVirtqBuf->aSegsIn;
933	PVIRTIOSGSEG paSegsOut = pVirtqBuf->aSegsOut;
934
935	do
936	{
937	PVIRTIOSGSEG pSeg;
938	/*
939	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
940	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
941	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
942	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
943	*/
944	if (cSegsIn + cSegsOut >= pVirtq->uQueueSize)
945	{
946	static volatile uint32_t s_cMessages = 0;
947	static volatile uint32_t s_cThreshold = 1;
948	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
949	{
950	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop (cSegsIn=%u cSegsOut=%u uQueueSize=%u uDescIdx=%u queue=%s).\n",
951	cSegsIn, cSegsOut, pVirtq->uQueueSize, uDescIdx, pVirtq->szName));
952	if (ASMAtomicReadU32(&s_cMessages) != 1)
953	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
954	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
955	}
956	break;
957	}
958	/* Check if the limit has been reached for input chain (see section 2.4.4.1 of virtio 1.0 spec). */
959	if (cSegsIn >= RT_ELEMENTS(pVirtqBuf->aSegsIn))
960	{
961	LogRelMax(64, ("Too many input descriptors (cSegsIn=%u).\n", cSegsIn));
962	break;
963	}
964	/* Check if the limit has been reached for output chain (see section 2.4.4.1 of virtio 1.0 spec). */
965	if (cSegsOut >= RT_ELEMENTS(pVirtqBuf->aSegsOut))
966	{
967	LogRelMax(64, ("Too many output descriptors (cSegsOut=%u).\n", cSegsOut));
968	break;
969	}
970	RT_UNTRUSTED_VALIDATED_FENCE();
971
972	virtioReadDesc(pDevIns, pVirtio, pVirtq, uDescIdx, &desc);
973
974	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
975	{
976	Log6Func(("%s IN idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
977	cbIn += desc.cb;
978	pSeg = &paSegsIn[cSegsIn++];
979	}
980	else
981	{
982	Log6Func(("%s OUT desc_idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
983	cbOut += desc.cb;
984	pSeg = &paSegsOut[cSegsOut++];
985	#ifdef DEEP_DEBUG
986	if (LogIs11Enabled())
987	{
988	virtioCoreGCPhysHexDump(pDevIns, desc.GCPhysBuf, desc.cb, 0, NULL);
989	Log(("\n"));
990	}
991	#endif
992	}
993	pSeg->GCPhys = desc.GCPhysBuf;
994	pSeg->cbSeg = desc.cb;
995	uDescIdx = desc.uDescIdxNext;
996	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
997
998	/*
999	* Add segments to the descriptor chain structure.
1000	*/
1001	if (cSegsIn)
1002	{
1003	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufIn, paSegsIn, cSegsIn);
1004	pVirtqBuf->pSgPhysReturn = &pVirtqBuf->SgBufIn;
1005	pVirtqBuf->cbPhysReturn = cbIn;
1006	#ifdef VBOX_WITH_STATISTICS
1007	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
1008	#endif
1009	}
1010
1011	if (cSegsOut)
1012	{
1013	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufOut, paSegsOut, cSegsOut);
1014	pVirtqBuf->pSgPhysSend = &pVirtqBuf->SgBufOut;
1015	pVirtqBuf->cbPhysSend = cbOut;
1016	#ifdef VBOX_WITH_STATISTICS
1017	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
1018	#endif
1019	}
1020
1021	#ifdef VBOX_WITH_STATISTICS
1022	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
1023	#endif
1024	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n",
1025	pVirtq->szName, cSegsOut, cbOut, cSegsIn, cbIn));
1026
1027	return VINF_SUCCESS;
1028	}
1029
1030	/** API function: See Header file */
1031	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1032	PVIRTQBUF pVirtqBuf, bool fRemove)
1033	{
1034	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1035	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1036
1037	if (IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq))
1038	return VERR_NOT_AVAILABLE;
1039
1040	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, pVirtq->uAvailIdxShadow);
1041
1042	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1043	virtioWriteUsedAvailEvent(pDevIns,pVirtio, pVirtq, pVirtq->uAvailIdxShadow + 1);
1044
1045	if (fRemove)
1046	pVirtq->uAvailIdxShadow++;
1047
1048	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, uHeadIdx, pVirtqBuf);
1049	}
1050
1051	/** API function: See Header file */
1052	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PRTSGBUF pSgVirtReturn,
1053	PVIRTQBUF pVirtqBuf, bool fFence)
1054	{
1055	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1056	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1057
1058	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1059
1060	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1061	Assert(pVirtqBuf->cRefs > 0);
1062
1063	/*
1064	* Workaround for a bug in FreeBSD's virtio-net driver up until 12.3 which supports only the legacy style devive.
1065	* When the device is re-initialized from the driver it violates the spec and posts commands to the control queue
1066	* before setting the DRIVER_OK flag, breaking the following check and rendering the device non-functional.
1067	* The queues are properly set up at this stage however so no real harm is done and we can safely continue here,
1068	* for the legacy device only of course after making sure the queue is properly set up.
1069	*/
1070	AssertMsgReturn( IS_DRIVER_OK(pVirtio)
1071	\|\| ( pVirtio->fLegacyDriver
1072	&& pVirtq->GCPhysVirtqDesc),
1073	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1074
1075	Log6Func((" Copying device data to %s, [desc:%u -> used ring:%u]\n",
1076	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx, pVirtq->uUsedIdxShadow));
1077
1078	/* Copy s/g buf (virtual memory) to guest phys mem (VirtIO "IN" direction). */
1079
1080	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
1081
1082	if (pSgVirtReturn)
1083	{
1084	size_t cbTarget = virtioCoreGCPhysChainCalcBufSize(pSgPhysReturn);
1085	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
1086	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
1087	virtioCoreGCPhysChainReset(pSgPhysReturn);
1088	while (cbRemain)
1089	{
1090	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
1091	AssertReturn(cbCopy > 0, VERR_INVALID_PARAMETER);
1092	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
1093	RTSgBufAdvance(pSgVirtReturn, cbCopy);
1094	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1095	cbRemain -= cbCopy;
1096	}
1097
1098	if (fFence)
1099	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1100
1101	Assert(!(cbCopy >> 32));
1102	}
1103
1104	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1105	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1106	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1107	pVirtq->fUsedRingEvent = true;
1108
1109	/*
1110	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1111	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1112	*
1113	* @todo r=aeichner: The increment of the shadow index is not atomic but this code can be called
1114	* concurrently!!
1115	*/
1116	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbTotal);
1117
1118	#ifdef LOG_ENABLED
1119	if (LogIs6Enabled() && pSgVirtReturn)
1120	{
1121
1122	LogFunc((" ... %d segs, %zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1123	pSgVirtReturn->cSegs, cbTotal - cbRemain, pVirtqBuf->cbPhysReturn,
1124	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1125	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - (cbTotal - cbRemain)),
1126	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn) ));
1127
1128	uint16_t uPending = virtioCoreR3CountPendingBufs(
1129	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1130	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1131
1132	LogFunc((" %u used buf%s not synced in %s\n", uPending, uPending == 1 ? "" : "s ",
1133	VIRTQNAME(pVirtio, uVirtq)));
1134	}
1135	#endif
1136	return VINF_SUCCESS;
1137	}
1138
1139	/** API function: See Header file */
1140	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1141	size_t cb, void const *pv, PVIRTQBUF pVirtqBuf, size_t cbEnqueue, bool fFence)
1142	{
1143	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1144	Assert(pv);
1145
1146	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1147	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1148
1149	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1150	Assert(pVirtqBuf->cRefs > 0);
1151
1152	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1153
1154	Log6Func((" Copying device data to %s, [desc chain head idx:%u]\n",
1155	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx));
1156	/*
1157	* Convert virtual memory simple buffer to guest physical memory (VirtIO descriptor chain)
1158	*/
1159	uint8_t pvBuf = (uint8_t )pv;
1160	size_t cbRemain = cb, cbCopy = 0;
1161	while (cbRemain)
1162	{
1163	cbCopy = RT_MIN(pSgPhysReturn->cbSegLeft, cbRemain);
1164	Assert(cbCopy > 0);
1165	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pvBuf, cbCopy);
1166	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1167	pvBuf += cbCopy;
1168	cbRemain -= cbCopy;
1169	}
1170	LogFunc((" ...%zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1171	cb , pVirtqBuf->cbPhysReturn,
1172	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1173	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - cb),
1174	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)));
1175
1176	if (cbEnqueue)
1177	{
1178	if (fFence)
1179	{
1180	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1181	Assert(!(cbCopy >> 32));
1182	}
1183	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1184	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1185	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1186	pVirtq->fUsedRingEvent = true;
1187	/*
1188	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1189	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1190	*/
1191	Log6Func((" Enqueue desc chain head idx %u to %s used ring @ %u\n", pVirtqBuf->uHeadIdx,
1192	VIRTQNAME(pVirtio, uVirtq), pVirtq->uUsedIdxShadow));
1193
1194	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbEnqueue);
1195
1196	#ifdef LOG_ENABLED
1197	if (LogIs6Enabled())
1198	{
1199	uint16_t uPending = virtioCoreR3CountPendingBufs(
1200	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1201	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1202
1203	LogFunc((" %u used buf%s not synced in %s\n",
1204	uPending, uPending == 1 ? "" : "s ", VIRTQNAME(pVirtio, uVirtq)));
1205	}
1206	#endif
1207	} /* fEnqueue */
1208
1209	return VINF_SUCCESS;
1210	}
1211
1212
1213	#endif /* IN_RING3 */
1214
1215	/** API function: See Header file */
1216	DECLHIDDEN(int) virtioCoreVirtqUsedRingSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1217	{
1218	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1219	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1220
1221	if (!pVirtio->fLegacyDriver)
1222	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
1223	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1224
1225	Log6Func((" Sync %s used ring (%u -> idx)\n",
1226	pVirtq->szName, pVirtq->uUsedIdxShadow));
1227
1228	virtioWriteUsedRingIdx(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow);
1229	virtioCoreNotifyGuestDriver(pDevIns, pVirtio, uVirtq);
1230
1231	return VINF_SUCCESS;
1232	}
1233
1234	/**
1235	* This is called from the MMIO callback code when the guest does an MMIO access to the
1236	* mapped queue notification capability area corresponding to a particular queue, to notify
1237	* the queue handler of available data in the avail ring of the queue (VirtIO 1.0, 4.1.4.4.1)
1238	*
1239	* @param pDevIns The device instance.
1240	* @param pVirtio Pointer to the shared virtio state.
1241	* @param uVirtq Virtq to check for guest interrupt handling preference
1242	* @param uNotifyIdx Notification index
1243	*/
1244	static void virtioCoreVirtqNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, uint16_t uNotifyIdx)
1245	{
1246	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1247
1248	/* VirtIO 1.0, section 4.1.5.2 implies uVirtq and uNotifyIdx should match. Disregarding any of
1249	* these notifications (if those indicies disagree) may break device/driver synchronization,
1250	* causing eternal throughput starvation, yet there's no specified way to disambiguate
1251	* which queue to wake-up in any awkward situation where the two parameters differ.
1252	*/
1253	AssertMsg(uNotifyIdx == uVirtq,
1254	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1255	uVirtq, uNotifyIdx));
1256	RT_NOREF(uNotifyIdx);
1257
1258	AssertReturnVoid(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1259	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1260
1261	Log6Func(("%s: (desc chains: %u)\n", *pVirtq->szName ? pVirtq->szName : "?UNAMED QUEUE?",
1262	virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq)));
1263
1264	/* Inform client */
1265	pVirtioCC->pfnVirtqNotified(pDevIns, pVirtio, uVirtq);
1266	RT_NOREF2(pVirtio, pVirtq);
1267	}
1268
1269	/**
1270	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1271	* the specified virtq, depending on the interrupt configuration of the device
1272	* and depending on negotiated and realtime constraints flagged by the guest driver.
1273	*
1274	* See VirtIO 1.0 specification (section 2.4.7).
1275	*
1276	* @param pDevIns The device instance.
1277	* @param pVirtio Pointer to the shared virtio state.
1278	* @param uVirtq Virtq to check for guest interrupt handling preference
1279	*/
1280	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1281	{
1282	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1283	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1284
1285	if (!IS_DRIVER_OK(pVirtio))
1286	{
1287	LogFunc(("Guest driver not in ready state.\n"));
1288	return;
1289	}
1290
1291	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1292	{
1293	if (pVirtq->fUsedRingEvent)
1294	{
1295	#ifdef IN_RING3
1296	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1297	pVirtq->szName, (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq)));
1298	#endif
1299	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1300	pVirtq->fUsedRingEvent = false;
1301	return;
1302	}
1303	#ifdef IN_RING3
1304	Log6Func(("...skip interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1305	pVirtq->szName,(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq), pVirtq->uUsedIdxShadow));
1306	#endif
1307	}
1308	else
1309	{
1310	/** If guest driver hasn't suppressed interrupts, interrupt */
1311	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1312	{
1313	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1314	return;
1315	}
1316	Log6Func(("...skipping interrupt for %s (guest set VIRTQ_AVAIL_F_NO_INTERRUPT)\n", pVirtq->szName));
1317	}
1318	}
1319
1320	/**
1321	* Raise interrupt or MSI-X
1322	*
1323	* @param pDevIns The device instance.
1324	* @param pVirtio Pointer to the shared virtio state.
1325	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1326	* @param uVec MSI-X vector, if enabled
1327	*/
1328	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1329	{
1330	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1331	Log6Func(("Reason for interrupt - buffer added to 'used' ring.\n"));
1332	else
1333	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1334	Log6Func(("Reason for interrupt - device config change\n"));
1335
1336	if (pVirtio->uIrqMmio)
1337	{
1338	pVirtio->uISR \|= uCause;
1339	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_HIGH);
1340	}
1341	else if (!pVirtio->fMsiSupport)
1342	{
1343	pVirtio->uISR \|= uCause;
1344	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1345	}
1346	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1347	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1348	return VINF_SUCCESS;
1349	}
1350
1351	/**
1352	* Lower interrupt (Called when guest reads ISR and when resetting)
1353	*
1354	* @param pDevIns The device instance.
1355	*/
1356	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1357	{
1358	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1359	if (pVirtio->uIrqMmio)
1360	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_LOW);
1361	else if (!pVirtio->fMsiSupport)
1362	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1363	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1364	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1365	}
1366
1367	#ifdef IN_RING3
1368	static void virtioResetVirtq(PVIRTIOCORE pVirtio, uint16_t uVirtq)
1369	{
1370	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1371	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1372
1373	pVirtq->uQueueSize = VIRTQ_SIZE;
1374	pVirtq->uEnable = false;
1375	pVirtq->uNotifyOffset = uVirtq;
1376	pVirtq->fUsedRingEvent = false;
1377	pVirtq->uAvailIdxShadow = 0;
1378	pVirtq->uUsedIdxShadow = 0;
1379	pVirtq->uMsixVector = uVirtq + 2;
1380
1381	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1382	pVirtq->uMsixVector = VIRTIO_MSI_NO_VECTOR;
1383
1384	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtq->uMsixVector);
1385	}
1386
1387	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1388	{
1389	LogFunc(("Resetting device VirtIO state\n"));
1390	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy; /* Cleared if VIRTIO_F_VERSION_1 feature ack'd */
1391	pVirtio->uDeviceFeaturesSelect = 0;
1392	pVirtio->uDriverFeaturesSelect = 0;
1393	pVirtio->uConfigGeneration = 0;
1394	pVirtio->fDeviceStatus = 0;
1395	pVirtio->uISR = 0;
1396
1397	if (!pVirtio->fMsiSupport)
1398	virtioLowerInterrupt(pDevIns, 0);
1399	else
1400	{
1401	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1402	for (int i = 0; i < VIRTQ_MAX_COUNT; i++)
1403	virtioLowerInterrupt(pDevIns, pVirtio->aVirtqueues[i].uMsixVector);
1404	}
1405
1406	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1407	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1408
1409	for (uint16_t uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
1410	virtioResetVirtq(pVirtio, uVirtq);
1411	}
1412
1413	/**
1414	* Invoked by this implementation when guest driver resets the device.
1415	* The driver itself will not until the device has read the status change.
1416	*/
1417	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1418	{
1419	Log(("%-23s: Guest reset the device\n", __FUNCTION__));
1420
1421	/* Let the client know */
1422	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0 /* fDriverOk */);
1423	virtioResetDevice(pDevIns, pVirtio);
1424	}
1425
1426	DECLHIDDEN(void) virtioCoreR3ResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1427	{
1428	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1429	}
1430	#endif /* IN_RING3 */
1431
1432	/*
1433	* Determines whether guest virtio driver is modern or legacy and does callback
1434	* informing device-specific code that feature negotiation is complete.
1435	* Should be called only once (coordinated via the 'toggle' flag)
1436	*/
1437	#ifdef IN_RING3
1438	DECLINLINE(void) virtioR3DoFeaturesCompleteOnceOnly(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1439	{
1440	if (pVirtio->uDriverFeatures & VIRTIO_F_VERSION_1)
1441	{
1442	LogFunc(("VIRTIO_F_VERSION_1 feature ack'd by guest\n"));
1443	pVirtio->fLegacyDriver = 0;
1444	}
1445	else
1446	{
1447	if (pVirtio->fOfferLegacy)
1448	{
1449	pVirtio->fLegacyDriver = 1;
1450	LogFunc(("VIRTIO_F_VERSION_1 feature was NOT set by guest\n"));
1451	}
1452	else
1453	AssertMsgFailed(("Guest didn't accept VIRTIO_F_VERSION_1, but fLegacyOffered flag not set.\n"));
1454	}
1455	if (pVirtioCC->pfnFeatureNegotiationComplete)
1456	pVirtioCC->pfnFeatureNegotiationComplete(pVirtio, pVirtio->uDriverFeatures, pVirtio->fLegacyDriver);
1457	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_COMPLETE_HANDLED;
1458	}
1459	#endif
1460
1461
1462	/**
1463	* Handles a write to the device status register from the driver.
1464	*
1465	* @returns VBox status code
1466	*
1467	* @param pDevIns The device instance.
1468	* @param pVirtio Pointer to the shared virtio state.
1469	* @param pVirtioCC Pointer to the current context virtio state.
1470	* @param fDeviceStatus The device status to be written.
1471	*/
1472	DECLINLINE(int) virtioDeviceStatusWrite(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1473	uint8_t fDeviceStatus)
1474	{
1475	pVirtio->fDeviceStatus = fDeviceStatus;
1476	bool fDeviceReset = pVirtio->fDeviceStatus == 0;
1477	#ifdef LOG_ENABLED
1478	if (LogIs7Enabled())
1479	{
1480	char szOut[80] = { 0 };
1481	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1482	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1483	}
1484	#endif
1485	bool const fStatusChanged = IS_DRIVER_OK(pVirtio) != WAS_DRIVER_OK(pVirtio);
1486
1487	if (fDeviceReset \|\| fStatusChanged)
1488	{
1489	#ifdef IN_RING0
1490	/* Since VirtIO status changes are cumbersome by nature, e.g. not a benchmark priority,
1491	* handle the rest in R3 to facilitate logging or whatever dev-specific client needs to do */
1492	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1493	return VINF_IOM_R3_MMIO_WRITE;
1494	#endif
1495	}
1496
1497	#ifdef IN_RING3
1498	/*
1499	* Notify client only if status actually changed from last time and when we're reset.
1500	*/
1501	if (fDeviceReset)
1502	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1503
1504	if (fStatusChanged)
1505	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, IS_DRIVER_OK(pVirtio));
1506	#else
1507	RT_NOREF(pDevIns, pVirtioCC);
1508	#endif
1509	/*
1510	* Save the current status for the next write so we can see what changed.
1511	*/
1512	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1513	return VINF_SUCCESS;
1514	}
1515
1516
1517	/**
1518	* Handles a read from the device status register from the driver.
1519	*
1520	* @returns The device status register value.
1521	*
1522	* @param pVirtio Pointer to the shared virtio state.
1523	*/
1524	DECLINLINE(uint8_t) virtioDeviceStatusRead(PVIRTIOCORE pVirtio)
1525	{
1526	#ifdef LOG_ENABLED
1527	if (LogIs7Enabled())
1528	{
1529	char szOut[80] = { 0 };
1530	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1531	LogFunc(("Guest read fDeviceStatus ................ (%s)\n", szOut));
1532	}
1533	#endif
1534	return pVirtio->fDeviceStatus;
1535	}
1536
1537
1538	/**
1539	* Handle accesses to Common Configuration capability
1540	*
1541	* @returns VBox status code
1542	*
1543	* @param pDevIns The device instance.
1544	* @param pVirtio Pointer to the shared virtio state.
1545	* @param pVirtioCC Pointer to the current context virtio state.
1546	* @param fWrite Set if write access, clear if read access.
1547	* @param uOffsetOfAccess The common configuration capability offset.
1548	* @param cb Number of bytes to read or write
1549	* @param pv Pointer to location to write to or read from
1550	*/
1551	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1552	int fWrite, uint32_t uOffsetOfAccess, unsigned cb, void *pv)
1553	{
1554	uint16_t uVirtq = pVirtio->uVirtqSelect;
1555	int rc = VINF_SUCCESS;
1556	uint64_t val;
1557	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1558	{
1559	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1560	{
1561	/* VirtIO 1.0, 4.1.4.3 states device_feature is a (guest) driver readonly field,
1562	* yet the linux driver attempts to write/read it back twice */
1563	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1564	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1565	return VINF_IOM_MMIO_UNUSED_00;
1566	}
1567	else /* Guest READ pCommonCfg->uDeviceFeatures */
1568	{
1569	switch (pVirtio->uDeviceFeaturesSelect)
1570	{
1571	case 0:
1572	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1573	memcpy(pv, &val, cb);
1574	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1575	break;
1576	case 1:
1577	val = pVirtio->uDeviceFeatures >> 32;
1578	memcpy(pv, &val, cb);
1579	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1580	break;
1581	default:
1582	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1583	pVirtio->uDeviceFeaturesSelect));
1584	return VINF_IOM_MMIO_UNUSED_00;
1585	}
1586	}
1587	}
1588	else
1589	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1590	{
1591	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1592	{
1593	switch (pVirtio->uDriverFeaturesSelect)
1594	{
1595	case 0:
1596	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1597	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
1598	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1599	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1600	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1601	#ifdef IN_RING0
1602	return VINF_IOM_R3_MMIO_WRITE;
1603	#endif
1604	#ifdef IN_RING3
1605	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1606	#endif
1607	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1608	break;
1609	case 1:
1610	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1611	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
1612	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1613	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1614	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1615	#ifdef IN_RING0
1616	return VINF_IOM_R3_MMIO_WRITE;
1617	#endif
1618	#ifdef IN_RING3
1619	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1620	#endif
1621	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1622	break;
1623	default:
1624	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1625	pVirtio->uDriverFeaturesSelect));
1626	return VINF_SUCCESS;
1627	}
1628	}
1629	else /* Guest READ pCommonCfg->udriverFeatures */
1630	{
1631	switch (pVirtio->uDriverFeaturesSelect)
1632	{
1633	case 0:
1634	val = pVirtio->uDriverFeatures & 0xffffffff;
1635	memcpy(pv, &val, cb);
1636	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1637	break;
1638	case 1:
1639	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1640	memcpy(pv, &val, cb);
1641	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + 4);
1642	break;
1643	default:
1644	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1645	pVirtio->uDriverFeaturesSelect));
1646	return VINF_IOM_MMIO_UNUSED_00;
1647	}
1648	}
1649	}
1650	else
1651	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1652	{
1653	if (fWrite)
1654	{
1655	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1656	return VINF_SUCCESS;
1657	}
1658	(uint16_t )pv = VIRTQ_MAX_COUNT;
1659	VIRTIO_DEV_CONFIG_LOG_ACCESS(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1660	}
1661	else
1662	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1663	{
1664	if (fWrite) /* Guest WRITE pCommonCfg->fDeviceStatus */
1665	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t )pv);
1666	else /* Guest READ pCommonCfg->fDeviceStatus */
1667	(uint8_t )pv = virtioDeviceStatusRead(pVirtio);
1668	}
1669	else
1670	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1671	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1672	else
1673	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1674	VIRTIO_DEV_CONFIG_ACCESS( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1675	else
1676	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1677	VIRTIO_DEV_CONFIG_ACCESS( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1678	else
1679	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1680	VIRTIO_DEV_CONFIG_ACCESS( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1681	else
1682	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1683	{
1684	if (fWrite) {
1685	uint16_t uVirtqNew = (uint16_t )pv;
1686
1687	if (uVirtqNew < RT_ELEMENTS(pVirtio->aVirtqueues))
1688	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1689	else
1690	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1691	}
1692	else
1693	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1694	}
1695	else
1696	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqDesc, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1697	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqDesc, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1698	else
1699	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqAvail, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1700	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqAvail, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1701	else
1702	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqUsed, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1703	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqUsed, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1704	else
1705	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1706	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1707	else
1708	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uEnable, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1709	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uEnable, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1710	else
1711	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uNotifyOffset, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1712	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uNotifyOffset, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1713	else
1714	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1715	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1716	else
1717	{
1718	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: uOffsetOfAccess=%#x (%d), cb=%d\n",
1719	fWrite ? "write" : "read ", uOffsetOfAccess, uOffsetOfAccess, cb));
1720	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1721	}
1722
1723	#ifndef IN_RING3
1724	RT_NOREF(pDevIns, pVirtioCC);
1725	#endif
1726	return rc;
1727	}
1728
1729	/**
1730	* @callback_method_impl{FNIOMIOPORTNEWIN)
1731	*
1732	* This I/O handler exists only to handle access from legacy drivers.
1733	*/
1734	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortIn(PPDMDEVINS pDevIns, void pvUser, RTIOPORT offPort, uint32_t pu32, unsigned cb)
1735	{
1736	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1737	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1738
1739	RT_NOREF(pvUser);
1740	Log(("%-23s: Port read at offset=%RTiop, cb=%#x%s",
1741	__FUNCTION__, offPort, cb,
1742	VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort) ? "" : "\n"));
1743
1744	void pv = pu32; / To use existing macros */
1745	int fWrite = 0; /* To use existing macros */
1746
1747	uint16_t uVirtq = pVirtio->uVirtqSelect;
1748
1749	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1750	{
1751	uint32_t val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1752	memcpy(pu32, &val, cb);
1753	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1754	}
1755	else
1756	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1757	{
1758	uint32_t val = pVirtio->uDriverFeatures & UINT32_C(0xffffffff);
1759	memcpy(pu32, &val, cb);
1760	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1761	}
1762	else
1763	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1764	{
1765	(uint8_t )pu32 = pVirtio->fDeviceStatus;
1766	#ifdef LOG_ENABLED
1767	if (LogIs7Enabled())
1768	{
1769	char szOut[80] = { 0 };
1770	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1771	Log(("%-23s: Guest read fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1772	}
1773	#endif
1774	}
1775	else
1776	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1777	{
1778	ASSERT_GUEST_MSG(cb == 1, ("%d\n", cb));
1779	(uint8_t )pu32 = pVirtio->uISR;
1780	pVirtio->uISR = 0;
1781	virtioLowerInterrupt( pDevIns, 0);
1782	Log((" (ISR read and cleared)\n"));
1783	}
1784	else
1785	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1786	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1787	else
1788	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1789	{
1790	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[uVirtq];
1791	*pu32 = pVirtQueue->GCPhysVirtqDesc >> GUEST_PAGE_SHIFT;
1792	Log(("%-23s: Guest read uVirtqPfn .................... %#x\n", __FUNCTION__, *pu32));
1793	}
1794	else
1795	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1796	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1797	else
1798	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1799	VIRTIO_DEV_CONFIG_ACCESS( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1800	#ifdef LEGACY_MSIX_SUPPORTED
1801	else
1802	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1803	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1804	else
1805	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1806	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1807	#endif
1808	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1809	{
1810	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1811	#ifdef IN_RING3
1812	/* Access device-specific configuration */
1813	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1814	int rc = pVirtioCC->pfnDevCapRead(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1815	return rc;
1816	#else
1817	return VINF_IOM_R3_IOPORT_READ;
1818	#endif
1819	}
1820	else
1821	{
1822	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1823	Log2Func(("Bad guest read access to virtio_legacy_pci_common_cfg: offset=%#x, cb=%x\n",
1824	offPort, cb));
1825	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1826	"virtioLegacyIOPortIn: no valid port at offset offset=%RTiop cb=%#x\n", offPort, cb);
1827	return rc;
1828	}
1829	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1830	return VINF_SUCCESS;
1831	}
1832
1833	/**
1834	* @callback_method_impl{ * @callback_method_impl{FNIOMIOPORTNEWOUT}
1835	*
1836	* This I/O Port interface exists only to handle access from legacy drivers.
1837	*/
1838	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT offPort, uint32_t u32, unsigned cb)
1839	{
1840	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1841	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1842	RT_NOREF(pvUser);
1843
1844	uint16_t uVirtq = pVirtio->uVirtqSelect;
1845	uint32_t u32OnStack = u32; /* allows us to use this impl's MMIO parsing macros */
1846	void pv = &u32OnStack; / To use existing macros */
1847	int fWrite = 1; /* To use existing macros */
1848
1849	Log(("%-23s: Port written at offset=%RTiop, cb=%#x, u32=%#x\n", __FUNCTION__, offPort, cb, u32));
1850
1851	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1852	{
1853	if (u32 < RT_ELEMENTS(pVirtio->aVirtqueues))
1854	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1855	else
1856	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1857	}
1858	else
1859	#ifdef LEGACY_MSIX_SUPPORTED
1860	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1861	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1862	else
1863	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1864	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1865	else
1866	#endif
1867	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1868	{
1869	/* Check to see if guest acknowledged unsupported features */
1870	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1871	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1872	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1873	return VINF_SUCCESS;
1874	}
1875	else
1876	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1877	{
1878	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1879	if ((pVirtio->uDriverFeatures & ~VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED) == 0)
1880	{
1881	Log(("Guest asked for features host does not support! (host=%x guest=%x)\n",
1882	VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED, pVirtio->uDriverFeatures));
1883	pVirtio->uDriverFeatures &= VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED;
1884	}
1885	if (!(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1886	{
1887	#ifdef IN_RING0
1888	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1889	return VINF_IOM_R3_IOPORT_WRITE;
1890	#endif
1891	#ifdef IN_RING3
1892	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1893	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1894	#endif
1895	}
1896	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1897	}
1898	else
1899	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1900	{
1901	VIRTIO_DEV_CONFIG_LOG_ACCESS(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1902	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (queue size) (ignoring)\n"));
1903	return VINF_SUCCESS;
1904	}
1905	else
1906	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1907	{
1908	bool const fDriverInitiatedReset = (pVirtio->fDeviceStatus = (uint8_t)u32) == 0;
1909	bool const fDriverStateImproved = IS_DRIVER_OK(pVirtio) && !WAS_DRIVER_OK(pVirtio);
1910	#ifdef LOG_ENABLED
1911	if (LogIs7Enabled())
1912	{
1913	char szOut[80] = { 0 };
1914	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1915	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1916	}
1917	#endif
1918	if (fDriverStateImproved \|\| fDriverInitiatedReset)
1919	{
1920	#ifdef IN_RING0
1921	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1922	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1923	return VINF_IOM_R3_IOPORT_WRITE;
1924	#endif
1925	}
1926
1927	#ifdef IN_RING3
1928	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1929	if (fDriverInitiatedReset)
1930	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1931
1932	else if (fDriverStateImproved)
1933	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 1 /* fDriverOk */);
1934
1935	#endif
1936	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1937	}
1938	else
1939	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1940	{
1941	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1942	uint64_t uVirtqPfn = (uint64_t)u32;
1943
1944	if (uVirtqPfn)
1945	{
1946	/* Transitional devices calculate ring physical addresses using rigid spec-defined formulae,
1947	* instead of guest conveying respective address of each ring, as "modern" VirtIO drivers do,
1948	* thus there is no virtq PFN or single base queue address stored in instance data for
1949	* this transitional device, but rather it is derived, when read back, from GCPhysVirtqDesc */
1950
1951	pVirtq->GCPhysVirtqDesc = uVirtqPfn * VIRTIO_PAGE_SIZE;
1952	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
1953	pVirtq->GCPhysVirtqUsed =
1954	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
1955	}
1956	else
1957	{
1958	/* Don't set ring addresses for queue (to meaningless values), when guest resets the virtq's PFN */
1959	pVirtq->GCPhysVirtqDesc = 0;
1960	pVirtq->GCPhysVirtqAvail = 0;
1961	pVirtq->GCPhysVirtqUsed = 0;
1962	}
1963	Log(("%-23s: Guest wrote uVirtqPfn .................... %#x:\n"
1964	"%68s... %p -> GCPhysVirtqDesc\n%68s... %p -> GCPhysVirtqAvail\n%68s... %p -> GCPhysVirtqUsed\n",
1965	__FUNCTION__, u32, " ", pVirtq->GCPhysVirtqDesc, " ", pVirtq->GCPhysVirtqAvail, " ", pVirtq->GCPhysVirtqUsed));
1966	}
1967	else
1968	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1969	{
1970	#ifdef IN_RING3
1971	ASSERT_GUEST_MSG(cb == 2, ("cb=%u\n", cb));
1972	pVirtio->uQueueNotify = u32 & 0xFFFF;
1973	if (uVirtq < VIRTQ_MAX_COUNT)
1974	{
1975	RT_UNTRUSTED_VALIDATED_FENCE();
1976
1977	/* Need to check that queue is configured. Legacy spec didn't have a queue enabled flag */
1978	if (pVirtio->aVirtqueues[pVirtio->uQueueNotify].GCPhysVirtqDesc)
1979	virtioCoreVirtqNotified(pDevIns, pVirtio, pVirtio->uQueueNotify, pVirtio->uQueueNotify /* uNotifyIdx */);
1980	else
1981	Log(("The queue (#%d) being notified has not been initialized.\n", pVirtio->uQueueNotify));
1982	}
1983	else
1984	Log(("Invalid queue number (%d)\n", pVirtio->uQueueNotify));
1985	#else
1986	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1987	return VINF_IOM_R3_IOPORT_WRITE;
1988	#endif
1989	}
1990	else
1991	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1992	{
1993	VIRTIO_DEV_CONFIG_LOG_ACCESS( fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1994	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (ISR status) (ignoring)\n"));
1995	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1996	return VINF_SUCCESS;
1997	}
1998	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1999	{
2000	#ifdef IN_RING3
2001
2002	/* Access device-specific configuration */
2003	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2004	return pVirtioCC->pfnDevCapWrite(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
2005	#else
2006	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2007	return VINF_IOM_R3_IOPORT_WRITE;
2008	#endif
2009	}
2010	else
2011	{
2012	Log2Func(("Bad guest write access to virtio_legacy_pci_common_cfg: offset=%#x, cb=0x%x\n",
2013	offPort, cb));
2014	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2015	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2016	"virtioLegacyIOPortOut: no valid port at offset offset=%RTiop cb=0x%#x\n", offPort, cb);
2017	return rc;
2018	}
2019
2020	RT_NOREF(uVirtq);
2021	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2022	return VINF_SUCCESS;
2023	}
2024
2025
2026	/**
2027	* Read from the device specific configuration at the given offset.
2028	*
2029	* @returns VBox status code.
2030	* @param pDevIns
2031	*/
2032	DECLINLINE(VBOXSTRICTRC) virtioDeviceCfgRead(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
2033	uint32_t offDevCfg, void *pv, unsigned cb)
2034	{
2035	#ifdef IN_RING3
2036	/*
2037	* Callback to client to manage device-specific configuration.
2038	*/
2039	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offDevCfg, pv, cb);
2040
2041	/*
2042	* Anytime any part of the dev-specific dev config (which this virtio core implementation sees
2043	* as a blob, and virtio dev-specific code separates into fields) is READ, it must be compared
2044	* for deltas from previous read to maintain a config gen. seq. counter (VirtIO 1.0, section 4.1.4.3.1)
2045	*/
2046	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offDevCfg,
2047	pVirtioCC->pbPrevDevSpecificCfg + offDevCfg,
2048	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offDevCfg)));
2049
2050	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
2051
2052	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
2053	{
2054	++pVirtio->uConfigGeneration;
2055	Log6Func(("Bumped cfg. generation to %d because %s%s\n", pVirtio->uConfigGeneration,
2056	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
2057	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
2058	pVirtio->fGenUpdatePending = false;
2059	}
2060
2061	virtioLowerInterrupt(pDevIns, 0);
2062	return rcStrict;
2063	#else
2064	RT_NOREF(pDevIns, pVirtio, pVirtioCC, offDevCfg, pv, cb);
2065	return VINF_IOM_R3_MMIO_READ;
2066	#endif
2067	}
2068
2069
2070	/**
2071	* @callback_method_impl{FNIOMMMIONEWREAD,
2072	* Memory mapped I/O Handler for PCI Capabilities read operations.}
2073	*
2074	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2075	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
2076	* of 1, 2 or 4 bytes, only.
2077	*
2078	*/
2079	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2080	{
2081	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2082	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2083	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VINF_IOM_MMIO_UNUSED_FF);
2084	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2085
2086	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2087
2088	VBOXSTRICTRC rcStrict;
2089	uint32_t uOffset;
2090	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2091	rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, uOffset, pv, cb);
2092	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2093	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, uOffset, cb, pv);
2094	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap))
2095	{
2096	(uint8_t )pv = pVirtio->uISR;
2097	Log6Func(("Read and clear ISR\n"));
2098	pVirtio->uISR = 0; /* VirtIO spec requires reads of ISR to clear it */
2099	virtioLowerInterrupt(pDevIns, 0);
2100	rcStrict = VINF_SUCCESS;
2101	}
2102	else
2103	{
2104	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2105	memset(pv, 0xFF, cb);
2106	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2107	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2108	}
2109
2110	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2111	return rcStrict;
2112	}
2113
2114	/**
2115	* @callback_method_impl{FNIOMMMIONEWREAD,
2116	* Memory mapped I/O Handler for PCI Capabilities write operations.}
2117	*
2118	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2119	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
2120	* of 1, 2 or 4 bytes, only.
2121	*/
2122	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2123	{
2124	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2125	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2126	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
2127	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2128	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2129
2130	VBOXSTRICTRC rcStrict;
2131	uint32_t uOffset;
2132	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2133	{
2134	#ifdef IN_RING3
2135	/*
2136	* Foreward this MMIO write access for client to deal with.
2137	*/
2138	rcStrict = pVirtioCC->pfnDevCapWrite(pDevIns, uOffset, pv, cb);
2139	#else
2140	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2141	rcStrict = VINF_IOM_R3_MMIO_WRITE;
2142	#endif
2143	}
2144	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2145	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, uOffset, cb, (void )pv);
2146	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
2147	{
2148	pVirtio->uISR = (uint8_t )pv;
2149	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
2150	pVirtio->uISR & 0xff,
2151	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
2152	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
2153	rcStrict = VINF_SUCCESS;
2154	}
2155	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
2156	{
2157	/* This should be guest driver dropping index of a new descriptor in avail ring */
2158	virtioCoreVirtqNotified(pDevIns, pVirtio, uOffset / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
2159	rcStrict = VINF_SUCCESS;
2160	}
2161	else
2162	{
2163	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2164	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2165	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2166	}
2167
2168	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2169	return rcStrict;
2170	}
2171
2172
2173	/**
2174	* @callback_method_impl{FNIOMMMIONEWREAD,
2175	* Memory mapped I/O Handler for Virtio over MMIO read operations.}
2176	*
2177	*/
2178	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2179	{
2180	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2181	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2182	RT_NOREF(pvUser);
2183	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2184
2185	if (off >= VIRTIO_MMIO_SIZE)
2186	{
2187	VBOXSTRICTRC rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2188	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2189	return rcStrict;
2190	}
2191
2192	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2193	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2194	("Bad read access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2195	VINF_IOM_MMIO_UNUSED_FF);
2196
2197	int rc = VINF_SUCCESS;
2198	uint32_t pu32 = (uint32_t )pv;
2199	switch (off)
2200	{
2201	case VIRTIO_MMIO_REG_MAGIC_OFF:
2202	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_MAGIC_VALUE);
2203	break;
2204	case VIRTIO_MMIO_REG_VERSION_OFF:
2205	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_VERSION_VALUE);
2206	break;
2207	case VIRTIO_MMIO_REG_DEVICEID_OFF:
2208	*pu32 = pVirtio->uDeviceType;
2209	break;
2210	case VIRTIO_MMIO_REG_VENDORID_OFF:
2211	*pu32 = RT_H2LE_U32(DEVICE_PCI_VENDOR_ID_VIRTIO);
2212	break;
2213	case VIRTIO_MMIO_REG_DEVICEFEAT_OFF:
2214	{
2215	switch (pVirtio->uDeviceFeaturesSelect)
2216	{
2217	case 0:
2218	*pu32 = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
2219	break;
2220	case 1:
2221	*pu32 = pVirtio->uDeviceFeatures >> 32;
2222	break;
2223	default:
2224	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
2225	pVirtio->uDeviceFeaturesSelect));
2226	rc = VINF_IOM_MMIO_UNUSED_00;
2227	}
2228	break;
2229	}
2230	case VIRTIO_MMIO_REG_QUEUENUMMAX_OFF:
2231	pu32 = VIRTQ_SIZE; /* @todo */
2232	break;
2233	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2234	{
2235	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2236	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2237	*pu32 = pVirtQueue->uEnable;
2238	break;
2239	}
2240	case VIRTIO_MMIO_REG_INTRSTATUS_OFF:
2241	*pu32 = pVirtio->uISR;
2242	break;
2243	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2244	*pu32 = virtioDeviceStatusRead(pVirtio);
2245	break;
2246	case VIRTIO_MMIO_REG_CFGGEN_OFF:
2247	*pu32 = pVirtio->uConfigGeneration;
2248	break;
2249	default:
2250	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2251	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2252	"virtioMmioTransportRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2253	}
2254
2255	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2256	return rc;
2257	}
2258
2259	/**
2260	* @callback_method_impl{FNIOMMMIONEWREAD,
2261	* Memory mapped I/O Handler for Virtio over MMIO write operations.}
2262	*/
2263	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2264	{
2265	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2266	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2267	RT_NOREF(pvUser);
2268	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2269
2270	if (off >= VIRTIO_MMIO_SIZE)
2271	{
2272	#ifdef IN_RING3
2273	/*
2274	* Forward this MMIO write access for client to deal with.
2275	*/
2276	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2277	return pVirtioCC->pfnDevCapWrite(pDevIns, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2278	#else
2279	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2280	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2281	return VINF_IOM_R3_MMIO_WRITE;
2282	#endif
2283	}
2284
2285	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2286	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2287	("Bad write access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2288	VINF_SUCCESS);
2289
2290	int rc = VINF_SUCCESS;
2291	uint32_t const u32Val = (const uint32_t )pv;
2292	switch (off)
2293	{
2294	case VIRTIO_MMIO_REG_DEVICEFEATSEL_OFF:
2295	{
2296	pVirtio->uDeviceFeaturesSelect = u32Val;
2297	break;
2298	}
2299	case VIRTIO_MMIO_REG_DRIVERFEAT_OFF:
2300	{
2301	switch (pVirtio->uDriverFeaturesSelect)
2302	{
2303	case 0:
2304	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0xffffffff00000000)) \| u32Val;
2305	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
2306	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2307	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2308	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2309	#ifdef IN_RING0
2310	return VINF_IOM_R3_MMIO_WRITE;
2311	#endif
2312	#ifdef IN_RING3
2313	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2314	#endif
2315	break;
2316	case 1:
2317	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2318	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
2319	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2320	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2321	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2322	#ifdef IN_RING0
2323	return VINF_IOM_R3_MMIO_WRITE;
2324	#endif
2325	#ifdef IN_RING3
2326	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2327	#endif
2328	break;
2329	default:
2330	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
2331	pVirtio->uDriverFeaturesSelect));
2332	return VINF_SUCCESS;
2333	}
2334	break;
2335	}
2336	case VIRTIO_MMIO_REG_DRIVERFEATSEL_OFF:
2337	{
2338	pVirtio->uDriverFeaturesSelect = u32Val;
2339	break;
2340	}
2341	case VIRTIO_MMIO_REG_QUEUESEL_OFF:
2342	{
2343	if (u32Val < RT_ELEMENTS(pVirtio->aVirtqueues))
2344	pVirtio->uVirtqSelect = (uint16_t)u32Val;
2345	else
2346	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
2347	break;
2348	}
2349	case VIRTIO_MMIO_REG_QUEUENUM_OFF:
2350	{
2351	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2352	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2353	pVirtQueue->uQueueSize = (uint16_t)u32Val;
2354	break;
2355	}
2356	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2357	{
2358	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2359	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2360	pVirtQueue->uEnable = (uint16_t)u32Val;
2361	break;
2362	}
2363	case VIRTIO_MMIO_REG_QUEUENOTIFY_OFF:
2364	{
2365	virtioCoreVirtqNotified(pDevIns, pVirtio, u32Val, (uint16_t)u32Val);
2366	break;
2367	}
2368	case VIRTIO_MMIO_REG_INTRACK_OFF:
2369	{
2370	pVirtio->uISR &= ~u32Val;
2371	if (!pVirtio->uISR)
2372	virtioLowerInterrupt(pDevIns, 0);
2373	break;
2374	}
2375	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2376	{
2377	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t)u32Val);
2378	break;
2379	}
2380	case VIRTIO_MMIO_REG_QUEUEALIGN_LEGACY_OFF:
2381	{
2382	/* Written by edk2 even though we don't offer legacy mode, ignore. */
2383	break;
2384	}
2385	case VIRTIO_MMIO_REG_QUEUEDESCLOW_OFF:
2386	{
2387	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2388	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2389	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0xffffffff00000000)) \| u32Val;
2390	break;
2391	}
2392	case VIRTIO_MMIO_REG_QUEUEDESCHIGH_OFF:
2393	{
2394	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2395	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2396	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2397	break;
2398	}
2399	case VIRTIO_MMIO_REG_QUEUEDRVLOW_OFF:
2400	{
2401	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2402	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2403	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0xffffffff00000000)) \| u32Val;
2404	break;
2405	}
2406	case VIRTIO_MMIO_REG_QUEUEDRVHIGH_OFF:
2407	{
2408	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2409	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2410	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2411	break;
2412	}
2413	case VIRTIO_MMIO_REG_QUEUEDEVLOW_OFF:
2414	{
2415	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2416	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2417	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0xffffffff00000000)) \| u32Val;
2418	break;
2419	}
2420	case VIRTIO_MMIO_REG_QUEUEDEVHIGH_OFF:
2421	{
2422	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2423	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2424	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2425	break;
2426	}
2427	default:
2428	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2429	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2430	"virtioMmioTransportWrite: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2431	}
2432
2433	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2434	return rc;
2435	}
2436
2437
2438	#ifdef IN_RING3
2439
2440	/**
2441	* @callback_method_impl{FNPCICONFIGREAD}
2442	*/
2443	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2444	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
2445	{
2446	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2447	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2448	RT_NOREF(pPciDev);
2449
2450	if (uAddress == pVirtio->uPciCfgDataOff)
2451	{
2452	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2453	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2454	uint32_t uLength = pPciCap->uLength;
2455
2456	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u uLength=%d, bar=%d\n",
2457	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, uLength, pPciCap->uBar));
2458
2459	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2460	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2461	{
2462	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. "
2463	"Ignoring\n"));
2464	*pu32Value = UINT32_MAX;
2465	return VINF_SUCCESS;
2466	}
2467
2468	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
2469	Log7Func((" Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=0x%x -> %Rrc\n",
2470	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
2471	return rcStrict;
2472	}
2473	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u pu32Value=%p\n",
2474	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, pu32Value));
2475	return VINF_PDM_PCI_DO_DEFAULT;
2476	}
2477
2478	/**
2479	* @callback_method_impl{FNPCICONFIGWRITE}
2480	*/
2481	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2482	uint32_t uAddress, unsigned cb, uint32_t u32Value)
2483	{
2484	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2485	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2486	RT_NOREF(pPciDev);
2487
2488	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x %scb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, uAddress < 0xf ? " " : "", cb, u32Value));
2489	if (uAddress == pVirtio->uPciCfgDataOff)
2490	{
2491	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2492	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2493	uint32_t uLength = pPciCap->uLength;
2494
2495	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2496	\|\| cb != uLength
2497	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2498	{
2499	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
2500	return VINF_SUCCESS;
2501	}
2502
2503	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
2504	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
2505	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
2506	return rcStrict;
2507	}
2508	return VINF_PDM_PCI_DO_DEFAULT;
2509	}
2510
2511
2512	/*********************************************************************************************************************************
2513	* Saved state (SSM) *
2514	*********************************************************************************************************************************/
2515
2516
2517	/**
2518	* Loads a saved device state (called from device-specific code on SSM final pass)
2519	*
2520	* @param pVirtio Pointer to the shared virtio state.
2521	* @param pHlp The ring-3 device helpers.
2522	* @param pSSM The saved state handle.
2523	* @returns VBox status code.
2524	*/
2525	DECLHIDDEN(int) virtioCoreR3LegacyDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2526	uint32_t uVersion, uint32_t uVirtioLegacy_3_1_Beta)
2527	{
2528	int rc;
2529	uint32_t uDriverFeaturesLegacy32bit;
2530
2531	rc = pHlp->pfnSSMGetU32( pSSM, &uDriverFeaturesLegacy32bit);
2532	AssertRCReturn(rc, rc);
2533	pVirtio->uDriverFeatures = (uint64_t)uDriverFeaturesLegacy32bit;
2534
2535	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2536	AssertRCReturn(rc, rc);
2537
2538	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2539	AssertRCReturn(rc, rc);
2540
2541	#ifdef LOG_ENABLED
2542	char szOut[80] = { 0 };
2543	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
2544	Log(("Loaded legacy device status = (%s)\n", szOut));
2545	#endif
2546
2547	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2548	AssertRCReturn(rc, rc);
2549
2550	uint32_t cQueues = 3; /* This constant default value copied from earliest v0.9 code */
2551	if (uVersion > uVirtioLegacy_3_1_Beta)
2552	{
2553	rc = pHlp->pfnSSMGetU32(pSSM, &cQueues);
2554	AssertRCReturn(rc, rc);
2555	}
2556
2557	AssertLogRelMsgReturn(cQueues <= VIRTQ_MAX_COUNT, ("%#x\n", cQueues), VERR_SSM_LOAD_CONFIG_MISMATCH);
2558	AssertLogRelMsgReturn(pVirtio->uVirtqSelect < cQueues \|\| (cQueues == 0 && pVirtio->uVirtqSelect),
2559	("uVirtqSelect=%u cQueues=%u\n", pVirtio->uVirtqSelect, cQueues),
2560	VERR_SSM_LOAD_CONFIG_MISMATCH);
2561
2562	Log(("\nRestoring %d legacy-only virtio-net device queues from saved state:\n", cQueues));
2563	for (unsigned uVirtq = 0; uVirtq < cQueues; uVirtq++)
2564	{
2565	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
2566
2567	if (uVirtq == cQueues - 1)
2568	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-ctrlq");
2569	else if (uVirtq % 2)
2570	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-xmitq<%d>", uVirtq / 2);
2571	else
2572	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-recvq<%d>", uVirtq / 2);
2573
2574	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uQueueSize);
2575	AssertRCReturn(rc, rc);
2576
2577	uint32_t uVirtqPfn;
2578	rc = pHlp->pfnSSMGetU32(pSSM, &uVirtqPfn);
2579	AssertRCReturn(rc, rc);
2580
2581	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uAvailIdxShadow);
2582	AssertRCReturn(rc, rc);
2583
2584	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uUsedIdxShadow);
2585	AssertRCReturn(rc, rc);
2586
2587	if (uVirtqPfn)
2588	{
2589	pVirtq->GCPhysVirtqDesc = (uint64_t)uVirtqPfn * VIRTIO_PAGE_SIZE;
2590	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
2591	pVirtq->GCPhysVirtqUsed =
2592	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
2593	pVirtq->uEnable = 1;
2594	}
2595	else
2596	{
2597	LogFunc(("WARNING: QUEUE \"%s\" PAGE NUMBER ZERO IN SAVED STATE\n", pVirtq->szName));
2598	pVirtq->uEnable = 0;
2599	}
2600	pVirtq->uNotifyOffset = 0; /* unused in legacy mode */
2601	pVirtq->uMsixVector = 0; /* unused in legacy mode */
2602	}
2603	pVirtio->fGenUpdatePending = 0; /* unused in legacy mode */
2604	pVirtio->uConfigGeneration = 0; /* unused in legacy mode */
2605	pVirtio->uPciCfgDataOff = 0; /* unused in legacy mode (port I/O used instead) */
2606
2607	return VINF_SUCCESS;
2608	}
2609
2610	/**
2611	* Loads a saved device state (called from device-specific code on SSM final pass)
2612	*
2613	* Note: This loads state saved by a Modern (VirtIO 1.0+) device, of which this transitional device is one,
2614	* and thus supports both legacy and modern guest virtio drivers.
2615	*
2616	* @param pVirtio Pointer to the shared virtio state.
2617	* @param pHlp The ring-3 device helpers.
2618	* @param pSSM The saved state handle.
2619	* @returns VBox status code.
2620	*/
2621	DECLHIDDEN(int) virtioCoreR3ModernDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2622	uint32_t uVersion, uint32_t uTestVersion, uint32_t cQueues)
2623	{
2624	RT_NOREF2(cQueues, uVersion);
2625	LogFunc(("\n"));
2626	/*
2627	* Check the marker and (embedded) version number.
2628	*/
2629	uint64_t uMarker = 0;
2630	int rc;
2631
2632	rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
2633	AssertRCReturn(rc, rc);
2634	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
2635	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2636	N_("Expected marker value %#RX64 found %#RX64 instead"),
2637	VIRTIO_SAVEDSTATE_MARKER, uMarker);
2638	uint32_t uVersionSaved = 0;
2639	rc = pHlp->pfnSSMGetU32(pSSM, &uVersionSaved);
2640	AssertRCReturn(rc, rc);
2641	if (uVersionSaved != uTestVersion)
2642	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2643	N_("Unsupported virtio version: %u"), uVersionSaved);
2644	/*
2645	* Load the state.
2646	*/
2647	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->fLegacyDriver);
2648	AssertRCReturn(rc, rc);
2649	rc = pHlp->pfnSSMGetBool( pSSM, &pVirtio->fGenUpdatePending);
2650	AssertRCReturn(rc, rc);
2651	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2652	AssertRCReturn(rc, rc);
2653	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uConfigGeneration);
2654	AssertRCReturn(rc, rc);
2655	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uPciCfgDataOff);
2656	AssertRCReturn(rc, rc);
2657	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2658	AssertRCReturn(rc, rc);
2659	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2660	AssertRCReturn(rc, rc);
2661	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDeviceFeaturesSelect);
2662	AssertRCReturn(rc, rc);
2663	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDriverFeaturesSelect);
2664	AssertRCReturn(rc, rc);
2665	rc = pHlp->pfnSSMGetU64( pSSM, &pVirtio->uDriverFeatures);
2666	AssertRCReturn(rc, rc);
2667
2668	/** @todo Adapt this loop use cQueues argument instead of static queue count (safely with SSM versioning) */
2669	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2670	{
2671	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2672	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqDesc);
2673	AssertRCReturn(rc, rc);
2674	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqAvail);
2675	AssertRCReturn(rc, rc);
2676	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqUsed);
2677	AssertRCReturn(rc, rc);
2678	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uNotifyOffset);
2679	AssertRCReturn(rc, rc);
2680	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uMsixVector);
2681	AssertRCReturn(rc, rc);
2682	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uEnable);
2683	AssertRCReturn(rc, rc);
2684	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uQueueSize);
2685	AssertRCReturn(rc, rc);
2686	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uAvailIdxShadow);
2687	AssertRCReturn(rc, rc);
2688	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uUsedIdxShadow);
2689	AssertRCReturn(rc, rc);
2690	rc = pHlp->pfnSSMGetMem( pSSM, pVirtq->szName, sizeof(pVirtq->szName));
2691	AssertRCReturn(rc, rc);
2692	}
2693	return VINF_SUCCESS;
2694	}
2695
2696	/**
2697	* Called from the FNSSMDEVSAVEEXEC function of the device.
2698	*
2699	* @param pVirtio Pointer to the shared virtio state.
2700	* @param pHlp The ring-3 device helpers.
2701	* @param pSSM The saved state handle.
2702	* @returns VBox status code.
2703	*/
2704	DECLHIDDEN(int) virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t cQueues)
2705	{
2706	RT_NOREF(cQueues);
2707	/** @todo figure out a way to save cQueues (with SSM versioning) */
2708
2709	LogFunc(("\n"));
2710	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
2711	pHlp->pfnSSMPutU32(pSSM, uVersion);
2712
2713	pHlp->pfnSSMPutU32( pSSM, pVirtio->fLegacyDriver);
2714	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
2715	pHlp->pfnSSMPutU8( pSSM, pVirtio->fDeviceStatus);
2716	pHlp->pfnSSMPutU8( pSSM, pVirtio->uConfigGeneration);
2717	pHlp->pfnSSMPutU8( pSSM, pVirtio->uPciCfgDataOff);
2718	pHlp->pfnSSMPutU8( pSSM, pVirtio->uISR);
2719	pHlp->pfnSSMPutU16( pSSM, pVirtio->uVirtqSelect);
2720	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDeviceFeaturesSelect);
2721	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDriverFeaturesSelect);
2722	pHlp->pfnSSMPutU64( pSSM, pVirtio->uDriverFeatures);
2723
2724	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2725	{
2726	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2727
2728	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqDesc);
2729	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqAvail);
2730	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqUsed);
2731	pHlp->pfnSSMPutU16( pSSM, pVirtq->uNotifyOffset);
2732	pHlp->pfnSSMPutU16( pSSM, pVirtq->uMsixVector);
2733	pHlp->pfnSSMPutU16( pSSM, pVirtq->uEnable);
2734	pHlp->pfnSSMPutU16( pSSM, pVirtq->uQueueSize);
2735	pHlp->pfnSSMPutU16( pSSM, pVirtq->uAvailIdxShadow);
2736	pHlp->pfnSSMPutU16( pSSM, pVirtq->uUsedIdxShadow);
2737	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtq->szName, 32);
2738	AssertRCReturn(rc, rc);
2739	}
2740	return VINF_SUCCESS;
2741	}
2742
2743
2744	/*********************************************************************************************************************************
2745	* Device Level *
2746	*********************************************************************************************************************************/
2747
2748	/**
2749	* This must be called by the client to handle VM state changes after the client takes care of its device-specific
2750	* tasks for the state change (i.e. reset, suspend, power-off, resume)
2751	*
2752	* @param pDevIns The device instance.
2753	* @param pVirtio Pointer to the shared virtio state.
2754	*/
2755	DECLHIDDEN(void) virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
2756	{
2757	LogFunc(("State changing to %s\n",
2758	virtioCoreGetStateChangeText(enmState)));
2759
2760	switch(enmState)
2761	{
2762	case kvirtIoVmStateChangedReset:
2763	virtioCoreResetAll(pVirtio);
2764	break;
2765	case kvirtIoVmStateChangedSuspend:
2766	break;
2767	case kvirtIoVmStateChangedPowerOff:
2768	break;
2769	case kvirtIoVmStateChangedResume:
2770	for (int uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
2771	{
2772	if ((!pVirtio->fLegacyDriver && pVirtio->aVirtqueues[uVirtq].uEnable)
2773	\| pVirtio->aVirtqueues[uVirtq].GCPhysVirtqDesc)
2774	virtioCoreNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, uVirtq);
2775	}
2776	break;
2777	default:
2778	LogRelFunc(("Bad enum value"));
2779	return;
2780	}
2781	}
2782
2783	/**
2784	* This should be called from PDMDEVREGR3::pfnDestruct.
2785	*
2786	* @param pDevIns The device instance.
2787	* @param pVirtio Pointer to the shared virtio state.
2788	* @param pVirtioCC Pointer to the ring-3 virtio state.
2789	*/
2790	DECLHIDDEN(void) virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2791	{
2792	if (pVirtioCC->pbPrevDevSpecificCfg)
2793	{
2794	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
2795	pVirtioCC->pbPrevDevSpecificCfg = NULL;
2796	}
2797
2798	RT_NOREF(pDevIns, pVirtio);
2799	}
2800
2801
2802	/**
2803	* Setup the Virtio device as a PCI device.
2804	*
2805	* @returns VBox status code.
2806	* @param pDevIns Device instance.
2807	* @param pVirtio Pointer to the shared virtio state. This
2808	* must be the first member in the shared
2809	* device instance data!
2810	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2811	* must be the first member in the ring-3
2812	* device instance data!
2813	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
2814	* @param pcszInstance Device instance name (format-specifier)
2815	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2816	*/
2817	static int virtioR3PciTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2818	const char *pcszInstance, uint16_t cbDevSpecificCfg)
2819	{
2820	/* Set PCI config registers (assume 32-bit mode) */
2821	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
2822	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
2823
2824	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2825	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
2826
2827	if (pPciParams->uDeviceId < DEVICE_PCI_DEVICE_ID_VIRTIO_BASE)
2828	/* Transitional devices MUST have a PCI Revision ID of 0. */
2829	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_TRANS);
2830	else
2831	/* Non-transitional devices SHOULD have a PCI Revision ID of 1 or higher. */
2832	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_V1);
2833
2834	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
2835	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2836	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
2837	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
2838	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
2839	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
2840	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
2841
2842	/* Register PCI device */
2843	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2844	if (RT_FAILURE(rc))
2845	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2846
2847	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2848	AssertRCReturn(rc, rc);
2849
2850	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2851
2852	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2853	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2854	do { \
2855	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2856	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2857	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2858	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2859	} while (0)
2860
2861	PVIRTIO_PCI_CAP_T pCfg;
2862	uint32_t cbRegion = 0;
2863
2864	/*
2865	* Common capability (VirtIO 1.0, section 4.1.4.3)
2866	*/
2867	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2868	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2869	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2870	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2871	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2872	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2873	pCfg->uOffset = RT_ALIGN_32(0, 4); /* Currently 0, but reminder to 32-bit align if changing this */
2874	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2875	cbRegion += pCfg->uLength;
2876	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2877	pVirtioCC->pCommonCfgCap = pCfg;
2878
2879	/*
2880	* Notify capability (VirtIO 1.0, section 4.1.4.4).
2881	*
2882	* The size of the spec-defined subregion described by this VirtIO capability is
2883	* based-on the choice of this implementation to make the notification area of each
2884	* queue equal to queue's ordinal position (e.g. queue selector value). The VirtIO
2885	* specification leaves it up to implementation to define queue notification area layout.
2886	*/
2887	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2888	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2889	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2890	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2891	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2892	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2893	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2894	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2895	pCfg->uLength = VIRTQ_MAX_COUNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2896	cbRegion += pCfg->uLength;
2897	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2898	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2899	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2900
2901	/* ISR capability (VirtIO 1.0, section 4.1.4.5)
2902	*
2903	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. The specification example/diagram
2904	* illustrates this capability as 32-bit field with upper bits 'reserved'. Those depictions
2905	* differ. The spec's wording, not the diagram, is seen to work in practice.
2906	*/
2907	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2908	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2909	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2910	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2911	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2912	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2913	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2914	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2915	pCfg->uLength = sizeof(uint8_t);
2916	cbRegion += pCfg->uLength;
2917	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2918	pVirtioCC->pIsrCap = pCfg;
2919
2920	/* PCI Cfg capability (VirtIO 1.0, section 4.1.4.7)
2921	*
2922	* This capability facilitates early-boot access to this device (BIOS).
2923	* This region isn't page-MMIO mapped. PCI configuration accesses are intercepted,
2924	* wherein uBar, uOffset and uLength are modulated by consumers to locate and read/write
2925	* values in any part of any region. (NOTE: Linux driver doesn't utilize this feature.
2926	* This capability only appears in lspci output on Linux if uLength is non-zero, 4-byte aligned,
2927	* during initialization of linux virtio driver).
2928	*/
2929	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2930	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2931	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2932	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2933	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2934	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2935	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2936	pCfg->uOffset = 0;
2937	pCfg->uLength = 4;
2938	cbRegion += pCfg->uLength;
2939	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2940	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2941
2942	if (pVirtioCC->pbDevSpecificCfg)
2943	{
2944	/* Device-specific config capability (VirtIO 1.0, section 4.1.4.6).
2945	*
2946	* Client defines the device-specific config struct and passes size to virtioCoreR3Init()
2947	* to inform this.
2948	*/
2949	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2950	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2951	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2952	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2953	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2954	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2955	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2956	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2957	pCfg->uLength = cbDevSpecificCfg;
2958	cbRegion += pCfg->uLength;
2959	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2960	pVirtioCC->pDeviceCap = pCfg;
2961	}
2962	else
2963	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2964
2965	if (pVirtio->fMsiSupport)
2966	{
2967	PDMMSIREG aMsiReg;
2968	RT_ZERO(aMsiReg);
2969	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2970	aMsiReg.iMsixNextOffset = 0;
2971	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2972	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2973	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2974	if (RT_FAILURE(rc))
2975	{
2976	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2977	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2978	pVirtio->fMsiSupport = false;
2979	}
2980	else
2981	Log2Func(("Using MSI-X for guest driver notification\n"));
2982	}
2983	else
2984	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2985
2986	/* Set offset to first capability and enable PCI dev capabilities */
2987	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2988	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2989
2990	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2991	if (cbSize <= 0)
2992	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2993
2994	cbSize = RTStrPrintf(pVirtioCC->szPortIoName, sizeof(pVirtioCC->szPortIoName), "%s (legacy)", pcszInstance);
2995	if (cbSize <= 0)
2996	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2997
2998	if (pVirtio->fOfferLegacy)
2999	{
3000	/* As a transitional device that supports legacy VirtIO drivers, this VirtIO device generic implementation presents
3001	* legacy driver interface in I/O space at BAR0. The following maps the common (e.g. device independent)
3002	* dev config area as well as device-specific dev config area (whose size is passed to init function of this VirtIO
3003	* generic device code) for access via Port I/O, since legacy drivers (e.g. pre VirtIO 1.0) don't use MMIO callbacks.
3004	* (See VirtIO 1.1, Section 4.1.4.8).
3005	*/
3006	rc = PDMDevHlpPCIIORegionCreateIo(pDevIns, VIRTIO_REGION_LEGACY_IO, sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T) + cbDevSpecificCfg,
3007	virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/, pVirtioCC->szPortIoName,
3008	NULL /paExtDescs/, &pVirtio->hLegacyIoPorts);
3009	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register legacy config in I/O space at BAR0 */")));
3010	}
3011
3012	/* Note: The Linux driver at drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
3013	* 'unknown' device-specific capability without querying the capability to determine size, so pad w/extra page.
3014	*/
3015	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + VIRTIO_PAGE_SIZE, VIRTIO_PAGE_SIZE),
3016	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
3017	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
3018	pVirtioCC->szMmioName,
3019	&pVirtio->hMmioPciCap);
3020	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
3021	return VINF_SUCCESS;
3022	}
3023
3024
3025	/**
3026	* Initializes the VirtIO device using the VirtIO over MMIO transport mode.
3027	*
3028	* @returns VBox status code.
3029	* @param pDevIns Device instance.
3030	* @param pVirtio Pointer to the shared virtio state. This
3031	* must be the first member in the shared
3032	* device instance data!
3033	* @param pVirtioCC Pointer to the ring-3 virtio state. This
3034	* must be the first member in the ring-3
3035	* device instance data!
3036	* @param pcszInstance Device instance name (format-specifier)
3037	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
3038	* @param GCPhysMmioBase The physical guest address of the start of the MMIO area.
3039	* @param u16Irq The interrupt number to use for the virtio device.
3040	*/
3041	static int virtioR3MmioTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, const char *pcszInstance,
3042	uint16_t cbDevSpecificCfg, RTGCPHYS GCPhysMmioBase, uint16_t u16Irq)
3043	{
3044	pVirtio->uIrqMmio = u16Irq;
3045
3046	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
3047	if (cbSize <= 0)
3048	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
3049
3050	/*
3051	* Register and map the MMIO region.
3052	*/
3053	int rc = PDMDevHlpMmioCreateAndMap(pDevIns, GCPhysMmioBase, RT_ALIGN_32(cbDevSpecificCfg + VIRTIO_MMIO_SIZE, 512),
3054	virtioMmioTransportWrite, virtioMmioTransportRead,
3055	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
3056	pVirtioCC->szMmioName, &pVirtio->hMmioPciCap);
3057	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
3058	return VINF_SUCCESS;
3059	}
3060
3061
3062	/** API Function: See header file */
3063	DECLHIDDEN(int) virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
3064	const char *pcszInstance, uint64_t fDevSpecificFeatures, uint32_t fOfferLegacy,
3065	void *pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
3066	{
3067	/*
3068	* Virtio state must be the first member of shared device instance data,
3069	* otherwise can't get our bearings in PCI config callbacks.
3070	*/
3071	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3072	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
3073
3074	pVirtio->pDevInsR3 = pDevIns;
3075
3076	/*
3077	* Caller must initialize these.
3078	*/
3079	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
3080	AssertReturn(pVirtioCC->pfnVirtqNotified, VERR_INVALID_POINTER);
3081	AssertReturn(VIRTQ_SIZE > 0 && VIRTQ_SIZE <= 32768, VERR_OUT_OF_RANGE); /* VirtIO specification-defined limit */
3082
3083	PCPDMDEVHLPR3 pHlp = pDevIns->pHlpR3;
3084
3085	uint16_t u16Irq = 0;
3086	int rc = pHlp->pfnCFGMQueryU16Def(pDevIns->pCfg, "Irq", &u16Irq, 0);
3087	if (RT_FAILURE(rc))
3088	return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to get the \"Irq\" value"));
3089
3090	RTGCPHYS GCPhysMmioBase = 0;
3091	rc = pHlp->pfnCFGMQueryU64Def(pDevIns->pCfg, "MmioBase", &GCPhysMmioBase, NIL_RTGCPHYS);
3092	if (RT_FAILURE(rc))
3093	return PDMDEV_SET_ERROR(pDevIns, rc,
3094	N_("Configuration error: Failed to get the \"MmioBase\" value"));
3095
3096	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed
3097	* VBox legacy MSI support has not been implemented yet
3098	*/
3099	# ifdef VBOX_WITH_MSI_DEVICES
3100	pVirtio->fMsiSupport = true;
3101	# endif
3102	#endif
3103
3104	/*
3105	* Host features (presented as a buffet for guest to select from)
3106	* include both dev-specific features & reserved dev-independent features (bitmask).
3107	*/
3108	pVirtio->uDeviceType = pPciParams->uDeviceType;
3109	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
3110	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
3111	\| fDevSpecificFeatures;
3112
3113	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy = fOfferLegacy;
3114
3115	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
3116	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
3117	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
3118	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
3119	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
3120
3121	if (GCPhysMmioBase != NIL_RTGCPHYS)
3122	rc = virtioR3MmioTransportInit(pDevIns, pVirtio, pVirtioCC, pcszInstance, cbDevSpecificCfg,
3123	GCPhysMmioBase, u16Irq);
3124	else
3125	rc = virtioR3PciTransportInit(pDevIns, pVirtio, pVirtioCC, pPciParams, pcszInstance, cbDevSpecificCfg);
3126	AssertLogRelRCReturn(rc, rc);
3127
3128	/*
3129	* Statistics.
3130	*/
3131	# ifdef VBOX_WITH_STATISTICS
3132	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3133	"Total number of allocated descriptor chains", "DescChainsAllocated");
3134	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3135	"Total number of freed descriptor chains", "DescChainsFreed");
3136	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3137	"Total number of inbound segments", "DescChainsSegsIn");
3138	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3139	"Total number of outbound segments", "DescChainsSegsOut");
3140	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR3, STAMTYPE_PROFILE, "IO/ReadR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R3");
3141	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR0, STAMTYPE_PROFILE, "IO/ReadR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R0");
3142	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadRC, STAMTYPE_PROFILE, "IO/ReadRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in RC");
3143	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR3, STAMTYPE_PROFILE, "IO/WriteR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R3");
3144	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR0, STAMTYPE_PROFILE, "IO/WriteR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R0");
3145	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteRC, STAMTYPE_PROFILE, "IO/WriteRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in RC");
3146	# endif /* VBOX_WITH_STATISTICS */
3147
3148	return VINF_SUCCESS;
3149	}
3150
3151	#else /* !IN_RING3 */
3152
3153	/**
3154	* Sets up the core ring-0/raw-mode virtio bits.
3155	*
3156	* @returns VBox status code.
3157	* @param pDevIns The device instance.
3158	* @param pVirtio Pointer to the shared virtio state. This must be the first
3159	* member in the shared device instance data!
3160	*/
3161	DECLHIDDEN(int) virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
3162	{
3163	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3164	int rc;
3165	#ifdef FUTURE_OPTIMIZATION
3166	rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
3167	AssertRCReturn(rc, rc);
3168	#endif
3169
3170	if (pVirtio->uIrqMmio != 0)
3171	{
3172	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioTransportWrite, virtioMmioTransportRead, pVirtio);
3173	AssertRCReturn(rc, rc);
3174	}
3175	else
3176	{
3177	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
3178	AssertRCReturn(rc, rc);
3179
3180	if (pVirtio->fOfferLegacy)
3181	{
3182	rc = PDMDevHlpIoPortSetUpContext(pDevIns, pVirtio->hLegacyIoPorts, virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/);
3183	AssertRCReturn(rc, rc);
3184	}
3185	}
3186	return rc;
3187	}
3188
3189	#endif /* !IN_RING3 */
3190

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: vbox/trunk/src/VBox/Devices/VirtIO/VirtioCore.cpp

Download in other formats: