VirtioCore.cpp@ 103914

Last change on this file since 103914 was 102875, checked in by vboxsync, 9 months ago
Devices/VirtIO: More verbose logging
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 132.0 KB

Line
1	/* $Id: VirtioCore.cpp 102875 2024-01-15 12:52:45Z vboxsync $ */
2
3	/** @file
4	* VirtioCore - Virtio Core (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
5	*/
6
7	/*
8	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
9	*
10	* This file is part of VirtualBox base platform packages, as
11	* available from https://www.virtualbox.org.
12	*
13	* This program is free software; you can redistribute it and/or
14	* modify it under the terms of the GNU General Public License
15	* as published by the Free Software Foundation, in version 3 of the
16	* License.
17	*
18	* This program is distributed in the hope that it will be useful, but
19	* WITHOUT ANY WARRANTY; without even the implied warranty of
20	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21	* General Public License for more details.
22	*
23	* You should have received a copy of the GNU General Public License
24	* along with this program; if not, see <https://www.gnu.org/licenses>.
25	*
26	* SPDX-License-Identifier: GPL-3.0-only
27	*/
28
29
30	/*********************************************************************************************************************************
31	* Header Files *
32	*********************************************************************************************************************************/
33	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
34
35	#include <iprt/assert.h>
36	#include <iprt/uuid.h>
37	#include <iprt/mem.h>
38	#include <iprt/sg.h>
39	#include <iprt/assert.h>
40	#include <iprt/string.h>
41	#include <iprt/param.h>
42	#include <iprt/types.h>
43	#include <VBox/log.h>
44	#include <VBox/msi.h>
45	#include <iprt/types.h>
46	#include <VBox/AssertGuest.h>
47	#include <VBox/vmm/pdmdev.h>
48	#include "VirtioCore.h"
49
50
51	/*********************************************************************************************************************************
52	* Defined Constants And Macros *
53	*********************************************************************************************************************************/
54
55	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
56	#define VIRTQNAME(a_pVirtio, a_uVirtq) ((a_pVirtio)->aVirtqueues[(a_uVirtq)].szName)
57
58	#define IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq) \
59	(virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq) == 0)
60
61	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
62	#define WAS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
63
64	/**
65	* These defines are used to track guest virtio-net driver writing driver features accepted flags
66	* in two 32-bit operations (in arbitrary order), and one bit dedicated to ensured 'features complete'
67	* is handled once.
68	*/
69	#define DRIVER_FEATURES_0_WRITTEN 1 /*< fDriverFeatures[0] written by guest virtio-net /
70	#define DRIVER_FEATURES_1_WRITTEN 2 /*< fDriverFeatures[1] written by guest virtio-net /
71	#define DRIVER_FEATURES_0_AND_1_WRITTEN 3 /*< Both 32-bit parts of fDriverFeatures[] written /
72	#define DRIVER_FEATURES_COMPLETE_HANDLED 4 /*< Features negotiation complete handler called /
73
74	/**
75	* This macro returns true if the @a a_offAccess and access length (@a
76	* a_cbAccess) are within the range of the mapped capability struct described by
77	* @a a_LocCapData.
78	*
79	* @param[in] a_offAccess Input: The offset into the MMIO bar of the access.
80	* @param[in] a_cbAccess Input: The access size.
81	* @param[out] a_offsetIntoCap Output: uint32_t variable to return the intra-capability offset into.
82	* @param[in] a_LocCapData Input: The capability location info.
83	*/
84	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offsetIntoCap, a_LocCapData) \
85	( ((a_offsetIntoCap) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
86	&& (a_offsetIntoCap) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
87
88
89	/*********************************************************************************************************************************
90	* Structures and Typedefs *
91	*********************************************************************************************************************************/
92
93	/** @name virtq related flags
94	* @{ */
95	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
96	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
97	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
98
99	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
100	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
101	/** @} */
102
103	/**
104	* virtq-related structs
105	* (struct names follow VirtIO 1.0 spec, field names use VBox styled naming, w/respective spec'd name in comments)
106	*/
107	typedef struct virtq_desc
108	{
109	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
110	uint32_t cb; /*< len Buffer length /
111	uint16_t fFlags; /*< flags Buffer specific flags /
112	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
113	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
114
115	typedef struct virtq_avail
116	{
117	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
118	uint16_t uIdx; /*< idx Index of next free ring slot /
119	RT_FLEXIBLE_ARRAY_EXTENSION
120	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
121	//uint16_t uUsedEventIdx; /*< used_event (if VIRTQ_USED_F_EVENT_IDX) /
122	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
123
124	typedef struct virtq_used_elem
125	{
126	uint32_t uDescIdx; /*< idx Start of used desc chain /
127	uint32_t cbElem; /*< len Total len of used desc chain /
128	} VIRTQ_USED_ELEM_T;
129
130	typedef struct virt_used
131	{
132	uint16_t fFlags; /*< flags used ring host-to-guest flags /
133	uint16_t uIdx; /*< idx Index of next ring slot /
134	RT_FLEXIBLE_ARRAY_EXTENSION
135	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
136	//uint16_t uAvailEventIdx; /*< avail_event if (VIRTQ_USED_F_EVENT_IDX) /
137	} VIRTQ_USED_T, *PVIRTQ_USED_T;
138
139	DECLHIDDEN(const char *) virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
140	{
141	switch (enmState)
142	{
143	case kvirtIoVmStateChangedReset: return "VM RESET";
144	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
145	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
146	case kvirtIoVmStateChangedResume: return "VM RESUME";
147	default: return "<BAD ENUM>";
148	}
149	}
150
151	/* Internal Functions */
152
153	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq);
154	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
155
156	#ifdef IN_RING3
157	# ifdef LOG_ENABLED
158	DECLINLINE(uint16_t) virtioCoreR3CountPendingBufs(uint16_t uRingIdx, uint16_t uShadowIdx, uint16_t uQueueSize)
159	{
160	if (uShadowIdx == uRingIdx)
161	return 0;
162	else
163	if (uShadowIdx > uRingIdx)
164	return uShadowIdx - uRingIdx;
165	return uQueueSize - (uRingIdx - uShadowIdx);
166	}
167	# endif
168	#endif
169	/** @name Internal queue operations
170	* @{ */
171
172	/**
173	* Accessor for virtq descriptor
174	*/
175	#ifdef IN_RING3
176	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
177	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
178	{
179	/*
180	* Shut up assertion for legacy virtio-net driver in FreeBSD up to 12.3 (see virtioCoreR3VirtqUsedBufPut()
181	* for more information).
182	*/
183	AssertMsg( IS_DRIVER_OK(pVirtio)
184	\|\| ( pVirtio->fLegacyDriver
185	&& pVirtq->GCPhysVirtqDesc),
186	("Called with guest driver not ready\n"));
187	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
188
189	virtioCoreGCPhysRead(pVirtio, pDevIns,
190	pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * (idxDesc % cVirtqItems),
191	pDesc, sizeof(VIRTQ_DESC_T));
192	}
193	#endif
194
195	/**
196	* Accessors for virtq avail ring
197	*/
198	#ifdef IN_RING3
199	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t availIdx)
200	{
201	uint16_t uDescIdx;
202
203	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
204	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
205	virtioCoreGCPhysRead(pVirtio, pDevIns,
206	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cVirtqItems]),
207	&uDescIdx, sizeof(uDescIdx));
208	return uDescIdx;
209	}
210
211	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
212	{
213	uint16_t uUsedEventIdx;
214	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
215	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
216	virtioCoreGCPhysRead(pVirtio, pDevIns,
217	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]),
218	&uUsedEventIdx, sizeof(uUsedEventIdx));
219	return uUsedEventIdx;
220	}
221	#endif
222
223	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
224	{
225	uint16_t uIdx = 0;
226	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
227	virtioCoreGCPhysRead(pVirtio, pDevIns,
228	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
229	&uIdx, sizeof(uIdx));
230	return uIdx;
231	}
232
233	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
234	{
235	uint16_t fFlags = 0;
236	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
237	virtioCoreGCPhysRead(pVirtio, pDevIns,
238	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
239	&fFlags, sizeof(fFlags));
240	return fFlags;
241	}
242
243	/** @} */
244
245	/** @name Accessors for virtq used ring
246	* @{
247	*/
248
249	#ifdef IN_RING3
250	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
251	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
252	{
253	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
254	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
255	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
256	virtioCoreGCPhysWrite(pVirtio, pDevIns,
257	pVirtq->GCPhysVirtqUsed
258	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cVirtqItems]),
259	&elem, sizeof(elem));
260	}
261
262	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t fFlags)
263	{
264	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
265	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
266	virtioCoreGCPhysWrite(pVirtio, pDevIns,
267	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
268	&fFlags, sizeof(fFlags));
269	}
270	#endif
271
272	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t uIdx)
273	{
274	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
275	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
276	virtioCoreGCPhysWrite(pVirtio, pDevIns,
277	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
278	&uIdx, sizeof(uIdx));
279	}
280
281	#ifdef IN_RING3
282	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
283	{
284	uint16_t uIdx = 0;
285	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
286	virtioCoreGCPhysRead(pVirtio, pDevIns,
287	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
288	&uIdx, sizeof(uIdx));
289	return uIdx;
290	}
291
292	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
293	{
294	uint16_t fFlags = 0;
295	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
296	virtioCoreGCPhysRead(pVirtio, pDevIns,
297	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
298	&fFlags, sizeof(fFlags));
299	return fFlags;
300	}
301
302	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t uAvailEventIdx)
303	{
304	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
305	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
306	virtioCoreGCPhysWrite(pVirtio, pDevIns,
307	pVirtq->GCPhysVirtqUsed
308	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
309	&uAvailEventIdx, sizeof(uAvailEventIdx));
310	}
311	#endif
312	/** @} */
313
314
315	DECLINLINE(uint16_t) virtioCoreVirtqAvailCnt(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
316	{
317	uint16_t uIdxActual = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
318	uint16_t uIdxShadow = pVirtq->uAvailIdxShadow;
319	uint16_t uIdxDelta;
320
321	if (uIdxActual < uIdxShadow)
322	uIdxDelta = (uIdxActual + pVirtq->uQueueSize) - uIdxShadow;
323	else
324	uIdxDelta = uIdxActual - uIdxShadow;
325
326	return uIdxDelta;
327	}
328	/**
329	* Get count of new (e.g. pending) elements in available ring.
330	*
331	* @param pDevIns The device instance.
332	* @param pVirtio Pointer to the shared virtio state.
333	* @param uVirtq Virtq number
334	*
335	* @returns how many entries have been added to ring as a delta of the consumer's
336	* avail index and the queue's guest-side current avail index.
337	*/
338	DECLHIDDEN(uint16_t) virtioCoreVirtqAvailBufCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
339	{
340	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues), ("uVirtq out of range"), 0);
341	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
342
343	if (!IS_DRIVER_OK(pVirtio))
344	{
345	LogRelFunc(("Driver not ready\n"));
346	return 0;
347	}
348	if (!pVirtio->fLegacyDriver && !pVirtq->uEnable)
349	{
350	LogRelFunc(("virtq: %s not enabled\n", VIRTQNAME(pVirtio, uVirtq)));
351	return 0;
352	}
353	return virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq);
354	}
355
356	#ifdef IN_RING3
357
358	static void virtioCoreR3FeatureDump(VIRTIOCORE pVirtio, PCDBGFINFOHLP pHlp, const VIRTIO_FEATURES_LIST s_aFeatures, int cFeatures, int fBanner)
359	{
360	#define MAXLINE 80
361	/* Display as a single buf to prevent interceding log messages */
362	uint16_t cbBuf = cFeatures * 132;
363	char pszBuf = (char )RTMemAllocZ(cbBuf);
364	Assert(pszBuf);
365	char *cp = pszBuf;
366	for (int i = 0; i < cFeatures; ++i)
367	{
368	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
369	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
370	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
371	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
372	}
373	if (pHlp) {
374	if (fBanner)
375	pHlp->pfnPrintf(pHlp, "VirtIO Features Configuration\n\n"
376	" Offered Accepted Feature Description\n"
377	" ------- -------- ------- -----------\n");
378	pHlp->pfnPrintf(pHlp, "%s\n", pszBuf);
379	}
380	#ifdef LOG_ENABLED
381	else
382	{
383	if (fBanner)
384	Log(("VirtIO Features Configuration\n\n"
385	" Offered Accepted Feature Description\n"
386	" ------- -------- ------- -----------\n"));
387	Log(("%s\n", pszBuf));
388	}
389	#endif
390	RTMemFree(pszBuf);
391	}
392
393	/** API Function: See header file*/
394	DECLHIDDEN(void) virtioCorePrintDeviceFeatures(VIRTIOCORE *pVirtio, PCDBGFINFOHLP pHlp,
395	const VIRTIO_FEATURES_LIST *s_aDevSpecificFeatures, int cFeatures) {
396	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aCoreFeatures, RT_ELEMENTS(s_aCoreFeatures), 1 /fBanner /);
397	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aDevSpecificFeatures, cFeatures, 0 /fBanner /);
398	}
399
400	#endif
401
402	#ifdef LOG_ENABLED
403
404	/** API Function: See header file */
405	DECLHIDDEN(void) virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
406	{
407	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
408	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
409	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
410	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
411	if (pszTitle)
412	{
413	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
414	ADJCURSOR(cbPrint);
415	}
416	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
417	{
418	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
419	ADJCURSOR(cbPrint);
420	for (uint8_t col = 0; col < 16; col++)
421	{
422	uint32_t idx = row * 16 + col;
423	if (idx >= cb)
424	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
425	else
426	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", pv[idx], (col + 1) % 8 ? "" : " ");
427	ADJCURSOR(cbPrint);
428	}
429	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
430	{
431	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.'));
432	ADJCURSOR(cbPrint);
433	}
434	*pszOut++ = '\n';
435	--cbRemain;
436	}
437	Log(("%s\n", pszBuf));
438	RTMemFree(pszBuf);
439	RT_NOREF2(uBase, pv);
440	#undef ADJCURSOR
441	}
442
443	/* API FUnction: See header file */
444	DECLHIDDEN(void) virtioCoreGCPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint16_t cb, uint32_t uBase, const char *pszTitle)
445	{
446	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
447	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
448	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
449	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
450	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
451	if (pszTitle)
452	{
453	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
454	ADJCURSOR(cbPrint);
455	}
456	for (uint16_t row = 0; row < (uint16_t)RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
457	{
458	uint8_t c;
459	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
460	ADJCURSOR(cbPrint);
461	for (uint8_t col = 0; col < 16; col++)
462	{
463	uint32_t idx = row * 16 + col;
464	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
465	if (idx >= cb)
466	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
467	else
468	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", c, (col + 1) % 8 ? "" : " ");
469	ADJCURSOR(cbPrint);
470	}
471	for (uint16_t idx = row * 16; idx < row * 16 + 16; idx++)
472	{
473	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
474	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.'));
475	ADJCURSOR(cbPrint);
476	}
477	*pszOut++ = '\n';
478	--cbRemain;
479	}
480	Log(("%s\n", pszBuf));
481	RTMemFree(pszBuf);
482	RT_NOREF(uBase);
483	#undef ADJCURSOR
484	}
485
486
487	/** API function: See header file */
488	DECLHIDDEN(void) virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
489	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
490	int fHasIndex, uint32_t idx)
491	{
492	if (LogIs6Enabled())
493	{
494	char szIdx[16];
495	if (fHasIndex)
496	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
497	else
498	szIdx[0] = '\0';
499
500	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
501	{
502	char szDepiction[64];
503	size_t cchDepiction;
504	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
505	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
506	pszMember, szIdx, uOffset, uOffset + cb - 1);
507	else
508	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
509
510	/* padding */
511	if (cchDepiction < 30)
512	szDepiction[cchDepiction++] = ' ';
513	while (cchDepiction < 30)
514	szDepiction[cchDepiction++] = '.';
515	szDepiction[cchDepiction] = '\0';
516
517	RTUINT64U uValue;
518	uValue.u = 0;
519	memcpy(uValue.au8, pv, cb);
520	Log6(("%-23s: Guest %s %s %#0*RX64\n",
521	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
522	}
523	else /* odd number or oversized access, ... log inline hex-dump style */
524	{
525	Log6(("%-23s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
526	pszFunc, fWrite ? "wrote" : "read ", pszMember,
527	szIdx, uOffset, uOffset + cb, cb, pv));
528	}
529	}
530	RT_NOREF2(fWrite, pszFunc);
531	}
532
533	/**
534	* Log MMIO-mapped Virtio fDeviceStatus register bitmask, naming the bits
535	*/
536	DECLINLINE(void) virtioCoreFormatDeviceStatus(uint8_t bStatus, char *pszBuf, size_t uSize)
537	{
538	# define ADJCURSOR(len) { cp += len; uSize -= len; sep = (char *)" \| "; }
539	memset(pszBuf, 0, uSize);
540	char cp = pszBuf, sep = (char *)"";
541	size_t len;
542	if (bStatus == 0)
543	RTStrPrintf(cp, uSize, "RESET");
544	else
545	{
546	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
547	{
548	len = RTStrPrintf(cp, uSize, "ACKNOWLEDGE");
549	ADJCURSOR(len);
550	}
551	if (bStatus & VIRTIO_STATUS_DRIVER)
552	{
553	len = RTStrPrintf(cp, uSize, "%sDRIVER", sep);
554	ADJCURSOR(len);
555	}
556	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
557	{
558	len = RTStrPrintf(cp, uSize, "%sFEATURES_OK", sep);
559	ADJCURSOR(len);
560	}
561	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
562	{
563	len = RTStrPrintf(cp, uSize, "%sDRIVER_OK", sep);
564	ADJCURSOR(len);
565	}
566	if (bStatus & VIRTIO_STATUS_FAILED)
567	{
568	len = RTStrPrintf(cp, uSize, "%sFAILED", sep);
569	ADJCURSOR(len);
570	}
571	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
572	RTStrPrintf(cp, uSize, "%sNEEDS_RESET", sep);
573	}
574	# undef ADJCURSOR
575	}
576
577	#endif /* LOG_ENABLED */
578
579	/** API function: See header file */
580	DECLHIDDEN(int) virtioCoreIsLegacyMode(PVIRTIOCORE pVirtio)
581	{
582	return pVirtio->fLegacyDriver;
583	}
584
585	#ifdef IN_RING3
586
587	DECLHIDDEN(int) virtioCoreR3VirtqAttach(PVIRTIOCORE pVirtio, uint16_t uVirtq, const char *pcszName)
588	{
589	LogFunc(("Attaching %s to VirtIO core\n", pcszName));
590	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
591	pVirtq->uVirtq = uVirtq;
592	pVirtq->fUsedRingEvent = false;
593	pVirtq->fAttached = true;
594	RTStrCopy(pVirtq->szName, sizeof(pVirtq->szName), pcszName);
595	return VINF_SUCCESS;
596	}
597
598	DECLHIDDEN(int) virtioCoreR3VirtqDetach(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
599	{
600	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
601	pVirtq->uVirtq = 0;
602	pVirtq->uAvailIdxShadow = 0;
603	pVirtq->uUsedIdxShadow = 0;
604	pVirtq->fUsedRingEvent = false;
605	pVirtq->fAttached = false;
606	memset(pVirtq->szName, 0, sizeof(pVirtq->szName));
607	return VINF_SUCCESS;
608	}
609
610	DECLHIDDEN(bool) virtioCoreR3VirtqIsAttached(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
611	{
612	return pVirtio->aVirtqueues[uVirtqNbr].fAttached;
613	}
614
615	DECLHIDDEN(bool) virtioCoreR3VirtqIsEnabled(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
616	{
617	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
618	return (bool)pVirtq->uEnable && pVirtq->GCPhysVirtqDesc;
619	}
620
621	/** API Fuunction: See header file */
622	DECLHIDDEN(void) virtioCoreR3VirtqInfo(PPDMDEVINS pDevIns, PCDBGFINFOHLP pHlp, const char *pszArgs, int uVirtq)
623	{
624	RT_NOREF(pszArgs);
625	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
626	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
627
628	/** @todo add ability to dump physical contents described by any descriptor (using existing VirtIO core API function) */
629	// bool fDump = pszArgs && (pszArgs == 'd' \|\| pszArgs == 'D'); /* "dump" (avail phys descriptor)"
630
631	uint16_t uAvailIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
632	uint16_t uAvailIdxShadow = pVirtq->uAvailIdxShadow;
633
634	uint16_t uUsedIdx = virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq);
635	uint16_t uUsedIdxShadow = pVirtq->uUsedIdxShadow;
636
637	VIRTQBUF_T VirtqBuf;
638	PVIRTQBUF pVirtqBuf = &VirtqBuf;
639	bool fEmpty = IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq);
640
641	LogFunc(("%s, empty = %s\n", pVirtq->szName, fEmpty ? "true" : "false"));
642
643	int cSendSegs = 0, cReturnSegs = 0;
644	if (!fEmpty)
645	{
646	virtioCoreR3VirtqAvailBufPeek(pDevIns, pVirtio, uVirtq, pVirtqBuf);
647	cSendSegs = pVirtqBuf->pSgPhysSend ? pVirtqBuf->pSgPhysSend->cSegs : 0;
648	cReturnSegs = pVirtqBuf->pSgPhysReturn ? pVirtqBuf->pSgPhysReturn->cSegs : 0;
649	}
650
651	bool fAvailNoInterrupt = virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT;
652	bool fUsedNoNotify = virtioReadUsedRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_USED_F_NO_NOTIFY;
653
654	pHlp->pfnPrintf(pHlp, " queue enabled: ........... %s\n", pVirtq->uEnable ? "true" : "false");
655	pHlp->pfnPrintf(pHlp, " size: .................... %d\n", pVirtq->uQueueSize);
656	pHlp->pfnPrintf(pHlp, " notify offset: ........... %d\n", pVirtq->uNotifyOffset);
657	if (pVirtio->fMsiSupport)
658	pHlp->pfnPrintf(pHlp, " MSIX vector: ....... %4.4x\n", pVirtq->uMsixVector);
659	pHlp->pfnPrintf(pHlp, "\n");
660	pHlp->pfnPrintf(pHlp, " avail ring (%d entries):\n", uAvailIdx - uAvailIdxShadow);
661	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uAvailIdx);
662	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uAvailIdxShadow);
663	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fAvailNoInterrupt ? "NO_INTERRUPT" : "");
664	pHlp->pfnPrintf(pHlp, "\n");
665	pHlp->pfnPrintf(pHlp, " used ring (%d entries):\n", uUsedIdx - uUsedIdxShadow);
666	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uUsedIdx);
667	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uUsedIdxShadow);
668	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fUsedNoNotify ? "NO_NOTIFY" : "");
669	pHlp->pfnPrintf(pHlp, "\n");
670	if (!fEmpty)
671	{
672	pHlp->pfnPrintf(pHlp, " desc chain:\n");
673	pHlp->pfnPrintf(pHlp, " head idx: ............. %d\n", uUsedIdx);
674	pHlp->pfnPrintf(pHlp, " segs: ................. %d\n", cSendSegs + cReturnSegs);
675	pHlp->pfnPrintf(pHlp, " refCnt ................ %d\n", pVirtqBuf->cRefs);
676	pHlp->pfnPrintf(pHlp, "\n");
677	pHlp->pfnPrintf(pHlp, " host-to-guest (%d bytes):\n", pVirtqBuf->cbPhysSend);
678	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cSendSegs);
679	if (cSendSegs)
680	{
681	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysSend->idxSeg);
682	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysSend->cbSegLeft);
683	}
684	pHlp->pfnPrintf(pHlp, "\n");
685	pHlp->pfnPrintf(pHlp, " guest-to-host (%d bytes)\n", pVirtqBuf->cbPhysReturn);
686	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cReturnSegs);
687	if (cReturnSegs)
688	{
689	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysReturn->idxSeg);
690	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysReturn->cbSegLeft);
691	}
692	} else
693	pHlp->pfnPrintf(pHlp, " No desc chains available\n");
694	pHlp->pfnPrintf(pHlp, "\n");
695	}
696
697
698	/** API Function: See header file */
699	DECLHIDDEN(PVIRTQBUF) virtioCoreR3VirtqBufAlloc(void)
700	{
701	PVIRTQBUF pVirtqBuf = (PVIRTQBUF)RTMemAllocZ(sizeof(VIRTQBUF_T));
702	AssertReturn(pVirtqBuf, NULL);
703	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
704	pVirtqBuf->cRefs = 1;
705	return pVirtqBuf;
706	}
707
708
709	/** API Function: See header file */
710	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRetain(PVIRTQBUF pVirtqBuf)
711	{
712	AssertReturn(pVirtqBuf, UINT32_MAX);
713	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, UINT32_MAX);
714	uint32_t cRefs = ASMAtomicIncU32(&pVirtqBuf->cRefs);
715	Assert(cRefs > 1);
716	Assert(cRefs < 16);
717	return cRefs;
718	}
719
720	/** API Function: See header file */
721	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRelease(PVIRTIOCORE pVirtio, PVIRTQBUF pVirtqBuf)
722	{
723	if (!pVirtqBuf)
724	return 0;
725	AssertReturn(pVirtqBuf, 0);
726	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, 0);
727	uint32_t cRefs = ASMAtomicDecU32(&pVirtqBuf->cRefs);
728	Assert(cRefs < 16);
729	if (cRefs == 0)
730	{
731	pVirtqBuf->u32Magic = ~VIRTQBUF_MAGIC;
732	RTMemFree(pVirtqBuf);
733	#ifdef VBOX_WITH_STATISTICS
734	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
735	#endif
736	}
737	RT_NOREF(pVirtio);
738	return cRefs;
739	}
740
741	/** API Function: See header file */
742	DECLHIDDEN(void) virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
743	{
744	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
745	}
746
747
748	/** API Function: See header file */
749	DECLHIDDEN(void) virtioCoreVirtqEnableNotify(PVIRTIOCORE pVirtio, uint16_t uVirtq, bool fEnable)
750	{
751	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
752	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
753
754	if (IS_DRIVER_OK(pVirtio))
755	{
756	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq);
757
758	if (fEnable)
759	fFlags &= ~VIRTQ_USED_F_NO_NOTIFY;
760	else
761	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
762
763	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq, fFlags);
764	}
765	}
766
767	/** API function: See Header file */
768	DECLHIDDEN(void) virtioCoreResetAll(PVIRTIOCORE pVirtio)
769	{
770	LogFunc(("\n"));
771	pVirtio->fDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
772	if (IS_DRIVER_OK(pVirtio))
773	{
774	if (!pVirtio->fLegacyDriver)
775	pVirtio->fGenUpdatePending = true;
776	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
777	}
778	}
779
780	/** API function: See Header file */
781	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufPeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PVIRTQBUF pVirtqBuf)
782	{
783	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, pVirtqBuf, false);
784	}
785
786
787	/** API function: See Header file */
788	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufNext(PVIRTIOCORE pVirtio, uint16_t uVirtq)
789	{
790	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
791	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
792
793	if (!pVirtio->fLegacyDriver)
794	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
795	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
796
797	if (IS_VIRTQ_EMPTY(pVirtio->pDevInsR3, pVirtio, pVirtq))
798	return VERR_NOT_AVAILABLE;
799
800	Log6Func(("%s avail shadow idx: %u\n", pVirtq->szName, pVirtq->uAvailIdxShadow));
801	pVirtq->uAvailIdxShadow++;
802
803	return VINF_SUCCESS;
804	}
805
806	/** API Function: See header file */
807	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
808	uint16_t uHeadIdx, PVIRTQBUF pVirtqBuf)
809	{
810	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues),
811	("uVirtq out of range"), VERR_INVALID_PARAMETER);
812
813	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
814
815	if (!pVirtio->fLegacyDriver)
816	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
817	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
818
819	uint16_t uDescIdx = uHeadIdx;
820
821	Log6Func(("%s DESC CHAIN: (head idx = %u)\n", pVirtio->aVirtqueues[uVirtq].szName, uHeadIdx));
822
823	/*
824	* Allocate and initialize the descriptor chain structure.
825	*/
826	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
827	pVirtqBuf->cRefs = 1;
828	pVirtqBuf->uHeadIdx = uHeadIdx;
829	pVirtqBuf->uVirtq = uVirtq;
830
831	/*
832	* Gather segments.
833	*/
834	VIRTQ_DESC_T desc;
835
836	uint32_t cbIn = 0;
837	uint32_t cbOut = 0;
838	uint32_t cSegsIn = 0;
839	uint32_t cSegsOut = 0;
840
841	PVIRTIOSGSEG paSegsIn = pVirtqBuf->aSegsIn;
842	PVIRTIOSGSEG paSegsOut = pVirtqBuf->aSegsOut;
843
844	do
845	{
846	PVIRTIOSGSEG pSeg;
847	/*
848	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
849	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
850	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
851	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
852	*/
853	if (cSegsIn + cSegsOut >= pVirtq->uQueueSize)
854	{
855	static volatile uint32_t s_cMessages = 0;
856	static volatile uint32_t s_cThreshold = 1;
857	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
858	{
859	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop (cSegsIn=%u cSegsOut=%u uQueueSize=%u).\n",
860	cSegsIn, cSegsOut, pVirtq->uQueueSize));
861	if (ASMAtomicReadU32(&s_cMessages) != 1)
862	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
863	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
864	}
865	break;
866	}
867	RT_UNTRUSTED_VALIDATED_FENCE();
868
869	virtioReadDesc(pDevIns, pVirtio, pVirtq, uDescIdx, &desc);
870
871	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
872	{
873	Log6Func(("%s IN idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
874	cbIn += desc.cb;
875	pSeg = &paSegsIn[cSegsIn++];
876	}
877	else
878	{
879	Log6Func(("%s OUT desc_idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
880	cbOut += desc.cb;
881	pSeg = &paSegsOut[cSegsOut++];
882	#ifdef DEEP_DEBUG
883	if (LogIs11Enabled())
884	{
885	virtioCoreGCPhysHexDump(pDevIns, desc.GCPhysBuf, desc.cb, 0, NULL);
886	Log(("\n"));
887	}
888	#endif
889	}
890	pSeg->GCPhys = desc.GCPhysBuf;
891	pSeg->cbSeg = desc.cb;
892	uDescIdx = desc.uDescIdxNext;
893	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
894
895	/*
896	* Add segments to the descriptor chain structure.
897	*/
898	if (cSegsIn)
899	{
900	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufIn, paSegsIn, cSegsIn);
901	pVirtqBuf->pSgPhysReturn = &pVirtqBuf->SgBufIn;
902	pVirtqBuf->cbPhysReturn = cbIn;
903	#ifdef VBOX_WITH_STATISTICS
904	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
905	#endif
906	}
907
908	if (cSegsOut)
909	{
910	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufOut, paSegsOut, cSegsOut);
911	pVirtqBuf->pSgPhysSend = &pVirtqBuf->SgBufOut;
912	pVirtqBuf->cbPhysSend = cbOut;
913	#ifdef VBOX_WITH_STATISTICS
914	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
915	#endif
916	}
917
918	#ifdef VBOX_WITH_STATISTICS
919	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
920	#endif
921	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n",
922	pVirtq->szName, cSegsOut, cbOut, cSegsIn, cbIn));
923
924	return VINF_SUCCESS;
925	}
926
927	/** API function: See Header file */
928	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
929	PVIRTQBUF pVirtqBuf, bool fRemove)
930	{
931	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
932	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
933
934	if (IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq))
935	return VERR_NOT_AVAILABLE;
936
937	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, pVirtq->uAvailIdxShadow);
938
939	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
940	virtioWriteUsedAvailEvent(pDevIns,pVirtio, pVirtq, pVirtq->uAvailIdxShadow + 1);
941
942	if (fRemove)
943	pVirtq->uAvailIdxShadow++;
944
945	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, uHeadIdx, pVirtqBuf);
946	}
947
948	/** API function: See Header file */
949	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PRTSGBUF pSgVirtReturn,
950	PVIRTQBUF pVirtqBuf, bool fFence)
951	{
952	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
953	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
954
955	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
956
957	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
958	Assert(pVirtqBuf->cRefs > 0);
959
960	/*
961	* Workaround for a bug in FreeBSD's virtio-net driver up until 12.3 which supports only the legacy style devive.
962	* When the device is re-initialized from the driver it violates the spec and posts commands to the control queue
963	* before setting the DRIVER_OK flag, breaking the following check and rendering the device non-functional.
964	* The queues are properly set up at this stage however so no real harm is done and we can safely continue here,
965	* for the legacy device only of course after making sure the queue is properly set up.
966	*/
967	AssertMsgReturn( IS_DRIVER_OK(pVirtio)
968	\|\| ( pVirtio->fLegacyDriver
969	&& pVirtq->GCPhysVirtqDesc),
970	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
971
972	Log6Func((" Copying device data to %s, [desc:%u -> used ring:%u]\n",
973	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx, pVirtq->uUsedIdxShadow));
974
975	/* Copy s/g buf (virtual memory) to guest phys mem (VirtIO "IN" direction). */
976
977	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
978
979	if (pSgVirtReturn)
980	{
981	size_t cbTarget = virtioCoreGCPhysChainCalcBufSize(pSgPhysReturn);
982	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
983	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
984	virtioCoreGCPhysChainReset(pSgPhysReturn);
985	while (cbRemain)
986	{
987	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
988	AssertReturn(cbCopy > 0, VERR_INVALID_PARAMETER);
989	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
990	RTSgBufAdvance(pSgVirtReturn, cbCopy);
991	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
992	cbRemain -= cbCopy;
993	}
994
995	if (fFence)
996	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
997
998	Assert(!(cbCopy >> 32));
999	}
1000
1001	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1002	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1003	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1004	pVirtq->fUsedRingEvent = true;
1005
1006	/*
1007	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1008	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1009	*
1010	* @todo r=aeichner: The increment of the shadow index is not atomic but this code can be called
1011	* concurrently!!
1012	*/
1013	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbTotal);
1014
1015	#ifdef LOG_ENABLED
1016	if (LogIs6Enabled() && pSgVirtReturn)
1017	{
1018
1019	LogFunc((" ... %d segs, %zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1020	pSgVirtReturn->cSegs, cbTotal - cbRemain, pVirtqBuf->cbPhysReturn,
1021	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1022	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - (cbTotal - cbRemain)),
1023	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn) ));
1024
1025	uint16_t uPending = virtioCoreR3CountPendingBufs(
1026	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1027	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1028
1029	LogFunc((" %u used buf%s not synced in %s\n", uPending, uPending == 1 ? "" : "s ",
1030	VIRTQNAME(pVirtio, uVirtq)));
1031	}
1032	#endif
1033	return VINF_SUCCESS;
1034	}
1035
1036	/** API function: See Header file */
1037	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1038	size_t cb, void const *pv, PVIRTQBUF pVirtqBuf, size_t cbEnqueue, bool fFence)
1039	{
1040	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1041	Assert(pv);
1042
1043	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1044	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1045
1046	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1047	Assert(pVirtqBuf->cRefs > 0);
1048
1049	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1050
1051	Log6Func((" Copying device data to %s, [desc chain head idx:%u]\n",
1052	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx));
1053	/*
1054	* Convert virtual memory simple buffer to guest physical memory (VirtIO descriptor chain)
1055	*/
1056	uint8_t pvBuf = (uint8_t )pv;
1057	size_t cbRemain = cb, cbCopy = 0;
1058	while (cbRemain)
1059	{
1060	cbCopy = RT_MIN(pSgPhysReturn->cbSegLeft, cbRemain);
1061	Assert(cbCopy > 0);
1062	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pvBuf, cbCopy);
1063	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1064	pvBuf += cbCopy;
1065	cbRemain -= cbCopy;
1066	}
1067	LogFunc((" ...%zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1068	cb , pVirtqBuf->cbPhysReturn,
1069	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1070	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - cb),
1071	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)));
1072
1073	if (cbEnqueue)
1074	{
1075	if (fFence)
1076	{
1077	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1078	Assert(!(cbCopy >> 32));
1079	}
1080	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1081	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1082	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1083	pVirtq->fUsedRingEvent = true;
1084	/*
1085	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1086	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1087	*/
1088	Log6Func((" Enqueue desc chain head idx %u to %s used ring @ %u\n", pVirtqBuf->uHeadIdx,
1089	VIRTQNAME(pVirtio, uVirtq), pVirtq->uUsedIdxShadow));
1090
1091	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbEnqueue);
1092
1093	#ifdef LOG_ENABLED
1094	if (LogIs6Enabled())
1095	{
1096	uint16_t uPending = virtioCoreR3CountPendingBufs(
1097	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1098	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1099
1100	LogFunc((" %u used buf%s not synced in %s\n",
1101	uPending, uPending == 1 ? "" : "s ", VIRTQNAME(pVirtio, uVirtq)));
1102	}
1103	#endif
1104	} /* fEnqueue */
1105
1106	return VINF_SUCCESS;
1107	}
1108
1109
1110	#endif /* IN_RING3 */
1111
1112	/** API function: See Header file */
1113	DECLHIDDEN(int) virtioCoreVirtqUsedRingSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1114	{
1115	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1116	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1117
1118	if (!pVirtio->fLegacyDriver)
1119	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
1120	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1121
1122	Log6Func((" Sync %s used ring (%u -> idx)\n",
1123	pVirtq->szName, pVirtq->uUsedIdxShadow));
1124
1125	virtioWriteUsedRingIdx(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow);
1126	virtioCoreNotifyGuestDriver(pDevIns, pVirtio, uVirtq);
1127
1128	return VINF_SUCCESS;
1129	}
1130
1131	/**
1132	* This is called from the MMIO callback code when the guest does an MMIO access to the
1133	* mapped queue notification capability area corresponding to a particular queue, to notify
1134	* the queue handler of available data in the avail ring of the queue (VirtIO 1.0, 4.1.4.4.1)
1135	*
1136	* @param pDevIns The device instance.
1137	* @param pVirtio Pointer to the shared virtio state.
1138	* @param uVirtq Virtq to check for guest interrupt handling preference
1139	* @param uNotifyIdx Notification index
1140	*/
1141	static void virtioCoreVirtqNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, uint16_t uNotifyIdx)
1142	{
1143	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1144
1145	/* VirtIO 1.0, section 4.1.5.2 implies uVirtq and uNotifyIdx should match. Disregarding any of
1146	* these notifications (if those indicies disagree) may break device/driver synchronization,
1147	* causing eternal throughput starvation, yet there's no specified way to disambiguate
1148	* which queue to wake-up in any awkward situation where the two parameters differ.
1149	*/
1150	AssertMsg(uNotifyIdx == uVirtq,
1151	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1152	uVirtq, uNotifyIdx));
1153	RT_NOREF(uNotifyIdx);
1154
1155	AssertReturnVoid(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1156	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1157
1158	Log6Func(("%s: (desc chains: %u)\n", *pVirtq->szName ? pVirtq->szName : "?UNAMED QUEUE?",
1159	virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq)));
1160
1161	/* Inform client */
1162	pVirtioCC->pfnVirtqNotified(pDevIns, pVirtio, uVirtq);
1163	RT_NOREF2(pVirtio, pVirtq);
1164	}
1165
1166	/**
1167	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1168	* the specified virtq, depending on the interrupt configuration of the device
1169	* and depending on negotiated and realtime constraints flagged by the guest driver.
1170	*
1171	* See VirtIO 1.0 specification (section 2.4.7).
1172	*
1173	* @param pDevIns The device instance.
1174	* @param pVirtio Pointer to the shared virtio state.
1175	* @param uVirtq Virtq to check for guest interrupt handling preference
1176	*/
1177	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1178	{
1179	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1180	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1181
1182	if (!IS_DRIVER_OK(pVirtio))
1183	{
1184	LogFunc(("Guest driver not in ready state.\n"));
1185	return;
1186	}
1187
1188	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1189	{
1190	if (pVirtq->fUsedRingEvent)
1191	{
1192	#ifdef IN_RING3
1193	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1194	pVirtq->szName, (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq)));
1195	#endif
1196	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1197	pVirtq->fUsedRingEvent = false;
1198	return;
1199	}
1200	#ifdef IN_RING3
1201	Log6Func(("...skip interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1202	pVirtq->szName,(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq), pVirtq->uUsedIdxShadow));
1203	#endif
1204	}
1205	else
1206	{
1207	/** If guest driver hasn't suppressed interrupts, interrupt */
1208	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1209	{
1210	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1211	return;
1212	}
1213	Log6Func(("...skipping interrupt for %s (guest set VIRTQ_AVAIL_F_NO_INTERRUPT)\n", pVirtq->szName));
1214	}
1215	}
1216
1217	/**
1218	* Raise interrupt or MSI-X
1219	*
1220	* @param pDevIns The device instance.
1221	* @param pVirtio Pointer to the shared virtio state.
1222	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1223	* @param uVec MSI-X vector, if enabled
1224	*/
1225	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1226	{
1227	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1228	Log6Func(("Reason for interrupt - buffer added to 'used' ring.\n"));
1229	else
1230	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1231	Log6Func(("Reason for interrupt - device config change\n"));
1232
1233	if (pVirtio->uIrqMmio)
1234	{
1235	pVirtio->uISR \|= uCause;
1236	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_HIGH);
1237	}
1238	else if (!pVirtio->fMsiSupport)
1239	{
1240	pVirtio->uISR \|= uCause;
1241	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1242	}
1243	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1244	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1245	return VINF_SUCCESS;
1246	}
1247
1248	/**
1249	* Lower interrupt (Called when guest reads ISR and when resetting)
1250	*
1251	* @param pDevIns The device instance.
1252	*/
1253	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1254	{
1255	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1256	if (pVirtio->uIrqMmio)
1257	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_LOW);
1258	else if (!pVirtio->fMsiSupport)
1259	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1260	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1261	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1262	}
1263
1264	#ifdef IN_RING3
1265	static void virtioResetVirtq(PVIRTIOCORE pVirtio, uint16_t uVirtq)
1266	{
1267	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1268	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1269
1270	pVirtq->uQueueSize = VIRTQ_SIZE;
1271	pVirtq->uEnable = false;
1272	pVirtq->uNotifyOffset = uVirtq;
1273	pVirtq->fUsedRingEvent = false;
1274	pVirtq->uAvailIdxShadow = 0;
1275	pVirtq->uUsedIdxShadow = 0;
1276	pVirtq->uMsixVector = uVirtq + 2;
1277
1278	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1279	pVirtq->uMsixVector = VIRTIO_MSI_NO_VECTOR;
1280
1281	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtq->uMsixVector);
1282	}
1283
1284	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1285	{
1286	LogFunc(("Resetting device VirtIO state\n"));
1287	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy; /* Cleared if VIRTIO_F_VERSION_1 feature ack'd */
1288	pVirtio->uDeviceFeaturesSelect = 0;
1289	pVirtio->uDriverFeaturesSelect = 0;
1290	pVirtio->uConfigGeneration = 0;
1291	pVirtio->fDeviceStatus = 0;
1292	pVirtio->uISR = 0;
1293
1294	if (!pVirtio->fMsiSupport)
1295	virtioLowerInterrupt(pDevIns, 0);
1296	else
1297	{
1298	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1299	for (int i = 0; i < VIRTQ_MAX_COUNT; i++)
1300	virtioLowerInterrupt(pDevIns, pVirtio->aVirtqueues[i].uMsixVector);
1301	}
1302
1303	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1304	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1305
1306	for (uint16_t uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
1307	virtioResetVirtq(pVirtio, uVirtq);
1308	}
1309
1310	/**
1311	* Invoked by this implementation when guest driver resets the device.
1312	* The driver itself will not until the device has read the status change.
1313	*/
1314	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1315	{
1316	Log(("%-23s: Guest reset the device\n", __FUNCTION__));
1317
1318	/* Let the client know */
1319	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0 /* fDriverOk */);
1320	virtioResetDevice(pDevIns, pVirtio);
1321	}
1322
1323	DECLHIDDEN(void) virtioCoreR3ResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1324	{
1325	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1326	}
1327	#endif /* IN_RING3 */
1328
1329	/*
1330	* Determines whether guest virtio driver is modern or legacy and does callback
1331	* informing device-specific code that feature negotiation is complete.
1332	* Should be called only once (coordinated via the 'toggle' flag)
1333	*/
1334	#ifdef IN_RING3
1335	DECLINLINE(void) virtioR3DoFeaturesCompleteOnceOnly(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1336	{
1337	if (pVirtio->uDriverFeatures & VIRTIO_F_VERSION_1)
1338	{
1339	LogFunc(("VIRTIO_F_VERSION_1 feature ack'd by guest\n"));
1340	pVirtio->fLegacyDriver = 0;
1341	}
1342	else
1343	{
1344	if (pVirtio->fOfferLegacy)
1345	{
1346	pVirtio->fLegacyDriver = 1;
1347	LogFunc(("VIRTIO_F_VERSION_1 feature was NOT set by guest\n"));
1348	}
1349	else
1350	AssertMsgFailed(("Guest didn't accept VIRTIO_F_VERSION_1, but fLegacyOffered flag not set.\n"));
1351	}
1352	if (pVirtioCC->pfnFeatureNegotiationComplete)
1353	pVirtioCC->pfnFeatureNegotiationComplete(pVirtio, pVirtio->uDriverFeatures, pVirtio->fLegacyDriver);
1354	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_COMPLETE_HANDLED;
1355	}
1356	#endif
1357
1358
1359	/**
1360	* Handles a write to the device status register from the driver.
1361	*
1362	* @returns VBox status code
1363	*
1364	* @param pDevIns The device instance.
1365	* @param pVirtio Pointer to the shared virtio state.
1366	* @param pVirtioCC Pointer to the current context virtio state.
1367	* @param fDeviceStatus The device status to be written.
1368	*/
1369	DECLINLINE(int) virtioDeviceStatusWrite(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1370	uint8_t fDeviceStatus)
1371	{
1372	pVirtio->fDeviceStatus = fDeviceStatus;
1373	bool fDeviceReset = pVirtio->fDeviceStatus == 0;
1374	#ifdef LOG_ENABLED
1375	if (LogIs7Enabled())
1376	{
1377	char szOut[80] = { 0 };
1378	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1379	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1380	}
1381	#endif
1382	bool const fStatusChanged = IS_DRIVER_OK(pVirtio) != WAS_DRIVER_OK(pVirtio);
1383
1384	if (fDeviceReset \|\| fStatusChanged)
1385	{
1386	#ifdef IN_RING0
1387	/* Since VirtIO status changes are cumbersome by nature, e.g. not a benchmark priority,
1388	* handle the rest in R3 to facilitate logging or whatever dev-specific client needs to do */
1389	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1390	return VINF_IOM_R3_MMIO_WRITE;
1391	#endif
1392	}
1393
1394	#ifdef IN_RING3
1395	/*
1396	* Notify client only if status actually changed from last time and when we're reset.
1397	*/
1398	if (fDeviceReset)
1399	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1400
1401	if (fStatusChanged)
1402	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, IS_DRIVER_OK(pVirtio));
1403	#else
1404	RT_NOREF(pDevIns, pVirtioCC);
1405	#endif
1406	/*
1407	* Save the current status for the next write so we can see what changed.
1408	*/
1409	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1410	return VINF_SUCCESS;
1411	}
1412
1413
1414	/**
1415	* Handles a read from the device status register from the driver.
1416	*
1417	* @returns The device status register value.
1418	*
1419	* @param pVirtio Pointer to the shared virtio state.
1420	*/
1421	DECLINLINE(uint8_t) virtioDeviceStatusRead(PVIRTIOCORE pVirtio)
1422	{
1423	#ifdef LOG_ENABLED
1424	if (LogIs7Enabled())
1425	{
1426	char szOut[80] = { 0 };
1427	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1428	LogFunc(("Guest read fDeviceStatus ................ (%s)\n", szOut));
1429	}
1430	#endif
1431	return pVirtio->fDeviceStatus;
1432	}
1433
1434
1435	/**
1436	* Handle accesses to Common Configuration capability
1437	*
1438	* @returns VBox status code
1439	*
1440	* @param pDevIns The device instance.
1441	* @param pVirtio Pointer to the shared virtio state.
1442	* @param pVirtioCC Pointer to the current context virtio state.
1443	* @param fWrite Set if write access, clear if read access.
1444	* @param uOffsetOfAccess The common configuration capability offset.
1445	* @param cb Number of bytes to read or write
1446	* @param pv Pointer to location to write to or read from
1447	*/
1448	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1449	int fWrite, uint32_t uOffsetOfAccess, unsigned cb, void *pv)
1450	{
1451	uint16_t uVirtq = pVirtio->uVirtqSelect;
1452	int rc = VINF_SUCCESS;
1453	uint64_t val;
1454	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1455	{
1456	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1457	{
1458	/* VirtIO 1.0, 4.1.4.3 states device_feature is a (guest) driver readonly field,
1459	* yet the linux driver attempts to write/read it back twice */
1460	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1461	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1462	return VINF_IOM_MMIO_UNUSED_00;
1463	}
1464	else /* Guest READ pCommonCfg->uDeviceFeatures */
1465	{
1466	switch (pVirtio->uDeviceFeaturesSelect)
1467	{
1468	case 0:
1469	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1470	memcpy(pv, &val, cb);
1471	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1472	break;
1473	case 1:
1474	val = pVirtio->uDeviceFeatures >> 32;
1475	memcpy(pv, &val, cb);
1476	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1477	break;
1478	default:
1479	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1480	pVirtio->uDeviceFeaturesSelect));
1481	return VINF_IOM_MMIO_UNUSED_00;
1482	}
1483	}
1484	}
1485	else
1486	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1487	{
1488	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1489	{
1490	switch (pVirtio->uDriverFeaturesSelect)
1491	{
1492	case 0:
1493	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1494	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
1495	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1496	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1497	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1498	#ifdef IN_RING0
1499	return VINF_IOM_R3_MMIO_WRITE;
1500	#endif
1501	#ifdef IN_RING3
1502	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1503	#endif
1504	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1505	break;
1506	case 1:
1507	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1508	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
1509	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1510	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1511	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1512	#ifdef IN_RING0
1513	return VINF_IOM_R3_MMIO_WRITE;
1514	#endif
1515	#ifdef IN_RING3
1516	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1517	#endif
1518	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1519	break;
1520	default:
1521	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1522	pVirtio->uDriverFeaturesSelect));
1523	return VINF_SUCCESS;
1524	}
1525	}
1526	else /* Guest READ pCommonCfg->udriverFeatures */
1527	{
1528	switch (pVirtio->uDriverFeaturesSelect)
1529	{
1530	case 0:
1531	val = pVirtio->uDriverFeatures & 0xffffffff;
1532	memcpy(pv, &val, cb);
1533	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1534	break;
1535	case 1:
1536	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1537	memcpy(pv, &val, cb);
1538	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + 4);
1539	break;
1540	default:
1541	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1542	pVirtio->uDriverFeaturesSelect));
1543	return VINF_IOM_MMIO_UNUSED_00;
1544	}
1545	}
1546	}
1547	else
1548	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1549	{
1550	if (fWrite)
1551	{
1552	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1553	return VINF_SUCCESS;
1554	}
1555	(uint16_t )pv = VIRTQ_MAX_COUNT;
1556	VIRTIO_DEV_CONFIG_LOG_ACCESS(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1557	}
1558	else
1559	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1560	{
1561	if (fWrite) /* Guest WRITE pCommonCfg->fDeviceStatus */
1562	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t )pv);
1563	else /* Guest READ pCommonCfg->fDeviceStatus */
1564	(uint8_t )pv = virtioDeviceStatusRead(pVirtio);
1565	}
1566	else
1567	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1568	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1569	else
1570	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1571	VIRTIO_DEV_CONFIG_ACCESS( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1572	else
1573	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1574	VIRTIO_DEV_CONFIG_ACCESS( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1575	else
1576	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1577	VIRTIO_DEV_CONFIG_ACCESS( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1578	else
1579	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1580	{
1581	if (fWrite) {
1582	uint16_t uVirtqNew = (uint16_t )pv;
1583
1584	if (uVirtqNew < RT_ELEMENTS(pVirtio->aVirtqueues))
1585	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1586	else
1587	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1588	}
1589	else
1590	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1591	}
1592	else
1593	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqDesc, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1594	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqDesc, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1595	else
1596	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqAvail, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1597	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqAvail, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1598	else
1599	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqUsed, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1600	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqUsed, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1601	else
1602	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1603	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1604	else
1605	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uEnable, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1606	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uEnable, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1607	else
1608	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uNotifyOffset, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1609	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uNotifyOffset, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1610	else
1611	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1612	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1613	else
1614	{
1615	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: uOffsetOfAccess=%#x (%d), cb=%d\n",
1616	fWrite ? "write" : "read ", uOffsetOfAccess, uOffsetOfAccess, cb));
1617	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1618	}
1619
1620	#ifndef IN_RING3
1621	RT_NOREF(pDevIns, pVirtioCC);
1622	#endif
1623	return rc;
1624	}
1625
1626	/**
1627	* @callback_method_impl{FNIOMIOPORTNEWIN)
1628	*
1629	* This I/O handler exists only to handle access from legacy drivers.
1630	*/
1631	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortIn(PPDMDEVINS pDevIns, void pvUser, RTIOPORT offPort, uint32_t pu32, unsigned cb)
1632	{
1633	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1634	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1635
1636	RT_NOREF(pvUser);
1637	Log(("%-23s: Port read at offset=%RTiop, cb=%#x%s",
1638	__FUNCTION__, offPort, cb,
1639	VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort) ? "" : "\n"));
1640
1641	void pv = pu32; / To use existing macros */
1642	int fWrite = 0; /* To use existing macros */
1643
1644	uint16_t uVirtq = pVirtio->uVirtqSelect;
1645
1646	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1647	{
1648	uint32_t val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1649	memcpy(pu32, &val, cb);
1650	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1651	}
1652	else
1653	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1654	{
1655	uint32_t val = pVirtio->uDriverFeatures & UINT32_C(0xffffffff);
1656	memcpy(pu32, &val, cb);
1657	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1658	}
1659	else
1660	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1661	{
1662	(uint8_t )pu32 = pVirtio->fDeviceStatus;
1663	#ifdef LOG_ENABLED
1664	if (LogIs7Enabled())
1665	{
1666	char szOut[80] = { 0 };
1667	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1668	Log(("%-23s: Guest read fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1669	}
1670	#endif
1671	}
1672	else
1673	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1674	{
1675	ASSERT_GUEST_MSG(cb == 1, ("%d\n", cb));
1676	(uint8_t )pu32 = pVirtio->uISR;
1677	pVirtio->uISR = 0;
1678	virtioLowerInterrupt( pDevIns, 0);
1679	Log((" (ISR read and cleared)\n"));
1680	}
1681	else
1682	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1683	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1684	else
1685	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1686	{
1687	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[uVirtq];
1688	*pu32 = pVirtQueue->GCPhysVirtqDesc >> GUEST_PAGE_SHIFT;
1689	Log(("%-23s: Guest read uVirtqPfn .................... %#x\n", __FUNCTION__, *pu32));
1690	}
1691	else
1692	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1693	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1694	else
1695	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1696	VIRTIO_DEV_CONFIG_ACCESS( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1697	#ifdef LEGACY_MSIX_SUPPORTED
1698	else
1699	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1700	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1701	else
1702	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1703	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1704	#endif
1705	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1706	{
1707	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1708	#ifdef IN_RING3
1709	/* Access device-specific configuration */
1710	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1711	int rc = pVirtioCC->pfnDevCapRead(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1712	return rc;
1713	#else
1714	return VINF_IOM_R3_IOPORT_READ;
1715	#endif
1716	}
1717	else
1718	{
1719	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1720	Log2Func(("Bad guest read access to virtio_legacy_pci_common_cfg: offset=%#x, cb=%x\n",
1721	offPort, cb));
1722	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1723	"virtioLegacyIOPortIn: no valid port at offset offset=%RTiop cb=%#x\n", offPort, cb);
1724	return rc;
1725	}
1726	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1727	return VINF_SUCCESS;
1728	}
1729
1730	/**
1731	* @callback_method_impl{ * @callback_method_impl{FNIOMIOPORTNEWOUT}
1732	*
1733	* This I/O Port interface exists only to handle access from legacy drivers.
1734	*/
1735	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT offPort, uint32_t u32, unsigned cb)
1736	{
1737	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1738	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1739	RT_NOREF(pvUser);
1740
1741	uint16_t uVirtq = pVirtio->uVirtqSelect;
1742	uint32_t u32OnStack = u32; /* allows us to use this impl's MMIO parsing macros */
1743	void pv = &u32OnStack; / To use existing macros */
1744	int fWrite = 1; /* To use existing macros */
1745
1746	Log(("%-23s: Port written at offset=%RTiop, cb=%#x, u32=%#x\n", __FUNCTION__, offPort, cb, u32));
1747
1748	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1749	{
1750	if (u32 < RT_ELEMENTS(pVirtio->aVirtqueues))
1751	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1752	else
1753	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1754	}
1755	else
1756	#ifdef LEGACY_MSIX_SUPPORTED
1757	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1758	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1759	else
1760	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1761	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1762	else
1763	#endif
1764	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1765	{
1766	/* Check to see if guest acknowledged unsupported features */
1767	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1768	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1769	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1770	return VINF_SUCCESS;
1771	}
1772	else
1773	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1774	{
1775	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1776	if ((pVirtio->uDriverFeatures & ~VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED) == 0)
1777	{
1778	Log(("Guest asked for features host does not support! (host=%x guest=%x)\n",
1779	VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED, pVirtio->uDriverFeatures));
1780	pVirtio->uDriverFeatures &= VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED;
1781	}
1782	if (!(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1783	{
1784	#ifdef IN_RING0
1785	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1786	return VINF_IOM_R3_IOPORT_WRITE;
1787	#endif
1788	#ifdef IN_RING3
1789	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1790	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1791	#endif
1792	}
1793	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1794	}
1795	else
1796	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1797	{
1798	VIRTIO_DEV_CONFIG_LOG_ACCESS(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1799	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (queue size) (ignoring)\n"));
1800	return VINF_SUCCESS;
1801	}
1802	else
1803	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1804	{
1805	bool const fDriverInitiatedReset = (pVirtio->fDeviceStatus = (uint8_t)u32) == 0;
1806	bool const fDriverStateImproved = IS_DRIVER_OK(pVirtio) && !WAS_DRIVER_OK(pVirtio);
1807	#ifdef LOG_ENABLED
1808	if (LogIs7Enabled())
1809	{
1810	char szOut[80] = { 0 };
1811	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1812	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1813	}
1814	#endif
1815	if (fDriverStateImproved \|\| fDriverInitiatedReset)
1816	{
1817	#ifdef IN_RING0
1818	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1819	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1820	return VINF_IOM_R3_IOPORT_WRITE;
1821	#endif
1822	}
1823
1824	#ifdef IN_RING3
1825	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1826	if (fDriverInitiatedReset)
1827	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1828
1829	else if (fDriverStateImproved)
1830	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 1 /* fDriverOk */);
1831
1832	#endif
1833	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1834	}
1835	else
1836	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1837	{
1838	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1839	uint64_t uVirtqPfn = (uint64_t)u32;
1840
1841	if (uVirtqPfn)
1842	{
1843	/* Transitional devices calculate ring physical addresses using rigid spec-defined formulae,
1844	* instead of guest conveying respective address of each ring, as "modern" VirtIO drivers do,
1845	* thus there is no virtq PFN or single base queue address stored in instance data for
1846	* this transitional device, but rather it is derived, when read back, from GCPhysVirtqDesc */
1847
1848	pVirtq->GCPhysVirtqDesc = uVirtqPfn * VIRTIO_PAGE_SIZE;
1849	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
1850	pVirtq->GCPhysVirtqUsed =
1851	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
1852	}
1853	else
1854	{
1855	/* Don't set ring addresses for queue (to meaningless values), when guest resets the virtq's PFN */
1856	pVirtq->GCPhysVirtqDesc = 0;
1857	pVirtq->GCPhysVirtqAvail = 0;
1858	pVirtq->GCPhysVirtqUsed = 0;
1859	}
1860	Log(("%-23s: Guest wrote uVirtqPfn .................... %#x:\n"
1861	"%68s... %p -> GCPhysVirtqDesc\n%68s... %p -> GCPhysVirtqAvail\n%68s... %p -> GCPhysVirtqUsed\n",
1862	__FUNCTION__, u32, " ", pVirtq->GCPhysVirtqDesc, " ", pVirtq->GCPhysVirtqAvail, " ", pVirtq->GCPhysVirtqUsed));
1863	}
1864	else
1865	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1866	{
1867	#ifdef IN_RING3
1868	ASSERT_GUEST_MSG(cb == 2, ("cb=%u\n", cb));
1869	pVirtio->uQueueNotify = u32 & 0xFFFF;
1870	if (uVirtq < VIRTQ_MAX_COUNT)
1871	{
1872	RT_UNTRUSTED_VALIDATED_FENCE();
1873
1874	/* Need to check that queue is configured. Legacy spec didn't have a queue enabled flag */
1875	if (pVirtio->aVirtqueues[pVirtio->uQueueNotify].GCPhysVirtqDesc)
1876	virtioCoreVirtqNotified(pDevIns, pVirtio, pVirtio->uQueueNotify, pVirtio->uQueueNotify /* uNotifyIdx */);
1877	else
1878	Log(("The queue (#%d) being notified has not been initialized.\n", pVirtio->uQueueNotify));
1879	}
1880	else
1881	Log(("Invalid queue number (%d)\n", pVirtio->uQueueNotify));
1882	#else
1883	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1884	return VINF_IOM_R3_IOPORT_WRITE;
1885	#endif
1886	}
1887	else
1888	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1889	{
1890	VIRTIO_DEV_CONFIG_LOG_ACCESS( fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1891	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (ISR status) (ignoring)\n"));
1892	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1893	return VINF_SUCCESS;
1894	}
1895	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1896	{
1897	#ifdef IN_RING3
1898
1899	/* Access device-specific configuration */
1900	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1901	return pVirtioCC->pfnDevCapWrite(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1902	#else
1903	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1904	return VINF_IOM_R3_IOPORT_WRITE;
1905	#endif
1906	}
1907	else
1908	{
1909	Log2Func(("Bad guest write access to virtio_legacy_pci_common_cfg: offset=%#x, cb=0x%x\n",
1910	offPort, cb));
1911	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1912	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1913	"virtioLegacyIOPortOut: no valid port at offset offset=%RTiop cb=0x%#x\n", offPort, cb);
1914	return rc;
1915	}
1916
1917	RT_NOREF(uVirtq);
1918	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1919	return VINF_SUCCESS;
1920	}
1921
1922
1923	/**
1924	* Read from the device specific configuration at the given offset.
1925	*
1926	* @returns VBox status code.
1927	* @param pDevIns
1928	*/
1929	DECLINLINE(VBOXSTRICTRC) virtioDeviceCfgRead(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1930	uint32_t offDevCfg, void *pv, unsigned cb)
1931	{
1932	#ifdef IN_RING3
1933	/*
1934	* Callback to client to manage device-specific configuration.
1935	*/
1936	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offDevCfg, pv, cb);
1937
1938	/*
1939	* Anytime any part of the dev-specific dev config (which this virtio core implementation sees
1940	* as a blob, and virtio dev-specific code separates into fields) is READ, it must be compared
1941	* for deltas from previous read to maintain a config gen. seq. counter (VirtIO 1.0, section 4.1.4.3.1)
1942	*/
1943	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offDevCfg,
1944	pVirtioCC->pbPrevDevSpecificCfg + offDevCfg,
1945	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offDevCfg)));
1946
1947	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1948
1949	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1950	{
1951	++pVirtio->uConfigGeneration;
1952	Log6Func(("Bumped cfg. generation to %d because %s%s\n", pVirtio->uConfigGeneration,
1953	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1954	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1955	pVirtio->fGenUpdatePending = false;
1956	}
1957
1958	virtioLowerInterrupt(pDevIns, 0);
1959	return rcStrict;
1960	#else
1961	RT_NOREF(pDevIns, pVirtio, pVirtioCC, offDevCfg, pv, cb);
1962	return VINF_IOM_R3_MMIO_READ;
1963	#endif
1964	}
1965
1966
1967	/**
1968	* @callback_method_impl{FNIOMMMIONEWREAD,
1969	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1970	*
1971	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1972	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
1973	* of 1, 2 or 4 bytes, only.
1974	*
1975	*/
1976	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1977	{
1978	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1979	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1980	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1981	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1982
1983	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1984
1985	VBOXSTRICTRC rcStrict;
1986	uint32_t uOffset;
1987	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
1988	rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, uOffset, pv, cb);
1989	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
1990	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, uOffset, cb, pv);
1991	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap))
1992	{
1993	(uint8_t )pv = pVirtio->uISR;
1994	Log6Func(("Read and clear ISR\n"));
1995	pVirtio->uISR = 0; /* VirtIO spec requires reads of ISR to clear it */
1996	virtioLowerInterrupt(pDevIns, 0);
1997	rcStrict = VINF_SUCCESS;
1998	}
1999	else
2000	{
2001	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2002	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2003	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2004	}
2005
2006	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2007	return rcStrict;
2008	}
2009
2010	/**
2011	* @callback_method_impl{FNIOMMMIONEWREAD,
2012	* Memory mapped I/O Handler for PCI Capabilities write operations.}
2013	*
2014	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2015	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
2016	* of 1, 2 or 4 bytes, only.
2017	*/
2018	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2019	{
2020	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2021	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2022	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
2023	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2024	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2025
2026	VBOXSTRICTRC rcStrict;
2027	uint32_t uOffset;
2028	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2029	{
2030	#ifdef IN_RING3
2031	/*
2032	* Foreward this MMIO write access for client to deal with.
2033	*/
2034	rcStrict = pVirtioCC->pfnDevCapWrite(pDevIns, uOffset, pv, cb);
2035	#else
2036	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2037	rcStrict = VINF_IOM_R3_MMIO_WRITE;
2038	#endif
2039	}
2040	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2041	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, uOffset, cb, (void )pv);
2042	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
2043	{
2044	pVirtio->uISR = (uint8_t )pv;
2045	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
2046	pVirtio->uISR & 0xff,
2047	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
2048	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
2049	rcStrict = VINF_SUCCESS;
2050	}
2051	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
2052	{
2053	/* This should be guest driver dropping index of a new descriptor in avail ring */
2054	virtioCoreVirtqNotified(pDevIns, pVirtio, uOffset / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
2055	rcStrict = VINF_SUCCESS;
2056	}
2057	else
2058	{
2059	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2060	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2061	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2062	}
2063
2064	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2065	return rcStrict;
2066	}
2067
2068
2069	/**
2070	* @callback_method_impl{FNIOMMMIONEWREAD,
2071	* Memory mapped I/O Handler for Virtio over MMIO read operations.}
2072	*
2073	*/
2074	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2075	{
2076	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2077	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2078	RT_NOREF(pvUser);
2079	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2080
2081	if (off >= VIRTIO_MMIO_SIZE)
2082	{
2083	VBOXSTRICTRC rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2084	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2085	return rcStrict;
2086	}
2087
2088	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2089	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2090	("Bad read access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2091	VINF_IOM_MMIO_UNUSED_FF);
2092
2093	int rc = VINF_SUCCESS;
2094	uint32_t pu32 = (uint32_t )pv;
2095	switch (off)
2096	{
2097	case VIRTIO_MMIO_REG_MAGIC_OFF:
2098	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_MAGIC_VALUE);
2099	break;
2100	case VIRTIO_MMIO_REG_VERSION_OFF:
2101	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_VERSION_VALUE);
2102	break;
2103	case VIRTIO_MMIO_REG_DEVICEID_OFF:
2104	*pu32 = pVirtio->uDeviceType;
2105	break;
2106	case VIRTIO_MMIO_REG_VENDORID_OFF:
2107	*pu32 = RT_H2LE_U32(DEVICE_PCI_VENDOR_ID_VIRTIO);
2108	break;
2109	case VIRTIO_MMIO_REG_DEVICEFEAT_OFF:
2110	{
2111	switch (pVirtio->uDeviceFeaturesSelect)
2112	{
2113	case 0:
2114	*pu32 = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
2115	break;
2116	case 1:
2117	*pu32 = pVirtio->uDeviceFeatures >> 32;
2118	break;
2119	default:
2120	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
2121	pVirtio->uDeviceFeaturesSelect));
2122	rc = VINF_IOM_MMIO_UNUSED_00;
2123	}
2124	break;
2125	}
2126	case VIRTIO_MMIO_REG_QUEUENUMMAX_OFF:
2127	pu32 = VIRTQ_SIZE; /* @todo */
2128	break;
2129	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2130	{
2131	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2132	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2133	*pu32 = pVirtQueue->uEnable;
2134	break;
2135	}
2136	case VIRTIO_MMIO_REG_INTRSTATUS_OFF:
2137	*pu32 = pVirtio->uISR;
2138	break;
2139	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2140	*pu32 = virtioDeviceStatusRead(pVirtio);
2141	break;
2142	case VIRTIO_MMIO_REG_CFGGEN_OFF:
2143	*pu32 = pVirtio->uConfigGeneration;
2144	break;
2145	default:
2146	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2147	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2148	"virtioMmioTransportRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2149	}
2150
2151	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2152	return rc;
2153	}
2154
2155	/**
2156	* @callback_method_impl{FNIOMMMIONEWREAD,
2157	* Memory mapped I/O Handler for Virtio over MMIO write operations.}
2158	*/
2159	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2160	{
2161	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2162	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2163	RT_NOREF(pvUser);
2164	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2165
2166	if (off >= VIRTIO_MMIO_SIZE)
2167	{
2168	#ifdef IN_RING3
2169	/*
2170	* Forward this MMIO write access for client to deal with.
2171	*/
2172	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2173	return pVirtioCC->pfnDevCapWrite(pDevIns, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2174	#else
2175	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2176	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2177	return VINF_IOM_R3_MMIO_WRITE;
2178	#endif
2179	}
2180
2181	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2182	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2183	("Bad write access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2184	VINF_SUCCESS);
2185
2186	int rc = VINF_SUCCESS;
2187	uint32_t const u32Val = (const uint32_t )pv;
2188	switch (off)
2189	{
2190	case VIRTIO_MMIO_REG_DEVICEFEATSEL_OFF:
2191	{
2192	pVirtio->uDeviceFeaturesSelect = u32Val;
2193	break;
2194	}
2195	case VIRTIO_MMIO_REG_DRIVERFEAT_OFF:
2196	{
2197	switch (pVirtio->uDriverFeaturesSelect)
2198	{
2199	case 0:
2200	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0xffffffff00000000)) \| u32Val;
2201	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
2202	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2203	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2204	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2205	#ifdef IN_RING0
2206	return VINF_IOM_R3_MMIO_WRITE;
2207	#endif
2208	#ifdef IN_RING3
2209	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2210	#endif
2211	break;
2212	case 1:
2213	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2214	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
2215	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2216	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2217	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2218	#ifdef IN_RING0
2219	return VINF_IOM_R3_MMIO_WRITE;
2220	#endif
2221	#ifdef IN_RING3
2222	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2223	#endif
2224	break;
2225	default:
2226	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
2227	pVirtio->uDriverFeaturesSelect));
2228	return VINF_SUCCESS;
2229	}
2230	break;
2231	}
2232	case VIRTIO_MMIO_REG_DRIVERFEATSEL_OFF:
2233	{
2234	pVirtio->uDriverFeaturesSelect = u32Val;
2235	break;
2236	}
2237	case VIRTIO_MMIO_REG_QUEUESEL_OFF:
2238	{
2239	if (u32Val < RT_ELEMENTS(pVirtio->aVirtqueues))
2240	pVirtio->uVirtqSelect = (uint16_t)u32Val;
2241	else
2242	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
2243	break;
2244	}
2245	case VIRTIO_MMIO_REG_QUEUENUM_OFF:
2246	{
2247	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2248	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2249	pVirtQueue->uQueueSize = (uint16_t)u32Val;
2250	break;
2251	}
2252	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2253	{
2254	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2255	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2256	pVirtQueue->uEnable = (uint16_t)u32Val;
2257	break;
2258	}
2259	case VIRTIO_MMIO_REG_QUEUENOTIFY_OFF:
2260	{
2261	virtioCoreVirtqNotified(pDevIns, pVirtio, u32Val, (uint16_t)u32Val);
2262	break;
2263	}
2264	case VIRTIO_MMIO_REG_INTRACK_OFF:
2265	{
2266	pVirtio->uISR &= ~u32Val;
2267	if (!pVirtio->uISR)
2268	virtioLowerInterrupt(pDevIns, 0);
2269	break;
2270	}
2271	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2272	{
2273	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t)u32Val);
2274	break;
2275	}
2276	case VIRTIO_MMIO_REG_QUEUEALIGN_LEGACY_OFF:
2277	{
2278	/* Written by edk2 even though we don't offer legacy mode, ignore. */
2279	break;
2280	}
2281	case VIRTIO_MMIO_REG_QUEUEDESCLOW_OFF:
2282	{
2283	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2284	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2285	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0xffffffff00000000)) \| u32Val;
2286	break;
2287	}
2288	case VIRTIO_MMIO_REG_QUEUEDESCHIGH_OFF:
2289	{
2290	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2291	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2292	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2293	break;
2294	}
2295	case VIRTIO_MMIO_REG_QUEUEDRVLOW_OFF:
2296	{
2297	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2298	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2299	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0xffffffff00000000)) \| u32Val;
2300	break;
2301	}
2302	case VIRTIO_MMIO_REG_QUEUEDRVHIGH_OFF:
2303	{
2304	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2305	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2306	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2307	break;
2308	}
2309	case VIRTIO_MMIO_REG_QUEUEDEVLOW_OFF:
2310	{
2311	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2312	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2313	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0xffffffff00000000)) \| u32Val;
2314	break;
2315	}
2316	case VIRTIO_MMIO_REG_QUEUEDEVHIGH_OFF:
2317	{
2318	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2319	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2320	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2321	break;
2322	}
2323	default:
2324	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2325	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2326	"virtioMmioTransportWrite: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2327	}
2328
2329	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2330	return rc;
2331	}
2332
2333
2334	#ifdef IN_RING3
2335
2336	/**
2337	* @callback_method_impl{FNPCICONFIGREAD}
2338	*/
2339	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2340	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
2341	{
2342	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2343	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2344	RT_NOREF(pPciDev);
2345
2346	if (uAddress == pVirtio->uPciCfgDataOff)
2347	{
2348	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2349	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2350	uint32_t uLength = pPciCap->uLength;
2351
2352	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u uLength=%d, bar=%d\n",
2353	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, uLength, pPciCap->uBar));
2354
2355	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2356	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2357	{
2358	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. "
2359	"Ignoring\n"));
2360	*pu32Value = UINT32_MAX;
2361	return VINF_SUCCESS;
2362	}
2363
2364	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
2365	Log7Func((" Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=0x%x -> %Rrc\n",
2366	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
2367	return rcStrict;
2368	}
2369	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u pu32Value=%p\n",
2370	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, pu32Value));
2371	return VINF_PDM_PCI_DO_DEFAULT;
2372	}
2373
2374	/**
2375	* @callback_method_impl{FNPCICONFIGWRITE}
2376	*/
2377	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2378	uint32_t uAddress, unsigned cb, uint32_t u32Value)
2379	{
2380	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2381	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2382	RT_NOREF(pPciDev);
2383
2384	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x %scb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, uAddress < 0xf ? " " : "", cb, u32Value));
2385	if (uAddress == pVirtio->uPciCfgDataOff)
2386	{
2387	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2388	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2389	uint32_t uLength = pPciCap->uLength;
2390
2391	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2392	\|\| cb != uLength
2393	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2394	{
2395	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
2396	return VINF_SUCCESS;
2397	}
2398
2399	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
2400	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
2401	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
2402	return rcStrict;
2403	}
2404	return VINF_PDM_PCI_DO_DEFAULT;
2405	}
2406
2407
2408	/*********************************************************************************************************************************
2409	* Saved state (SSM) *
2410	*********************************************************************************************************************************/
2411
2412
2413	/**
2414	* Loads a saved device state (called from device-specific code on SSM final pass)
2415	*
2416	* @param pVirtio Pointer to the shared virtio state.
2417	* @param pHlp The ring-3 device helpers.
2418	* @param pSSM The saved state handle.
2419	* @returns VBox status code.
2420	*/
2421	DECLHIDDEN(int) virtioCoreR3LegacyDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2422	uint32_t uVersion, uint32_t uVirtioLegacy_3_1_Beta)
2423	{
2424	int rc;
2425	uint32_t uDriverFeaturesLegacy32bit;
2426
2427	rc = pHlp->pfnSSMGetU32( pSSM, &uDriverFeaturesLegacy32bit);
2428	AssertRCReturn(rc, rc);
2429	pVirtio->uDriverFeatures = (uint64_t)uDriverFeaturesLegacy32bit;
2430
2431	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2432	AssertRCReturn(rc, rc);
2433
2434	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2435	AssertRCReturn(rc, rc);
2436
2437	#ifdef LOG_ENABLED
2438	char szOut[80] = { 0 };
2439	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
2440	Log(("Loaded legacy device status = (%s)\n", szOut));
2441	#endif
2442
2443	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2444	AssertRCReturn(rc, rc);
2445
2446	uint32_t cQueues = 3; /* This constant default value copied from earliest v0.9 code */
2447	if (uVersion > uVirtioLegacy_3_1_Beta)
2448	{
2449	rc = pHlp->pfnSSMGetU32(pSSM, &cQueues);
2450	AssertRCReturn(rc, rc);
2451	}
2452
2453	AssertLogRelMsgReturn(cQueues <= VIRTQ_MAX_COUNT, ("%#x\n", cQueues), VERR_SSM_LOAD_CONFIG_MISMATCH);
2454	AssertLogRelMsgReturn(pVirtio->uVirtqSelect < cQueues \|\| (cQueues == 0 && pVirtio->uVirtqSelect),
2455	("uVirtqSelect=%u cQueues=%u\n", pVirtio->uVirtqSelect, cQueues),
2456	VERR_SSM_LOAD_CONFIG_MISMATCH);
2457
2458	Log(("\nRestoring %d legacy-only virtio-net device queues from saved state:\n", cQueues));
2459	for (unsigned uVirtq = 0; uVirtq < cQueues; uVirtq++)
2460	{
2461	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
2462
2463	if (uVirtq == cQueues - 1)
2464	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-ctrlq");
2465	else if (uVirtq % 2)
2466	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-xmitq<%d>", uVirtq / 2);
2467	else
2468	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-recvq<%d>", uVirtq / 2);
2469
2470	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uQueueSize);
2471	AssertRCReturn(rc, rc);
2472
2473	uint32_t uVirtqPfn;
2474	rc = pHlp->pfnSSMGetU32(pSSM, &uVirtqPfn);
2475	AssertRCReturn(rc, rc);
2476
2477	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uAvailIdxShadow);
2478	AssertRCReturn(rc, rc);
2479
2480	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uUsedIdxShadow);
2481	AssertRCReturn(rc, rc);
2482
2483	if (uVirtqPfn)
2484	{
2485	pVirtq->GCPhysVirtqDesc = (uint64_t)uVirtqPfn * VIRTIO_PAGE_SIZE;
2486	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
2487	pVirtq->GCPhysVirtqUsed =
2488	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
2489	pVirtq->uEnable = 1;
2490	}
2491	else
2492	{
2493	LogFunc(("WARNING: QUEUE \"%s\" PAGE NUMBER ZERO IN SAVED STATE\n", pVirtq->szName));
2494	pVirtq->uEnable = 0;
2495	}
2496	pVirtq->uNotifyOffset = 0; /* unused in legacy mode */
2497	pVirtq->uMsixVector = 0; /* unused in legacy mode */
2498	}
2499	pVirtio->fGenUpdatePending = 0; /* unused in legacy mode */
2500	pVirtio->uConfigGeneration = 0; /* unused in legacy mode */
2501	pVirtio->uPciCfgDataOff = 0; /* unused in legacy mode (port I/O used instead) */
2502
2503	return VINF_SUCCESS;
2504	}
2505
2506	/**
2507	* Loads a saved device state (called from device-specific code on SSM final pass)
2508	*
2509	* Note: This loads state saved by a Modern (VirtIO 1.0+) device, of which this transitional device is one,
2510	* and thus supports both legacy and modern guest virtio drivers.
2511	*
2512	* @param pVirtio Pointer to the shared virtio state.
2513	* @param pHlp The ring-3 device helpers.
2514	* @param pSSM The saved state handle.
2515	* @returns VBox status code.
2516	*/
2517	DECLHIDDEN(int) virtioCoreR3ModernDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2518	uint32_t uVersion, uint32_t uTestVersion, uint32_t cQueues)
2519	{
2520	RT_NOREF2(cQueues, uVersion);
2521	LogFunc(("\n"));
2522	/*
2523	* Check the marker and (embedded) version number.
2524	*/
2525	uint64_t uMarker = 0;
2526	int rc;
2527
2528	rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
2529	AssertRCReturn(rc, rc);
2530	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
2531	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2532	N_("Expected marker value %#RX64 found %#RX64 instead"),
2533	VIRTIO_SAVEDSTATE_MARKER, uMarker);
2534	uint32_t uVersionSaved = 0;
2535	rc = pHlp->pfnSSMGetU32(pSSM, &uVersionSaved);
2536	AssertRCReturn(rc, rc);
2537	if (uVersionSaved != uTestVersion)
2538	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2539	N_("Unsupported virtio version: %u"), uVersionSaved);
2540	/*
2541	* Load the state.
2542	*/
2543	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->fLegacyDriver);
2544	AssertRCReturn(rc, rc);
2545	rc = pHlp->pfnSSMGetBool( pSSM, &pVirtio->fGenUpdatePending);
2546	AssertRCReturn(rc, rc);
2547	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2548	AssertRCReturn(rc, rc);
2549	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uConfigGeneration);
2550	AssertRCReturn(rc, rc);
2551	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uPciCfgDataOff);
2552	AssertRCReturn(rc, rc);
2553	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2554	AssertRCReturn(rc, rc);
2555	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2556	AssertRCReturn(rc, rc);
2557	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDeviceFeaturesSelect);
2558	AssertRCReturn(rc, rc);
2559	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDriverFeaturesSelect);
2560	AssertRCReturn(rc, rc);
2561	rc = pHlp->pfnSSMGetU64( pSSM, &pVirtio->uDriverFeatures);
2562	AssertRCReturn(rc, rc);
2563
2564	/** @todo Adapt this loop use cQueues argument instead of static queue count (safely with SSM versioning) */
2565	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2566	{
2567	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2568	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqDesc);
2569	AssertRCReturn(rc, rc);
2570	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqAvail);
2571	AssertRCReturn(rc, rc);
2572	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqUsed);
2573	AssertRCReturn(rc, rc);
2574	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uNotifyOffset);
2575	AssertRCReturn(rc, rc);
2576	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uMsixVector);
2577	AssertRCReturn(rc, rc);
2578	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uEnable);
2579	AssertRCReturn(rc, rc);
2580	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uQueueSize);
2581	AssertRCReturn(rc, rc);
2582	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uAvailIdxShadow);
2583	AssertRCReturn(rc, rc);
2584	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uUsedIdxShadow);
2585	AssertRCReturn(rc, rc);
2586	rc = pHlp->pfnSSMGetMem( pSSM, pVirtq->szName, sizeof(pVirtq->szName));
2587	AssertRCReturn(rc, rc);
2588	}
2589	return VINF_SUCCESS;
2590	}
2591
2592	/**
2593	* Called from the FNSSMDEVSAVEEXEC function of the device.
2594	*
2595	* @param pVirtio Pointer to the shared virtio state.
2596	* @param pHlp The ring-3 device helpers.
2597	* @param pSSM The saved state handle.
2598	* @returns VBox status code.
2599	*/
2600	DECLHIDDEN(int) virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t cQueues)
2601	{
2602	RT_NOREF(cQueues);
2603	/** @todo figure out a way to save cQueues (with SSM versioning) */
2604
2605	LogFunc(("\n"));
2606	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
2607	pHlp->pfnSSMPutU32(pSSM, uVersion);
2608
2609	pHlp->pfnSSMPutU32( pSSM, pVirtio->fLegacyDriver);
2610	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
2611	pHlp->pfnSSMPutU8( pSSM, pVirtio->fDeviceStatus);
2612	pHlp->pfnSSMPutU8( pSSM, pVirtio->uConfigGeneration);
2613	pHlp->pfnSSMPutU8( pSSM, pVirtio->uPciCfgDataOff);
2614	pHlp->pfnSSMPutU8( pSSM, pVirtio->uISR);
2615	pHlp->pfnSSMPutU16( pSSM, pVirtio->uVirtqSelect);
2616	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDeviceFeaturesSelect);
2617	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDriverFeaturesSelect);
2618	pHlp->pfnSSMPutU64( pSSM, pVirtio->uDriverFeatures);
2619
2620	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2621	{
2622	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2623
2624	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqDesc);
2625	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqAvail);
2626	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqUsed);
2627	pHlp->pfnSSMPutU16( pSSM, pVirtq->uNotifyOffset);
2628	pHlp->pfnSSMPutU16( pSSM, pVirtq->uMsixVector);
2629	pHlp->pfnSSMPutU16( pSSM, pVirtq->uEnable);
2630	pHlp->pfnSSMPutU16( pSSM, pVirtq->uQueueSize);
2631	pHlp->pfnSSMPutU16( pSSM, pVirtq->uAvailIdxShadow);
2632	pHlp->pfnSSMPutU16( pSSM, pVirtq->uUsedIdxShadow);
2633	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtq->szName, 32);
2634	AssertRCReturn(rc, rc);
2635	}
2636	return VINF_SUCCESS;
2637	}
2638
2639
2640	/*********************************************************************************************************************************
2641	* Device Level *
2642	*********************************************************************************************************************************/
2643
2644	/**
2645	* This must be called by the client to handle VM state changes after the client takes care of its device-specific
2646	* tasks for the state change (i.e. reset, suspend, power-off, resume)
2647	*
2648	* @param pDevIns The device instance.
2649	* @param pVirtio Pointer to the shared virtio state.
2650	*/
2651	DECLHIDDEN(void) virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
2652	{
2653	LogFunc(("State changing to %s\n",
2654	virtioCoreGetStateChangeText(enmState)));
2655
2656	switch(enmState)
2657	{
2658	case kvirtIoVmStateChangedReset:
2659	virtioCoreResetAll(pVirtio);
2660	break;
2661	case kvirtIoVmStateChangedSuspend:
2662	break;
2663	case kvirtIoVmStateChangedPowerOff:
2664	break;
2665	case kvirtIoVmStateChangedResume:
2666	for (int uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
2667	{
2668	if ((!pVirtio->fLegacyDriver && pVirtio->aVirtqueues[uVirtq].uEnable)
2669	\| pVirtio->aVirtqueues[uVirtq].GCPhysVirtqDesc)
2670	virtioCoreNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, uVirtq);
2671	}
2672	break;
2673	default:
2674	LogRelFunc(("Bad enum value"));
2675	return;
2676	}
2677	}
2678
2679	/**
2680	* This should be called from PDMDEVREGR3::pfnDestruct.
2681	*
2682	* @param pDevIns The device instance.
2683	* @param pVirtio Pointer to the shared virtio state.
2684	* @param pVirtioCC Pointer to the ring-3 virtio state.
2685	*/
2686	DECLHIDDEN(void) virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2687	{
2688	if (pVirtioCC->pbPrevDevSpecificCfg)
2689	{
2690	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
2691	pVirtioCC->pbPrevDevSpecificCfg = NULL;
2692	}
2693
2694	RT_NOREF(pDevIns, pVirtio);
2695	}
2696
2697
2698	/**
2699	* Setup the Virtio device as a PCI device.
2700	*
2701	* @returns VBox status code.
2702	* @param pDevIns Device instance.
2703	* @param pVirtio Pointer to the shared virtio state. This
2704	* must be the first member in the shared
2705	* device instance data!
2706	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2707	* must be the first member in the ring-3
2708	* device instance data!
2709	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
2710	* @param pcszInstance Device instance name (format-specifier)
2711	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2712	*/
2713	static int virtioR3PciTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2714	const char *pcszInstance, uint16_t cbDevSpecificCfg)
2715	{
2716	/* Set PCI config registers (assume 32-bit mode) */
2717	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
2718	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
2719
2720	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2721	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
2722
2723	if (pPciParams->uDeviceId < DEVICE_PCI_DEVICE_ID_VIRTIO_BASE)
2724	/* Transitional devices MUST have a PCI Revision ID of 0. */
2725	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_TRANS);
2726	else
2727	/* Non-transitional devices SHOULD have a PCI Revision ID of 1 or higher. */
2728	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_V1);
2729
2730	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
2731	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2732	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
2733	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
2734	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
2735	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
2736	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
2737
2738	/* Register PCI device */
2739	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2740	if (RT_FAILURE(rc))
2741	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2742
2743	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2744	AssertRCReturn(rc, rc);
2745
2746	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2747
2748	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2749	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2750	do { \
2751	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2752	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2753	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2754	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2755	} while (0)
2756
2757	PVIRTIO_PCI_CAP_T pCfg;
2758	uint32_t cbRegion = 0;
2759
2760	/*
2761	* Common capability (VirtIO 1.0, section 4.1.4.3)
2762	*/
2763	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2764	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2765	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2766	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2767	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2768	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2769	pCfg->uOffset = RT_ALIGN_32(0, 4); /* Currently 0, but reminder to 32-bit align if changing this */
2770	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2771	cbRegion += pCfg->uLength;
2772	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2773	pVirtioCC->pCommonCfgCap = pCfg;
2774
2775	/*
2776	* Notify capability (VirtIO 1.0, section 4.1.4.4).
2777	*
2778	* The size of the spec-defined subregion described by this VirtIO capability is
2779	* based-on the choice of this implementation to make the notification area of each
2780	* queue equal to queue's ordinal position (e.g. queue selector value). The VirtIO
2781	* specification leaves it up to implementation to define queue notification area layout.
2782	*/
2783	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2784	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2785	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2786	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2787	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2788	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2789	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2790	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2791	pCfg->uLength = VIRTQ_MAX_COUNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2792	cbRegion += pCfg->uLength;
2793	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2794	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2795	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2796
2797	/* ISR capability (VirtIO 1.0, section 4.1.4.5)
2798	*
2799	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. The specification example/diagram
2800	* illustrates this capability as 32-bit field with upper bits 'reserved'. Those depictions
2801	* differ. The spec's wording, not the diagram, is seen to work in practice.
2802	*/
2803	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2804	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2805	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2806	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2807	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2808	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2809	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2810	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2811	pCfg->uLength = sizeof(uint8_t);
2812	cbRegion += pCfg->uLength;
2813	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2814	pVirtioCC->pIsrCap = pCfg;
2815
2816	/* PCI Cfg capability (VirtIO 1.0, section 4.1.4.7)
2817	*
2818	* This capability facilitates early-boot access to this device (BIOS).
2819	* This region isn't page-MMIO mapped. PCI configuration accesses are intercepted,
2820	* wherein uBar, uOffset and uLength are modulated by consumers to locate and read/write
2821	* values in any part of any region. (NOTE: Linux driver doesn't utilize this feature.
2822	* This capability only appears in lspci output on Linux if uLength is non-zero, 4-byte aligned,
2823	* during initialization of linux virtio driver).
2824	*/
2825	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2826	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2827	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2828	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2829	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2830	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2831	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2832	pCfg->uOffset = 0;
2833	pCfg->uLength = 4;
2834	cbRegion += pCfg->uLength;
2835	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2836	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2837
2838	if (pVirtioCC->pbDevSpecificCfg)
2839	{
2840	/* Device-specific config capability (VirtIO 1.0, section 4.1.4.6).
2841	*
2842	* Client defines the device-specific config struct and passes size to virtioCoreR3Init()
2843	* to inform this.
2844	*/
2845	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2846	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2847	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2848	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2849	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2850	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2851	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2852	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2853	pCfg->uLength = cbDevSpecificCfg;
2854	cbRegion += pCfg->uLength;
2855	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2856	pVirtioCC->pDeviceCap = pCfg;
2857	}
2858	else
2859	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2860
2861	if (pVirtio->fMsiSupport)
2862	{
2863	PDMMSIREG aMsiReg;
2864	RT_ZERO(aMsiReg);
2865	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2866	aMsiReg.iMsixNextOffset = 0;
2867	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2868	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2869	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2870	if (RT_FAILURE(rc))
2871	{
2872	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2873	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2874	pVirtio->fMsiSupport = false;
2875	}
2876	else
2877	Log2Func(("Using MSI-X for guest driver notification\n"));
2878	}
2879	else
2880	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2881
2882	/* Set offset to first capability and enable PCI dev capabilities */
2883	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2884	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2885
2886	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2887	if (cbSize <= 0)
2888	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2889
2890	cbSize = RTStrPrintf(pVirtioCC->szPortIoName, sizeof(pVirtioCC->szPortIoName), "%s (legacy)", pcszInstance);
2891	if (cbSize <= 0)
2892	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2893
2894	if (pVirtio->fOfferLegacy)
2895	{
2896	/* As a transitional device that supports legacy VirtIO drivers, this VirtIO device generic implementation presents
2897	* legacy driver interface in I/O space at BAR0. The following maps the common (e.g. device independent)
2898	* dev config area as well as device-specific dev config area (whose size is passed to init function of this VirtIO
2899	* generic device code) for access via Port I/O, since legacy drivers (e.g. pre VirtIO 1.0) don't use MMIO callbacks.
2900	* (See VirtIO 1.1, Section 4.1.4.8).
2901	*/
2902	rc = PDMDevHlpPCIIORegionCreateIo(pDevIns, VIRTIO_REGION_LEGACY_IO, sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T) + cbDevSpecificCfg,
2903	virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/, pVirtioCC->szPortIoName,
2904	NULL /paExtDescs/, &pVirtio->hLegacyIoPorts);
2905	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register legacy config in I/O space at BAR0 */")));
2906	}
2907
2908	/* Note: The Linux driver at drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2909	* 'unknown' device-specific capability without querying the capability to determine size, so pad w/extra page.
2910	*/
2911	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + VIRTIO_PAGE_SIZE, VIRTIO_PAGE_SIZE),
2912	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2913	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2914	pVirtioCC->szMmioName,
2915	&pVirtio->hMmioPciCap);
2916	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2917	return VINF_SUCCESS;
2918	}
2919
2920
2921	/**
2922	* Initializes the VirtIO device using the VirtIO over MMIO transport mode.
2923	*
2924	* @returns VBox status code.
2925	* @param pDevIns Device instance.
2926	* @param pVirtio Pointer to the shared virtio state. This
2927	* must be the first member in the shared
2928	* device instance data!
2929	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2930	* must be the first member in the ring-3
2931	* device instance data!
2932	* @param pcszInstance Device instance name (format-specifier)
2933	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2934	* @param GCPhysMmioBase The physical guest address of the start of the MMIO area.
2935	* @param u16Irq The interrupt number to use for the virtio device.
2936	*/
2937	static int virtioR3MmioTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, const char *pcszInstance,
2938	uint16_t cbDevSpecificCfg, RTGCPHYS GCPhysMmioBase, uint16_t u16Irq)
2939	{
2940	pVirtio->uIrqMmio = u16Irq;
2941
2942	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2943	if (cbSize <= 0)
2944	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2945
2946	/*
2947	* Register and map the MMIO region.
2948	*/
2949	int rc = PDMDevHlpMmioCreateAndMap(pDevIns, GCPhysMmioBase, RT_ALIGN_32(cbDevSpecificCfg + VIRTIO_MMIO_SIZE, 512),
2950	virtioMmioTransportWrite, virtioMmioTransportRead,
2951	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2952	pVirtioCC->szMmioName, &pVirtio->hMmioPciCap);
2953	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2954	return VINF_SUCCESS;
2955	}
2956
2957
2958	/** API Function: See header file */
2959	DECLHIDDEN(int) virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2960	const char *pcszInstance, uint64_t fDevSpecificFeatures, uint32_t fOfferLegacy,
2961	void *pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
2962	{
2963	/*
2964	* Virtio state must be the first member of shared device instance data,
2965	* otherwise can't get our bearings in PCI config callbacks.
2966	*/
2967	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2968	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
2969
2970	pVirtio->pDevInsR3 = pDevIns;
2971
2972	/*
2973	* Caller must initialize these.
2974	*/
2975	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
2976	AssertReturn(pVirtioCC->pfnVirtqNotified, VERR_INVALID_POINTER);
2977	AssertReturn(VIRTQ_SIZE > 0 && VIRTQ_SIZE <= 32768, VERR_OUT_OF_RANGE); /* VirtIO specification-defined limit */
2978
2979	PCPDMDEVHLPR3 pHlp = pDevIns->pHlpR3;
2980
2981	uint16_t u16Irq = 0;
2982	int rc = pHlp->pfnCFGMQueryU16Def(pDevIns->pCfg, "Irq", &u16Irq, 0);
2983	if (RT_FAILURE(rc))
2984	return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to get the \"Irq\" value"));
2985
2986	RTGCPHYS GCPhysMmioBase = 0;
2987	rc = pHlp->pfnCFGMQueryU64Def(pDevIns->pCfg, "MmioBase", &GCPhysMmioBase, NIL_RTGCPHYS);
2988	if (RT_FAILURE(rc))
2989	return PDMDEV_SET_ERROR(pDevIns, rc,
2990	N_("Configuration error: Failed to get the \"MmioBase\" value"));
2991
2992	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed
2993	* VBox legacy MSI support has not been implemented yet
2994	*/
2995	# ifdef VBOX_WITH_MSI_DEVICES
2996	pVirtio->fMsiSupport = true;
2997	# endif
2998	#endif
2999
3000	/*
3001	* Host features (presented as a buffet for guest to select from)
3002	* include both dev-specific features & reserved dev-independent features (bitmask).
3003	*/
3004	pVirtio->uDeviceType = pPciParams->uDeviceType;
3005	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
3006	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
3007	\| fDevSpecificFeatures;
3008
3009	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy = fOfferLegacy;
3010
3011	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
3012	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
3013	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
3014	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
3015	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
3016
3017	if (GCPhysMmioBase != NIL_RTGCPHYS)
3018	rc = virtioR3MmioTransportInit(pDevIns, pVirtio, pVirtioCC, pcszInstance, cbDevSpecificCfg,
3019	GCPhysMmioBase, u16Irq);
3020	else
3021	rc = virtioR3PciTransportInit(pDevIns, pVirtio, pVirtioCC, pPciParams, pcszInstance, cbDevSpecificCfg);
3022	AssertLogRelRCReturn(rc, rc);
3023
3024	/*
3025	* Statistics.
3026	*/
3027	# ifdef VBOX_WITH_STATISTICS
3028	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3029	"Total number of allocated descriptor chains", "DescChainsAllocated");
3030	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3031	"Total number of freed descriptor chains", "DescChainsFreed");
3032	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3033	"Total number of inbound segments", "DescChainsSegsIn");
3034	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3035	"Total number of outbound segments", "DescChainsSegsOut");
3036	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR3, STAMTYPE_PROFILE, "IO/ReadR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R3");
3037	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR0, STAMTYPE_PROFILE, "IO/ReadR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R0");
3038	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadRC, STAMTYPE_PROFILE, "IO/ReadRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in RC");
3039	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR3, STAMTYPE_PROFILE, "IO/WriteR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R3");
3040	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR0, STAMTYPE_PROFILE, "IO/WriteR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R0");
3041	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteRC, STAMTYPE_PROFILE, "IO/WriteRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in RC");
3042	# endif /* VBOX_WITH_STATISTICS */
3043
3044	return VINF_SUCCESS;
3045	}
3046
3047	#else /* !IN_RING3 */
3048
3049	/**
3050	* Sets up the core ring-0/raw-mode virtio bits.
3051	*
3052	* @returns VBox status code.
3053	* @param pDevIns The device instance.
3054	* @param pVirtio Pointer to the shared virtio state. This must be the first
3055	* member in the shared device instance data!
3056	*/
3057	DECLHIDDEN(int) virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
3058	{
3059	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3060	int rc;
3061	#ifdef FUTURE_OPTIMIZATION
3062	rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
3063	AssertRCReturn(rc, rc);
3064	#endif
3065
3066	if (pVirtio->uIrqMmio != 0)
3067	{
3068	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioTransportWrite, virtioMmioTransportRead, pVirtio);
3069	AssertRCReturn(rc, rc);
3070	}
3071	else
3072	{
3073	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
3074	AssertRCReturn(rc, rc);
3075
3076	if (pVirtio->fOfferLegacy)
3077	{
3078	rc = PDMDevHlpIoPortSetUpContext(pDevIns, pVirtio->hLegacyIoPorts, virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/);
3079	AssertRCReturn(rc, rc);
3080	}
3081	}
3082	return rc;
3083	}
3084
3085	#endif /* !IN_RING3 */
3086

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/VirtioCore.cpp@ 103914

Download in other formats: