VirtualBox

source: vbox/trunk/src/VBox/HostDrivers/Support/SUPDrvGip.cpp@ 54335

Last change on this file since 54335 was 54334, checked in by vboxsync, 10 years ago

SUPDrvGip.cpp: We can skip the determin-async-tsc test on invariant tsc CPUs now.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 129.2 KB
Line 
1/* $Id: SUPDrvGip.cpp 54334 2015-02-20 15:01:15Z vboxsync $ */
2/** @file
3 * VBoxDrv - The VirtualBox Support Driver - Common code for GIP.
4 */
5
6/*
7 * Copyright (C) 2006-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27/*******************************************************************************
28* Header Files *
29*******************************************************************************/
30#define LOG_GROUP LOG_GROUP_SUP_DRV
31#define SUPDRV_AGNOSTIC
32#include "SUPDrvInternal.h"
33#ifndef PAGE_SHIFT
34# include <iprt/param.h>
35#endif
36#include <iprt/asm.h>
37#include <iprt/asm-amd64-x86.h>
38#include <iprt/asm-math.h>
39#include <iprt/cpuset.h>
40#include <iprt/handletable.h>
41#include <iprt/mem.h>
42#include <iprt/mp.h>
43#include <iprt/power.h>
44#include <iprt/process.h>
45#include <iprt/semaphore.h>
46#include <iprt/spinlock.h>
47#include <iprt/thread.h>
48#include <iprt/uuid.h>
49#include <iprt/net.h>
50#include <iprt/crc.h>
51#include <iprt/string.h>
52#include <iprt/timer.h>
53#if defined(RT_OS_DARWIN) || defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD)
54# include <iprt/rand.h>
55# include <iprt/path.h>
56#endif
57#include <iprt/uint128.h>
58#include <iprt/x86.h>
59
60#include <VBox/param.h>
61#include <VBox/log.h>
62#include <VBox/err.h>
63
64#if defined(RT_OS_SOLARIS) || defined(RT_OS_DARWIN)
65# include "dtrace/SUPDrv.h"
66#else
67/* ... */
68#endif
69
70
71/*******************************************************************************
72* Defined Constants And Macros *
73*******************************************************************************/
74/** The frequency by which we recalculate the u32UpdateHz and
75 * u32UpdateIntervalNS GIP members. The value must be a power of 2.
76 *
77 * Warning: Bumping this too high might overflow u32UpdateIntervalNS.
78 */
79#define GIP_UPDATEHZ_RECALC_FREQ 0x800
80
81/** A reserved TSC value used for synchronization as well as measurement of
82 * TSC deltas. */
83#define GIP_TSC_DELTA_RSVD UINT64_MAX
84/** The number of TSC delta measurement loops in total (includes primer and
85 * read-time loops). */
86#define GIP_TSC_DELTA_LOOPS 96
87/** The number of cache primer loops. */
88#define GIP_TSC_DELTA_PRIMER_LOOPS 4
89/** The number of loops until we keep computing the minumum read time. */
90#define GIP_TSC_DELTA_READ_TIME_LOOPS 24
91/** Stop measurement of TSC delta. */
92#define GIP_TSC_DELTA_SYNC_STOP 0
93/** Start measurement of TSC delta. */
94#define GIP_TSC_DELTA_SYNC_START 1
95/** Worker thread is ready for reading the TSC. */
96#define GIP_TSC_DELTA_SYNC_WORKER_READY 2
97/** Worker thread is done updating TSC delta info. */
98#define GIP_TSC_DELTA_SYNC_WORKER_DONE 3
99/** When IPRT is isn't concurrent safe: Master is ready and will wait for worker
100 * with a timeout. */
101#define GIP_TSC_DELTA_SYNC_PRESTART_MASTER 4
102/** When IPRT is isn't concurrent safe: Worker is ready after waiting for
103 * master with a timeout. */
104#define GIP_TSC_DELTA_SYNC_PRESTART_WORKER 5
105/** The TSC-refinement interval in seconds. */
106#define GIP_TSC_REFINE_INTERVAL 5
107/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_PRACTICALLY_ZERO rating */
108#define GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO 32
109/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_ROUGHLY_ZERO rating */
110#define GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO 448
111/** The TSC delta value for the initial GIP master - 0 in regular builds.
112 * To test the delta code this can be set to a non-zero value. */
113#if 1
114# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(170139095182512) /* 0x00009abd9854acb0 */
115#else
116# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(0)
117#endif
118
119AssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS < GIP_TSC_DELTA_READ_TIME_LOOPS);
120AssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS < GIP_TSC_DELTA_LOOPS);
121
122/** @def VBOX_SVN_REV
123 * The makefile should define this if it can. */
124#ifndef VBOX_SVN_REV
125# define VBOX_SVN_REV 0
126#endif
127
128#if 0 /* Don't start the GIP timers. Useful when debugging the IPRT timer code. */
129# define DO_NOT_START_GIP
130#endif
131
132
133/*******************************************************************************
134* Internal Functions *
135*******************************************************************************/
136static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
137static DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
138static void supdrvGipInitCpu(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS);
139#ifdef SUPDRV_USE_TSC_DELTA_THREAD
140static int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt);
141static void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt);
142static int supdrvTscDeltaThreadWaitForOnlineCpus(PSUPDRVDEVEXT pDevExt);
143#endif
144
145
146/*******************************************************************************
147* Global Variables *
148*******************************************************************************/
149DECLEXPORT(PSUPGLOBALINFOPAGE) g_pSUPGlobalInfoPage = NULL;
150
151
152
153/*
154 *
155 * Misc Common GIP Code
156 * Misc Common GIP Code
157 * Misc Common GIP Code
158 *
159 *
160 */
161
162
163/**
164 * Finds the GIP CPU index corresponding to @a idCpu.
165 *
166 * @returns GIP CPU array index, UINT32_MAX if not found.
167 * @param pGip The GIP.
168 * @param idCpu The CPU ID.
169 */
170static uint32_t supdrvGipFindCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
171{
172 uint32_t i;
173 for (i = 0; i < pGip->cCpus; i++)
174 if (pGip->aCPUs[i].idCpu == idCpu)
175 return i;
176 return UINT32_MAX;
177}
178
179
180/**
181 * Applies the TSC delta to the supplied raw TSC value.
182 *
183 * @returns VBox status code. (Ignored by all users, just FYI.)
184 * @param pGip Pointer to the GIP.
185 * @param puTsc Pointer to a valid TSC value before the TSC delta has been applied.
186 * @param idApic The APIC ID of the CPU @c puTsc corresponds to.
187 * @param fDeltaApplied Where to store whether the TSC delta was succesfully
188 * applied or not (optional, can be NULL).
189 *
190 * @remarks Maybe called with interrupts disabled in ring-0!
191 *
192 * @note Don't you dare change the delta calculation. If you really do, make
193 * sure you update all places where it's used (IPRT, SUPLibAll.cpp,
194 * SUPDrv.c, supdrvGipMpEvent, and more).
195 */
196DECLINLINE(int) supdrvTscDeltaApply(PSUPGLOBALINFOPAGE pGip, uint64_t *puTsc, uint16_t idApic, bool *pfDeltaApplied)
197{
198 int rc;
199
200 /*
201 * Validate input.
202 */
203 AssertPtr(puTsc);
204 AssertPtr(pGip);
205 Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
206
207 /*
208 * Carefully convert the idApic into a GIPCPU entry.
209 */
210 if (RT_LIKELY(idApic < RT_ELEMENTS(pGip->aiCpuFromApicId)))
211 {
212 uint16_t iCpu = pGip->aiCpuFromApicId[idApic];
213 if (RT_LIKELY(iCpu < pGip->cCpus))
214 {
215 PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
216
217 /*
218 * Apply the delta if valid.
219 */
220 if (RT_LIKELY(pGipCpu->i64TSCDelta != INT64_MAX))
221 {
222 *puTsc -= pGipCpu->i64TSCDelta;
223 if (pfDeltaApplied)
224 *pfDeltaApplied = true;
225 return VINF_SUCCESS;
226 }
227
228 rc = VINF_SUCCESS;
229 }
230 else
231 {
232 AssertMsgFailed(("iCpu=%u cCpus=%u\n", iCpu, pGip->cCpus));
233 rc = VERR_INVALID_CPU_INDEX;
234 }
235 }
236 else
237 {
238 AssertMsgFailed(("idApic=%u\n", idApic));
239 rc = VERR_INVALID_CPU_ID;
240 }
241 if (pfDeltaApplied)
242 *pfDeltaApplied = false;
243 return rc;
244}
245
246
247/*
248 *
249 * GIP Mapping and Unmapping Related Code.
250 * GIP Mapping and Unmapping Related Code.
251 * GIP Mapping and Unmapping Related Code.
252 *
253 *
254 */
255
256
257/**
258 * (Re-)initializes the per-cpu structure prior to starting or resuming the GIP
259 * updating.
260 *
261 * @param pGip Pointer to the GIP.
262 * @param pGipCpu The per CPU structure for this CPU.
263 * @param u64NanoTS The current time.
264 */
265static void supdrvGipReInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS)
266{
267 /*
268 * Here we don't really care about applying the TSC delta. The re-initialization of this
269 * value is not relevant especially while (re)starting the GIP as the first few ones will
270 * be ignored anyway, see supdrvGipDoUpdateCpu().
271 */
272 pGipCpu->u64TSC = ASMReadTSC() - pGipCpu->u32UpdateIntervalTSC;
273 pGipCpu->u64NanoTS = u64NanoTS;
274}
275
276
277/**
278 * Set the current TSC and NanoTS value for the CPU.
279 *
280 * @param idCpu The CPU ID. Unused - we have to use the APIC ID.
281 * @param pvUser1 Pointer to the ring-0 GIP mapping.
282 * @param pvUser2 Pointer to the variable holding the current time.
283 */
284static DECLCALLBACK(void) supdrvGipReInitCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
285{
286 PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser1;
287 unsigned iCpu = pGip->aiCpuFromApicId[ASMGetApicId()];
288
289 if (RT_LIKELY(iCpu < pGip->cCpus && pGip->aCPUs[iCpu].idCpu == idCpu))
290 supdrvGipReInitCpu(pGip, &pGip->aCPUs[iCpu], *(uint64_t *)pvUser2);
291
292 NOREF(pvUser2);
293 NOREF(idCpu);
294}
295
296
297/**
298 * State structure for supdrvGipDetectGetGipCpuCallback.
299 */
300typedef struct SUPDRVGIPDETECTGETCPU
301{
302 /** Bitmap of APIC IDs that has been seen (initialized to zero).
303 * Used to detect duplicate APIC IDs (paranoia). */
304 uint8_t volatile bmApicId[256 / 8];
305 /** Mask of supported GIP CPU getter methods (SUPGIPGETCPU_XXX) (all bits set
306 * initially). The callback clears the methods not detected. */
307 uint32_t volatile fSupported;
308 /** The first callback detecting any kind of range issues (initialized to
309 * NIL_RTCPUID). */
310 RTCPUID volatile idCpuProblem;
311} SUPDRVGIPDETECTGETCPU;
312/** Pointer to state structure for supdrvGipDetectGetGipCpuCallback. */
313typedef SUPDRVGIPDETECTGETCPU *PSUPDRVGIPDETECTGETCPU;
314
315
316/**
317 * Checks for alternative ways of getting the CPU ID.
318 *
319 * This also checks the APIC ID, CPU ID and CPU set index values against the
320 * GIP tables.
321 *
322 * @param idCpu The CPU ID. Unused - we have to use the APIC ID.
323 * @param pvUser1 Pointer to the state structure.
324 * @param pvUser2 Pointer to the GIP.
325 */
326static DECLCALLBACK(void) supdrvGipDetectGetGipCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
327{
328 PSUPDRVGIPDETECTGETCPU pState = (PSUPDRVGIPDETECTGETCPU)pvUser1;
329 PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser2;
330 uint32_t fSupported = 0;
331 uint16_t idApic;
332 int iCpuSet;
333
334 AssertMsg(idCpu == RTMpCpuId(), ("idCpu=%#x RTMpCpuId()=%#x\n", idCpu, RTMpCpuId())); /* paranoia^3 */
335
336 /*
337 * Check that the CPU ID and CPU set index are interchangable.
338 */
339 iCpuSet = RTMpCpuIdToSetIndex(idCpu);
340 if ((RTCPUID)iCpuSet == idCpu)
341 {
342 AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
343 if ( iCpuSet >= 0
344 && iCpuSet < RTCPUSET_MAX_CPUS
345 && RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS))
346 {
347 /*
348 * Check whether the IDTR.LIMIT contains a CPU number.
349 */
350#ifdef RT_ARCH_X86
351 uint16_t const cbIdt = sizeof(X86DESC64SYSTEM) * 256;
352#else
353 uint16_t const cbIdt = sizeof(X86DESCGATE) * 256;
354#endif
355 RTIDTR Idtr;
356 ASMGetIDTR(&Idtr);
357 if (Idtr.cbIdt >= cbIdt)
358 {
359 uint32_t uTmp = Idtr.cbIdt - cbIdt;
360 uTmp &= RTCPUSET_MAX_CPUS - 1;
361 if (uTmp == idCpu)
362 {
363 RTIDTR Idtr2;
364 ASMGetIDTR(&Idtr2);
365 if (Idtr2.cbIdt == Idtr.cbIdt)
366 fSupported |= SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS;
367 }
368 }
369
370 /*
371 * Check whether RDTSCP is an option.
372 */
373 if (ASMHasCpuId())
374 {
375 if ( ASMIsValidExtRange(ASMCpuId_EAX(UINT32_C(0x80000000)))
376 && (ASMCpuId_EDX(UINT32_C(0x80000001)) & X86_CPUID_EXT_FEATURE_EDX_RDTSCP) )
377 {
378 uint32_t uAux;
379 ASMReadTscWithAux(&uAux);
380 if ((uAux & (RTCPUSET_MAX_CPUS - 1)) == idCpu)
381 {
382 ASMNopPause();
383 ASMReadTscWithAux(&uAux);
384 if ((uAux & (RTCPUSET_MAX_CPUS - 1)) == idCpu)
385 fSupported |= SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS;
386 }
387 }
388 }
389 }
390 }
391
392 /*
393 * Check that the APIC ID is unique.
394 */
395 idApic = ASMGetApicId();
396 if (RT_LIKELY( idApic < RT_ELEMENTS(pGip->aiCpuFromApicId)
397 && !ASMAtomicBitTestAndSet(pState->bmApicId, idApic)))
398 fSupported |= SUPGIPGETCPU_APIC_ID;
399 else
400 {
401 AssertCompile(sizeof(pState->bmApicId) * 8 == RT_ELEMENTS(pGip->aiCpuFromApicId));
402 ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
403 LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - duplicate APIC ID.\n",
404 idCpu, iCpuSet, idApic));
405 }
406
407 /*
408 * Check that the iCpuSet is within the expected range.
409 */
410 if (RT_UNLIKELY( iCpuSet < 0
411 || (unsigned)iCpuSet >= RTCPUSET_MAX_CPUS
412 || (unsigned)iCpuSet >= RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
413 {
414 ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
415 LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU set index is out of range.\n",
416 idCpu, iCpuSet, idApic));
417 }
418 else
419 {
420 RTCPUID idCpu2 = RTMpCpuIdFromSetIndex(iCpuSet);
421 if (RT_UNLIKELY(idCpu2 != idCpu))
422 {
423 ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
424 LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU id/index roundtrip problem: %#x\n",
425 idCpu, iCpuSet, idApic, idCpu2));
426 }
427 }
428
429 /*
430 * Update the supported feature mask before we return.
431 */
432 ASMAtomicAndU32(&pState->fSupported, fSupported);
433
434 NOREF(pvUser2);
435}
436
437
438/**
439 * Increase the timer freqency on hosts where this is possible (NT).
440 *
441 * The idea is that more interrupts is better for us... Also, it's better than
442 * we increase the timer frequence, because we might end up getting inaccurate
443 * callbacks if someone else does it.
444 *
445 * @param pDevExt Sets u32SystemTimerGranularityGrant if increased.
446 */
447static void supdrvGipRequestHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
448{
449 if (pDevExt->u32SystemTimerGranularityGrant == 0)
450 {
451 uint32_t u32SystemResolution;
452 if ( RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 976563 /* 1024 HZ */, &u32SystemResolution))
453 || RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1000000 /* 1000 HZ */, &u32SystemResolution))
454 || RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1953125 /* 512 HZ */, &u32SystemResolution))
455 || RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 2000000 /* 500 HZ */, &u32SystemResolution))
456 )
457 {
458 Assert(RTTimerGetSystemGranularity() <= u32SystemResolution);
459 pDevExt->u32SystemTimerGranularityGrant = u32SystemResolution;
460 }
461 }
462}
463
464
465/**
466 * Undoes supdrvGipRequestHigherTimerFrequencyFromSystem.
467 *
468 * @param pDevExt Clears u32SystemTimerGranularityGrant.
469 */
470static void supdrvGipReleaseHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
471{
472 if (pDevExt->u32SystemTimerGranularityGrant)
473 {
474 int rc2 = RTTimerReleaseSystemGranularity(pDevExt->u32SystemTimerGranularityGrant);
475 AssertRC(rc2);
476 pDevExt->u32SystemTimerGranularityGrant = 0;
477 }
478}
479
480
481/**
482 * Maps the GIP into userspace and/or get the physical address of the GIP.
483 *
484 * @returns IPRT status code.
485 * @param pSession Session to which the GIP mapping should belong.
486 * @param ppGipR3 Where to store the address of the ring-3 mapping. (optional)
487 * @param pHCPhysGip Where to store the physical address. (optional)
488 *
489 * @remark There is no reference counting on the mapping, so one call to this function
490 * count globally as one reference. One call to SUPR0GipUnmap() is will unmap GIP
491 * and remove the session as a GIP user.
492 */
493SUPR0DECL(int) SUPR0GipMap(PSUPDRVSESSION pSession, PRTR3PTR ppGipR3, PRTHCPHYS pHCPhysGip)
494{
495 int rc;
496 PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
497 RTR3PTR pGipR3 = NIL_RTR3PTR;
498 RTHCPHYS HCPhys = NIL_RTHCPHYS;
499 LogFlow(("SUPR0GipMap: pSession=%p ppGipR3=%p pHCPhysGip=%p\n", pSession, ppGipR3, pHCPhysGip));
500
501 /*
502 * Validate
503 */
504 AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
505 AssertPtrNullReturn(ppGipR3, VERR_INVALID_POINTER);
506 AssertPtrNullReturn(pHCPhysGip, VERR_INVALID_POINTER);
507
508#ifdef SUPDRV_USE_MUTEX_FOR_GIP
509 RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
510#else
511 RTSemFastMutexRequest(pDevExt->mtxGip);
512#endif
513 if (pDevExt->pGip)
514 {
515 /*
516 * Map it?
517 */
518 rc = VINF_SUCCESS;
519 if (ppGipR3)
520 {
521 if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
522 rc = RTR0MemObjMapUser(&pSession->GipMapObjR3, pDevExt->GipMemObj, (RTR3PTR)-1, 0,
523 RTMEM_PROT_READ, RTR0ProcHandleSelf());
524 if (RT_SUCCESS(rc))
525 pGipR3 = RTR0MemObjAddressR3(pSession->GipMapObjR3);
526 }
527
528 /*
529 * Get physical address.
530 */
531 if (pHCPhysGip && RT_SUCCESS(rc))
532 HCPhys = pDevExt->HCPhysGip;
533
534 /*
535 * Reference globally.
536 */
537 if (!pSession->fGipReferenced && RT_SUCCESS(rc))
538 {
539 pSession->fGipReferenced = 1;
540 pDevExt->cGipUsers++;
541 if (pDevExt->cGipUsers == 1)
542 {
543 PSUPGLOBALINFOPAGE pGipR0 = pDevExt->pGip;
544 uint64_t u64NanoTS;
545
546 /*
547 * GIP starts/resumes updating again. On windows we bump the
548 * host timer frequency to make sure we don't get stuck in guest
549 * mode and to get better timer (and possibly clock) accuracy.
550 */
551 LogFlow(("SUPR0GipMap: Resumes GIP updating\n"));
552
553 supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
554
555 /*
556 * document me
557 */
558 if (pGipR0->aCPUs[0].u32TransactionId != 2 /* not the first time */)
559 {
560 unsigned i;
561 for (i = 0; i < pGipR0->cCpus; i++)
562 ASMAtomicUoWriteU32(&pGipR0->aCPUs[i].u32TransactionId,
563 (pGipR0->aCPUs[i].u32TransactionId + GIP_UPDATEHZ_RECALC_FREQ * 2)
564 & ~(GIP_UPDATEHZ_RECALC_FREQ * 2 - 1));
565 ASMAtomicWriteU64(&pGipR0->u64NanoTSLastUpdateHz, 0);
566 }
567
568 /*
569 * document me
570 */
571 u64NanoTS = RTTimeSystemNanoTS() - pGipR0->u32UpdateIntervalNS;
572 if ( pGipR0->u32Mode == SUPGIPMODE_INVARIANT_TSC
573 || pGipR0->u32Mode == SUPGIPMODE_SYNC_TSC
574 || RTMpGetOnlineCount() == 1)
575 supdrvGipReInitCpu(pGipR0, &pGipR0->aCPUs[0], u64NanoTS);
576 else
577 RTMpOnAll(supdrvGipReInitCpuCallback, pGipR0, &u64NanoTS);
578
579 /*
580 * Detect alternative ways to figure the CPU ID in ring-3 and
581 * raw-mode context. Check the sanity of the APIC IDs, CPU IDs,
582 * and CPU set indexes while we're at it.
583 */
584 if (RT_SUCCESS(rc))
585 {
586 SUPDRVGIPDETECTGETCPU DetectState;
587 RT_BZERO((void *)&DetectState.bmApicId, sizeof(DetectState.bmApicId));
588 DetectState.fSupported = UINT32_MAX;
589 DetectState.idCpuProblem = NIL_RTCPUID;
590 rc = RTMpOnAll(supdrvGipDetectGetGipCpuCallback, &DetectState, pGipR0);
591 if (DetectState.idCpuProblem == NIL_RTCPUID)
592 {
593 if ( DetectState.fSupported != UINT32_MAX
594 && DetectState.fSupported != 0)
595 {
596 if (pGipR0->fGetGipCpu != DetectState.fSupported)
597 {
598 pGipR0->fGetGipCpu = DetectState.fSupported;
599 LogRel(("SUPR0GipMap: fGetGipCpu=%#x\n", DetectState.fSupported));
600 }
601 }
602 else
603 {
604 LogRel(("SUPR0GipMap: No supported ways of getting the APIC ID or CPU number in ring-3! (%#x)\n",
605 DetectState.fSupported));
606 rc = VERR_UNSUPPORTED_CPU;
607 }
608 }
609 else
610 {
611 LogRel(("SUPR0GipMap: APIC ID, CPU ID or CPU set index problem detected on CPU #%u (%#x)!\n",
612 DetectState.idCpuProblem, DetectState.idCpuProblem));
613 rc = VERR_INVALID_CPU_ID;
614 }
615 }
616
617 /*
618 * Start the GIP timer if all is well..
619 */
620 if (RT_SUCCESS(rc))
621 {
622#ifndef DO_NOT_START_GIP
623 rc = RTTimerStart(pDevExt->pGipTimer, 0 /* fire ASAP */); AssertRC(rc);
624#endif
625 rc = VINF_SUCCESS;
626 }
627
628 /*
629 * Bail out on error.
630 */
631 if (RT_FAILURE(rc))
632 {
633 LogRel(("SUPR0GipMap: failed rc=%Rrc\n", rc));
634 pDevExt->cGipUsers = 0;
635 pSession->fGipReferenced = 0;
636 if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
637 {
638 int rc2 = RTR0MemObjFree(pSession->GipMapObjR3, false); AssertRC(rc2);
639 if (RT_SUCCESS(rc2))
640 pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
641 }
642 HCPhys = NIL_RTHCPHYS;
643 pGipR3 = NIL_RTR3PTR;
644 }
645 }
646 }
647 }
648 else
649 {
650 rc = VERR_GENERAL_FAILURE;
651 Log(("SUPR0GipMap: GIP is not available!\n"));
652 }
653#ifdef SUPDRV_USE_MUTEX_FOR_GIP
654 RTSemMutexRelease(pDevExt->mtxGip);
655#else
656 RTSemFastMutexRelease(pDevExt->mtxGip);
657#endif
658
659 /*
660 * Write returns.
661 */
662 if (pHCPhysGip)
663 *pHCPhysGip = HCPhys;
664 if (ppGipR3)
665 *ppGipR3 = pGipR3;
666
667#ifdef DEBUG_DARWIN_GIP
668 OSDBGPRINT(("SUPR0GipMap: returns %d *pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)pGipR3));
669#else
670 LogFlow(( "SUPR0GipMap: returns %d *pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)pGipR3));
671#endif
672 return rc;
673}
674
675
676/**
677 * Unmaps any user mapping of the GIP and terminates all GIP access
678 * from this session.
679 *
680 * @returns IPRT status code.
681 * @param pSession Session to which the GIP mapping should belong.
682 */
683SUPR0DECL(int) SUPR0GipUnmap(PSUPDRVSESSION pSession)
684{
685 int rc = VINF_SUCCESS;
686 PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
687#ifdef DEBUG_DARWIN_GIP
688 OSDBGPRINT(("SUPR0GipUnmap: pSession=%p pGip=%p GipMapObjR3=%p\n",
689 pSession,
690 pSession->GipMapObjR3 != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pSession->GipMapObjR3) : NULL,
691 pSession->GipMapObjR3));
692#else
693 LogFlow(("SUPR0GipUnmap: pSession=%p\n", pSession));
694#endif
695 AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
696
697#ifdef SUPDRV_USE_MUTEX_FOR_GIP
698 RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
699#else
700 RTSemFastMutexRequest(pDevExt->mtxGip);
701#endif
702
703 /*
704 * Unmap anything?
705 */
706 if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
707 {
708 rc = RTR0MemObjFree(pSession->GipMapObjR3, false);
709 AssertRC(rc);
710 if (RT_SUCCESS(rc))
711 pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
712 }
713
714 /*
715 * Dereference global GIP.
716 */
717 if (pSession->fGipReferenced && !rc)
718 {
719 pSession->fGipReferenced = 0;
720 if ( pDevExt->cGipUsers > 0
721 && !--pDevExt->cGipUsers)
722 {
723 LogFlow(("SUPR0GipUnmap: Suspends GIP updating\n"));
724#ifndef DO_NOT_START_GIP
725 rc = RTTimerStop(pDevExt->pGipTimer); AssertRC(rc); rc = VINF_SUCCESS;
726#endif
727 supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
728 }
729 }
730
731#ifdef SUPDRV_USE_MUTEX_FOR_GIP
732 RTSemMutexRelease(pDevExt->mtxGip);
733#else
734 RTSemFastMutexRelease(pDevExt->mtxGip);
735#endif
736
737 return rc;
738}
739
740
741/**
742 * Gets the GIP pointer.
743 *
744 * @returns Pointer to the GIP or NULL.
745 */
746SUPDECL(PSUPGLOBALINFOPAGE) SUPGetGIP(void)
747{
748 return g_pSUPGlobalInfoPage;
749}
750
751
752
753
754
755/*
756 *
757 *
758 * GIP Initialization, Termination and CPU Offline / Online Related Code.
759 * GIP Initialization, Termination and CPU Offline / Online Related Code.
760 * GIP Initialization, Termination and CPU Offline / Online Related Code.
761 *
762 *
763 */
764
765
766/**
767 * Timer callback function for TSC frequency refinement in invariant GIP mode.
768 *
769 * @param pTimer The timer.
770 * @param pvUser Opaque pointer to the device instance data.
771 * @param iTick The timer tick.
772 */
773static DECLCALLBACK(void) supdrvInitAsyncRefineTscTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
774{
775 PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
776 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
777 bool fDeltaApplied = false;
778 uint8_t idApic;
779 uint64_t u64DeltaNanoTS;
780 uint64_t u64DeltaTsc;
781 uint64_t u64NanoTS;
782 uint64_t u64Tsc;
783 RTCCUINTREG uFlags;
784
785 /* Paranoia. */
786 Assert(pGip);
787 Assert(pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC);
788
789#if !defined(RT_OS_OS2) /* PORTME: Disable if timers are called from clock interrupt handler or with interrupts disabled. */
790 u64NanoTS = RTTimeSystemNanoTS();
791 while (RTTimeSystemNanoTS() == u64NanoTS)
792 ASMNopPause();
793#endif
794 uFlags = ASMIntDisableFlags();
795 idApic = ASMGetApicId();
796 u64Tsc = ASMReadTSC();
797 u64NanoTS = RTTimeSystemNanoTS();
798 ASMSetFlags(uFlags);
799 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
800 supdrvTscDeltaApply(pGip, &u64Tsc, idApic, &fDeltaApplied);
801 u64DeltaNanoTS = u64NanoTS - pDevExt->u64NanoTSAnchor;
802 u64DeltaTsc = u64Tsc - pDevExt->u64TscAnchor;
803
804 if (RT_UNLIKELY( pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO
805 && !fDeltaApplied))
806 {
807 Log(("vboxdrv: failed to refine TSC frequency as TSC-deltas unavailable after %d seconds!\n",
808 GIP_TSC_REFINE_INTERVAL));
809 return;
810 }
811
812 /* Calculate the TSC frequency. */
813 if ( u64DeltaTsc < UINT64_MAX / RT_NS_1SEC
814 && u64DeltaNanoTS < UINT32_MAX)
815 pGip->u64CpuHz = ASMMultU64ByU32DivByU32(u64DeltaTsc, RT_NS_1SEC, (uint32_t)u64DeltaNanoTS);
816 else
817 {
818 RTUINT128U CpuHz, Tmp, Divisor;
819 CpuHz.s.Lo = CpuHz.s.Hi = 0;
820 RTUInt128MulU64ByU64(&Tmp, u64DeltaTsc, RT_NS_1SEC_64);
821 RTUInt128Div(&CpuHz, &Tmp, RTUInt128AssignU64(&Divisor, u64DeltaNanoTS));
822 pGip->u64CpuHz = CpuHz.s.Lo;
823 }
824
825 /* Update rest of GIP. */
826 Assert(pGip->u32Mode != SUPGIPMODE_ASYNC_TSC); /* See SUPGetCpuHzFromGIP().*/
827 pGip->aCPUs[0].u64CpuHz = pGip->u64CpuHz;
828}
829
830
831/**
832 * Starts the TSC-frequency refinement phase asynchronously.
833 *
834 * @param pDevExt Pointer to the device instance data.
835 */
836static void supdrvGipInitAsyncRefineTscFreq(PSUPDRVDEVEXT pDevExt)
837{
838 uint64_t u64NanoTS;
839 RTCCUINTREG uFlags;
840 uint8_t idApic;
841 int rc;
842 PSUPGLOBALINFOPAGE pGip;
843
844 /* Validate. */
845 Assert(pDevExt);
846 Assert(pDevExt->pGip);
847 pGip = pDevExt->pGip;
848
849#ifdef SUPDRV_USE_TSC_DELTA_THREAD
850 /*
851 * If the TSC-delta thread is created, wait until it's done calculating
852 * the TSC-deltas on the relevant online CPUs before we start the TSC refinement.
853 */
854 if ( pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED
855 && ASMAtomicReadS32(&pDevExt->rcTscDelta) == VERR_NOT_AVAILABLE)
856 {
857 rc = supdrvTscDeltaThreadWaitForOnlineCpus(pDevExt);
858 if (rc == VERR_TIMEOUT)
859 {
860 SUPR0Printf("vboxdrv: Skipping refinement of TSC frequency as TSC-delta measurement timed out!\n");
861 return;
862 }
863 }
864#endif
865
866 /*
867 * Record the TSC and NanoTS as the starting anchor point for refinement of the
868 * TSC. We deliberately avoid using SUPReadTSC() here as we want to keep the
869 * reading of the TSC and the NanoTS as close as possible.
870 */
871 u64NanoTS = RTTimeSystemNanoTS();
872 while (RTTimeSystemNanoTS() == u64NanoTS)
873 ASMNopPause();
874 uFlags = ASMIntDisableFlags();
875 idApic = ASMGetApicId();
876 pDevExt->u64TscAnchor = ASMReadTSC();
877 pDevExt->u64NanoTSAnchor = RTTimeSystemNanoTS();
878 ASMSetFlags(uFlags);
879 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
880 supdrvTscDeltaApply(pGip, &pDevExt->u64TscAnchor, idApic, NULL /* pfDeltaApplied */);
881
882 rc = RTTimerCreateEx(&pDevExt->pTscRefineTimer, 0 /* one-shot */, RTTIMER_FLAGS_CPU_ANY,
883 supdrvInitAsyncRefineTscTimer, pDevExt);
884 if (RT_SUCCESS(rc))
885 {
886 /*
887 * Refine the TSC frequency measurement over a long interval. Ideally, we want to keep the
888 * interval as small as possible while gaining the most consistent and accurate frequency
889 * (compared to what the host OS might have measured).
890 *
891 * In theory, we gain more accuracy with longer intervals, but we want VMs to startup with the
892 * same TSC frequency whenever possible so we need to keep the interval short.
893 */
894 rc = RTTimerStart(pDevExt->pTscRefineTimer, GIP_TSC_REFINE_INTERVAL * RT_NS_1SEC_64);
895 AssertRC(rc);
896 }
897 else
898 OSDBGPRINT(("RTTimerCreateEx failed to create one-shot timer. rc=%Rrc\n", rc));
899}
900
901
902/**
903 * Measures the TSC frequency of the system.
904 *
905 * Uses a busy-wait method for the async. case as it is intended to help push
906 * the CPU frequency up, while for the invariant cases using a sleeping method.
907 *
908 * The TSC frequency can vary on systems which are not reported as invariant.
909 * On such systems the object of this function is to find out what the nominal,
910 * maximum TSC frequency under 'normal' CPU operation.
911 *
912 * @returns VBox status code.
913 * @param pDevExt Pointer to the device instance.
914 *
915 * @remarks Must be called only -after- measuring the TSC deltas.
916 */
917static int supdrvGipInitMeasureTscFreq(PSUPDRVDEVEXT pDevExt)
918{
919 int cTriesLeft = 4;
920 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
921
922 /* Assert order. */
923 AssertReturn(pGip, VERR_INVALID_PARAMETER);
924 AssertReturn(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC, VERR_WRONG_ORDER);
925
926 while (cTriesLeft-- > 0)
927 {
928 RTCCUINTREG uFlags;
929 uint64_t u64NanoTsBefore;
930 uint64_t u64NanoTsAfter;
931 uint64_t u64TscBefore;
932 uint64_t u64TscAfter;
933 uint8_t idApicBefore;
934 uint8_t idApicAfter;
935
936 /*
937 * Synchronize with the host OS clock tick before reading the TSC.
938 * Especially important on older Windows version where the granularity is terrible.
939 */
940 u64NanoTsBefore = RTTimeSystemNanoTS();
941 while (RTTimeSystemNanoTS() == u64NanoTsBefore)
942 ASMNopPause();
943
944 uFlags = ASMIntDisableFlags();
945 idApicBefore = ASMGetApicId();
946 u64TscBefore = ASMReadTSC();
947 u64NanoTsBefore = RTTimeSystemNanoTS();
948 ASMSetFlags(uFlags);
949
950 if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
951 {
952 /*
953 * Sleep-wait since the TSC frequency is constant, it eases host load.
954 * Shorter interval produces more variance in the frequency (esp. Windows).
955 */
956 RTThreadSleep(200);
957 u64NanoTsAfter = RTTimeSystemNanoTS();
958 while (RTTimeSystemNanoTS() == u64NanoTsAfter)
959 ASMNopPause();
960 u64NanoTsAfter = RTTimeSystemNanoTS();
961 }
962 else
963 {
964 /* Busy-wait keeping the frequency up and measure. */
965 for (;;)
966 {
967 u64NanoTsAfter = RTTimeSystemNanoTS();
968 if (u64NanoTsAfter < RT_NS_100MS + u64NanoTsBefore)
969 ASMNopPause();
970 else
971 break;
972 }
973 }
974
975 uFlags = ASMIntDisableFlags();
976 idApicAfter = ASMGetApicId();
977 u64TscAfter = ASMReadTSC();
978 ASMSetFlags(uFlags);
979
980 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
981 {
982 int rc;
983 bool fAppliedBefore;
984 bool fAppliedAfter;
985 rc = supdrvTscDeltaApply(pGip, &u64TscBefore, idApicBefore, &fAppliedBefore); AssertRCReturn(rc, rc);
986 rc = supdrvTscDeltaApply(pGip, &u64TscAfter, idApicAfter, &fAppliedAfter); AssertRCReturn(rc, rc);
987
988 if ( !fAppliedBefore
989 || !fAppliedAfter)
990 {
991#ifdef SUPDRV_USE_TSC_DELTA_THREAD
992 /*
993 * The TSC-delta measurements are kicked-off asynchronously as each host CPU is initialized.
994 * Therefore, if we failed to have a delta for the CPU(s) we were scheduled on (idApicBefore
995 * and idApicAfter) then wait until we have TSC-delta measurements for all online CPUs and
996 * proceed. This should be triggered just once if we're rather unlucky.
997 */
998 rc = supdrvTscDeltaThreadWaitForOnlineCpus(pDevExt);
999 if (rc == VERR_TIMEOUT)
1000 {
1001 SUPR0Printf("vboxdrv: supdrvGipInitMeasureTscFreq: timedout waiting for TSC-delta measurements.\n");
1002 return VERR_SUPDRV_TSC_FREQ_MEASUREMENT_FAILED;
1003 }
1004#else
1005 SUPR0Printf("vboxdrv: supdrvGipInitMeasureTscFreq: idApicBefore=%u idApicAfter=%u cTriesLeft=%u\n",
1006 idApicBefore, idApicAfter, cTriesLeft);
1007#endif
1008 continue;
1009 }
1010 }
1011
1012 /*
1013 * Update GIP.
1014 */
1015 pGip->u64CpuHz = ((u64TscAfter - u64TscBefore) * RT_NS_1SEC_64) / (u64NanoTsAfter - u64NanoTsBefore);
1016 if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
1017 pGip->aCPUs[0].u64CpuHz = pGip->u64CpuHz;
1018 return VINF_SUCCESS;
1019 }
1020
1021 return VERR_SUPDRV_TSC_FREQ_MEASUREMENT_FAILED;
1022}
1023
1024
1025/**
1026 * Finds our (@a idCpu) entry, or allocates a new one if not found.
1027 *
1028 * @returns Index of the CPU in the cache set.
1029 * @param pGip The GIP.
1030 * @param idCpu The CPU ID.
1031 */
1032static uint32_t supdrvGipFindOrAllocCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
1033{
1034 uint32_t i, cTries;
1035
1036 /*
1037 * ASSUMES that CPU IDs are constant.
1038 */
1039 for (i = 0; i < pGip->cCpus; i++)
1040 if (pGip->aCPUs[i].idCpu == idCpu)
1041 return i;
1042
1043 cTries = 0;
1044 do
1045 {
1046 for (i = 0; i < pGip->cCpus; i++)
1047 {
1048 bool fRc;
1049 ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc);
1050 if (fRc)
1051 return i;
1052 }
1053 } while (cTries++ < 32);
1054 AssertReleaseFailed();
1055 return i - 1;
1056}
1057
1058
1059/**
1060 * The calling CPU should be accounted as online, update GIP accordingly.
1061 *
1062 * This is used by supdrvGipCreate() as well as supdrvGipMpEvent().
1063 *
1064 * @param pDevExt The device extension.
1065 * @param idCpu The CPU ID.
1066 */
1067static void supdrvGipMpEventOnlineOrInitOnCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
1068{
1069 int iCpuSet = 0;
1070 uint16_t idApic = UINT16_MAX;
1071 uint32_t i = 0;
1072 uint64_t u64NanoTS = 0;
1073 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1074
1075 AssertPtrReturnVoid(pGip);
1076 AssertRelease(idCpu == RTMpCpuId());
1077 Assert(pGip->cPossibleCpus == RTMpGetCount());
1078
1079 /*
1080 * Do this behind a spinlock with interrupts disabled as this can fire
1081 * on all CPUs simultaneously, see @bugref{6110}.
1082 */
1083 RTSpinlockAcquire(pDevExt->hGipSpinlock);
1084
1085 /*
1086 * Update the globals.
1087 */
1088 ASMAtomicWriteU16(&pGip->cPresentCpus, RTMpGetPresentCount());
1089 ASMAtomicWriteU16(&pGip->cOnlineCpus, RTMpGetOnlineCount());
1090 iCpuSet = RTMpCpuIdToSetIndex(idCpu);
1091 if (iCpuSet >= 0)
1092 {
1093 Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
1094 RTCpuSetAddByIndex(&pGip->OnlineCpuSet, iCpuSet);
1095 RTCpuSetAddByIndex(&pGip->PresentCpuSet, iCpuSet);
1096 }
1097
1098 /*
1099 * Update the entry.
1100 */
1101 u64NanoTS = RTTimeSystemNanoTS() - pGip->u32UpdateIntervalNS;
1102 i = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
1103 supdrvGipInitCpu(pDevExt, pGip, &pGip->aCPUs[i], u64NanoTS);
1104 idApic = ASMGetApicId();
1105 ASMAtomicWriteU16(&pGip->aCPUs[i].idApic, idApic);
1106 ASMAtomicWriteS16(&pGip->aCPUs[i].iCpuSet, (int16_t)iCpuSet);
1107 ASMAtomicWriteSize(&pGip->aCPUs[i].idCpu, idCpu);
1108
1109 /*
1110 * Update the APIC ID and CPU set index mappings.
1111 */
1112 ASMAtomicWriteU16(&pGip->aiCpuFromApicId[idApic], i);
1113 ASMAtomicWriteU16(&pGip->aiCpuFromCpuSetIdx[iCpuSet], i);
1114
1115 /* Update the Mp online/offline counter. */
1116 ASMAtomicIncU32(&pDevExt->cMpOnOffEvents);
1117
1118 /* Add this CPU to the set of CPUs for which we need to calculate their TSC-deltas. */
1119 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1120 {
1121 RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, iCpuSet);
1122#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1123 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
1124 if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
1125 || pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
1126 {
1127 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
1128 }
1129 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
1130#endif
1131 }
1132
1133 /* commit it */
1134 ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE);
1135
1136 RTSpinlockRelease(pDevExt->hGipSpinlock);
1137}
1138
1139
1140/**
1141 * The CPU should be accounted as offline, update the GIP accordingly.
1142 *
1143 * This is used by supdrvGipMpEvent.
1144 *
1145 * @param pDevExt The device extension.
1146 * @param idCpu The CPU ID.
1147 */
1148static void supdrvGipMpEventOffline(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
1149{
1150 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1151 int iCpuSet;
1152 unsigned i;
1153
1154 AssertPtrReturnVoid(pGip);
1155 RTSpinlockAcquire(pDevExt->hGipSpinlock);
1156
1157 iCpuSet = RTMpCpuIdToSetIndex(idCpu);
1158 AssertReturnVoid(iCpuSet >= 0);
1159
1160 i = pGip->aiCpuFromCpuSetIdx[iCpuSet];
1161 AssertReturnVoid(i < pGip->cCpus);
1162 AssertReturnVoid(pGip->aCPUs[i].idCpu == idCpu);
1163
1164 Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
1165 RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet);
1166
1167 /* Update the Mp online/offline counter. */
1168 ASMAtomicIncU32(&pDevExt->cMpOnOffEvents);
1169
1170 /* If we are the initiator going offline while measuring the TSC delta, unspin other waiting CPUs! */
1171 if (ASMAtomicReadU32(&pDevExt->idTscDeltaInitiator) == idCpu)
1172 {
1173 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_START);
1174 ASMAtomicWriteU64(&pGip->aCPUs[i].u64TSCSample, ~GIP_TSC_DELTA_RSVD);
1175 }
1176
1177 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1178 {
1179 /* Reset the TSC delta, we will recalculate it lazily. */
1180 ASMAtomicWriteS64(&pGip->aCPUs[i].i64TSCDelta, INT64_MAX);
1181 /* Remove this CPU from the set of CPUs that we have obtained the TSC deltas. */
1182 RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, iCpuSet);
1183 }
1184
1185 /* commit it */
1186 ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_OFFLINE);
1187
1188 RTSpinlockRelease(pDevExt->hGipSpinlock);
1189}
1190
1191
1192/**
1193 * Multiprocessor event notification callback.
1194 *
1195 * This is used to make sure that the GIP master gets passed on to
1196 * another CPU. It also updates the associated CPU data.
1197 *
1198 * @param enmEvent The event.
1199 * @param idCpu The cpu it applies to.
1200 * @param pvUser Pointer to the device extension.
1201 *
1202 * @remarks This function -must- fire on the newly online'd CPU for the
1203 * RTMPEVENT_ONLINE case and can fire on any CPU for the
1204 * RTMPEVENT_OFFLINE case.
1205 */
1206static DECLCALLBACK(void) supdrvGipMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
1207{
1208 PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
1209 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1210
1211 AssertRelease(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1212
1213 /*
1214 * Update the GIP CPU data.
1215 */
1216 if (pGip)
1217 {
1218 switch (enmEvent)
1219 {
1220 case RTMPEVENT_ONLINE:
1221 AssertRelease(idCpu == RTMpCpuId());
1222 supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
1223 break;
1224 case RTMPEVENT_OFFLINE:
1225 supdrvGipMpEventOffline(pDevExt, idCpu);
1226 break;
1227 }
1228 }
1229
1230 /*
1231 * Make sure there is a master GIP.
1232 */
1233 if (enmEvent == RTMPEVENT_OFFLINE)
1234 {
1235 RTCPUID idGipMaster = ASMAtomicReadU32(&pDevExt->idGipMaster);
1236 if (idGipMaster == idCpu)
1237 {
1238 /*
1239 * The GIP master is going offline, find a new one.
1240 */
1241 bool fIgnored;
1242 unsigned i;
1243 RTCPUID idNewGipMaster = NIL_RTCPUID;
1244 RTCPUSET OnlineCpus;
1245 RTMpGetOnlineSet(&OnlineCpus);
1246
1247 for (i = 0; i < RTCPUSET_MAX_CPUS; i++)
1248 if (RTCpuSetIsMemberByIndex(&OnlineCpus, i))
1249 {
1250 RTCPUID idCurCpu = RTMpCpuIdFromSetIndex(i);
1251 if (idCurCpu != idGipMaster)
1252 {
1253 idNewGipMaster = idCurCpu;
1254 break;
1255 }
1256 }
1257
1258 Log(("supdrvGipMpEvent: Gip master %#lx -> %#lx\n", (long)idGipMaster, (long)idNewGipMaster));
1259 ASMAtomicCmpXchgSize(&pDevExt->idGipMaster, idNewGipMaster, idGipMaster, fIgnored);
1260 NOREF(fIgnored);
1261 }
1262 }
1263}
1264
1265
1266/**
1267 * On CPU initialization callback for RTMpOnAll.
1268 *
1269 * @param idCpu The CPU ID.
1270 * @param pvUser1 The device extension.
1271 * @param pvUser2 The GIP.
1272 */
1273static DECLCALLBACK(void) supdrvGipInitOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
1274{
1275 /* This is good enough, even though it will update some of the globals a
1276 bit to much. */
1277 supdrvGipMpEventOnlineOrInitOnCpu((PSUPDRVDEVEXT)pvUser1, idCpu);
1278}
1279
1280
1281/**
1282 * Callback used by supdrvDetermineAsyncTSC to read the TSC on a CPU.
1283 *
1284 * @param idCpu Ignored.
1285 * @param pvUser1 Where to put the TSC.
1286 * @param pvUser2 Ignored.
1287 */
1288static DECLCALLBACK(void) supdrvGipInitDetermineAsyncTscWorker(RTCPUID idCpu, void *pvUser1, void *pvUser2)
1289{
1290 ASMAtomicWriteU64((uint64_t volatile *)pvUser1, ASMReadTSC());
1291}
1292
1293
1294/**
1295 * Determine if Async GIP mode is required because of TSC drift.
1296 *
1297 * When using the default/normal timer code it is essential that the time stamp counter
1298 * (TSC) runs never backwards, that is, a read operation to the counter should return
1299 * a bigger value than any previous read operation. This is guaranteed by the latest
1300 * AMD CPUs and by newer Intel CPUs which never enter the C2 state (P4). In any other
1301 * case we have to choose the asynchronous timer mode.
1302 *
1303 * @param poffMin Pointer to the determined difference between different
1304 * cores (optional, can be NULL).
1305 * @return false if the time stamp counters appear to be synchronized, true otherwise.
1306 */
1307static bool supdrvGipInitDetermineAsyncTsc(uint64_t *poffMin)
1308{
1309 /*
1310 * Just iterate all the cpus 8 times and make sure that the TSC is
1311 * ever increasing. We don't bother taking TSC rollover into account.
1312 */
1313 int iEndCpu = RTMpGetArraySize();
1314 int iCpu;
1315 int cLoops = 8;
1316 bool fAsync = false;
1317 int rc = VINF_SUCCESS;
1318 uint64_t offMax = 0;
1319 uint64_t offMin = ~(uint64_t)0;
1320 uint64_t PrevTsc = ASMReadTSC();
1321
1322 while (cLoops-- > 0)
1323 {
1324 for (iCpu = 0; iCpu < iEndCpu; iCpu++)
1325 {
1326 uint64_t CurTsc;
1327 rc = RTMpOnSpecific(RTMpCpuIdFromSetIndex(iCpu), supdrvGipInitDetermineAsyncTscWorker, &CurTsc, NULL);
1328 if (RT_SUCCESS(rc))
1329 {
1330 if (CurTsc <= PrevTsc)
1331 {
1332 fAsync = true;
1333 offMin = offMax = PrevTsc - CurTsc;
1334 Log(("supdrvGipInitDetermineAsyncTsc: iCpu=%d cLoops=%d CurTsc=%llx PrevTsc=%llx\n",
1335 iCpu, cLoops, CurTsc, PrevTsc));
1336 break;
1337 }
1338
1339 /* Gather statistics (except the first time). */
1340 if (iCpu != 0 || cLoops != 7)
1341 {
1342 uint64_t off = CurTsc - PrevTsc;
1343 if (off < offMin)
1344 offMin = off;
1345 if (off > offMax)
1346 offMax = off;
1347 Log2(("%d/%d: off=%llx\n", cLoops, iCpu, off));
1348 }
1349
1350 /* Next */
1351 PrevTsc = CurTsc;
1352 }
1353 else if (rc == VERR_NOT_SUPPORTED)
1354 break;
1355 else
1356 AssertMsg(rc == VERR_CPU_NOT_FOUND || rc == VERR_CPU_OFFLINE, ("%d\n", rc));
1357 }
1358
1359 /* broke out of the loop. */
1360 if (iCpu < iEndCpu)
1361 break;
1362 }
1363
1364 if (poffMin)
1365 *poffMin = offMin; /* Almost RTMpOnSpecific profiling. */
1366 Log(("supdrvGipInitDetermineAsyncTsc: returns %d; iEndCpu=%d rc=%d offMin=%llx offMax=%llx\n",
1367 fAsync, iEndCpu, rc, offMin, offMax));
1368#if !defined(RT_OS_SOLARIS) && !defined(RT_OS_OS2) && !defined(RT_OS_WINDOWS)
1369 OSDBGPRINT(("vboxdrv: fAsync=%d offMin=%#lx offMax=%#lx\n", fAsync, (long)offMin, (long)offMax));
1370#endif
1371 return fAsync;
1372}
1373
1374
1375/**
1376 * supdrvGipInit() worker that determines the GIP TSC mode.
1377 *
1378 * @returns The most suitable TSC mode.
1379 * @param pDevExt Pointer to the device instance data.
1380 */
1381static SUPGIPMODE supdrvGipInitDetermineTscMode(PSUPDRVDEVEXT pDevExt)
1382{
1383 uint64_t u64DiffCoresIgnored;
1384 uint32_t uEAX, uEBX, uECX, uEDX;
1385
1386 /*
1387 * Establish whether the CPU advertises TSC as invariant, we need that in
1388 * a couple of places below.
1389 */
1390 bool fInvariantTsc = false;
1391 if (ASMHasCpuId())
1392 {
1393 uEAX = ASMCpuId_EAX(0x80000000);
1394 if (ASMIsValidExtRange(uEAX) && uEAX >= 0x80000007)
1395 {
1396 uEDX = ASMCpuId_EDX(0x80000007);
1397 if (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR)
1398 fInvariantTsc = true;
1399 }
1400 }
1401
1402 /*
1403 * On single CPU systems, we don't need to consider ASYNC mode.
1404 */
1405 if (RTMpGetCount() <= 1)
1406 return fInvariantTsc ? SUPGIPMODE_INVARIANT_TSC : SUPGIPMODE_SYNC_TSC;
1407
1408 /*
1409 * Allow the user and/or OS specific bits to force async mode.
1410 */
1411 if (supdrvOSGetForcedAsyncTscMode(pDevExt))
1412 return SUPGIPMODE_ASYNC_TSC;
1413
1414
1415 /*
1416 * Use invariant mode if the CPU says TSC is invariant.
1417 */
1418 if (fInvariantTsc)
1419 return SUPGIPMODE_INVARIANT_TSC;
1420
1421 /*
1422 * TSC is not invariant and we're on SMP, this presents two problems:
1423 *
1424 * (1) There might be a skew between the CPU, so that cpu0
1425 * returns a TSC that is slightly different from cpu1.
1426 * This screw may be due to (2), bad TSC initialization
1427 * or slightly different TSC rates.
1428 *
1429 * (2) Power management (and other things) may cause the TSC
1430 * to run at a non-constant speed, and cause the speed
1431 * to be different on the cpus. This will result in (1).
1432 *
1433 * If any of the above is detected, we will have to use ASYNC mode.
1434 */
1435 /* (1). Try check for current differences between the cpus. */
1436 if (supdrvGipInitDetermineAsyncTsc(&u64DiffCoresIgnored))
1437 return SUPGIPMODE_ASYNC_TSC;
1438
1439 /* (2) If it's an AMD CPU with power management, we won't trust its TSC. */
1440 ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
1441 if ( ASMIsValidStdRange(uEAX)
1442 && ASMIsAmdCpuEx(uEBX, uECX, uEDX))
1443 {
1444 /* Check for APM support. */
1445 uEAX = ASMCpuId_EAX(0x80000000);
1446 if (ASMIsValidExtRange(uEAX) && uEAX >= 0x80000007)
1447 {
1448 uEDX = ASMCpuId_EDX(0x80000007);
1449 if (uEDX & 0x3e) /* STC|TM|THERMTRIP|VID|FID. Ignore TS. */
1450 return SUPGIPMODE_ASYNC_TSC;
1451 }
1452 }
1453
1454 return SUPGIPMODE_SYNC_TSC;
1455}
1456
1457
1458/**
1459 * Initializes per-CPU GIP information.
1460 *
1461 * @param pDevExt Pointer to the device instance data.
1462 * @param pGip Pointer to the GIP.
1463 * @param pCpu Pointer to which GIP CPU to initalize.
1464 * @param u64NanoTS The current nanosecond timestamp.
1465 */
1466static void supdrvGipInitCpu(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS)
1467{
1468 /* !!! Warning !!! The GIP may not be linked to the device instance data at this point!
1469 which is why we have 2 separate parameters. Don't dereference pDevExt->pGip here. */
1470 pCpu->u32TransactionId = 2;
1471 pCpu->u64NanoTS = u64NanoTS;
1472 pCpu->u64TSC = ASMReadTSC();
1473 pCpu->u64TSCSample = GIP_TSC_DELTA_RSVD;
1474 pCpu->i64TSCDelta = pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED ? INT64_MAX : 0;
1475
1476 ASMAtomicWriteSize(&pCpu->enmState, SUPGIPCPUSTATE_INVALID);
1477 ASMAtomicWriteSize(&pCpu->idCpu, NIL_RTCPUID);
1478 ASMAtomicWriteS16(&pCpu->iCpuSet, -1);
1479 ASMAtomicWriteU16(&pCpu->idApic, UINT16_MAX);
1480
1481 /*
1482 * We don't know the following values until we've executed updates.
1483 * So, we'll just pretend it's a 4 GHz CPU and adjust the history it on
1484 * the 2nd timer callout.
1485 */
1486 pCpu->u64CpuHz = _4G + 1; /* tstGIP-2 depends on this. */
1487 pCpu->u32UpdateIntervalTSC
1488 = pCpu->au32TSCHistory[0]
1489 = pCpu->au32TSCHistory[1]
1490 = pCpu->au32TSCHistory[2]
1491 = pCpu->au32TSCHistory[3]
1492 = pCpu->au32TSCHistory[4]
1493 = pCpu->au32TSCHistory[5]
1494 = pCpu->au32TSCHistory[6]
1495 = pCpu->au32TSCHistory[7]
1496 = (uint32_t)(_4G / pGip->u32UpdateHz);
1497}
1498
1499
1500/**
1501 * Initializes the GIP data.
1502 *
1503 * @param pDevExt Pointer to the device instance data.
1504 * @param pGip Pointer to the read-write kernel mapping of the GIP.
1505 * @param HCPhys The physical address of the GIP.
1506 * @param u64NanoTS The current nanosecond timestamp.
1507 * @param uUpdateHz The update frequency.
1508 * @param uUpdateIntervalNS The update interval in nanoseconds.
1509 * @param cCpus The CPU count.
1510 */
1511static void supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys,
1512 uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS, unsigned cCpus)
1513{
1514 size_t const cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), PAGE_SIZE);
1515 unsigned i;
1516#ifdef DEBUG_DARWIN_GIP
1517 OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
1518#else
1519 LogFlow(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
1520#endif
1521
1522 /*
1523 * Initialize the structure.
1524 */
1525 memset(pGip, 0, cbGip);
1526
1527 pGip->u32Magic = SUPGLOBALINFOPAGE_MAGIC;
1528 pGip->u32Version = SUPGLOBALINFOPAGE_VERSION;
1529 pGip->u32Mode = supdrvGipInitDetermineTscMode(pDevExt);
1530 if ( pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC
1531 /*|| pGip->u32Mode == SUPGIPMODE_SYNC_TSC */)
1532 pGip->enmUseTscDelta = supdrvOSAreTscDeltasInSync() /* Allow OS override (windows). */
1533 ? SUPGIPUSETSCDELTA_ZERO_CLAIMED : SUPGIPUSETSCDELTA_PRACTICALLY_ZERO /* downgrade later */;
1534 else
1535 pGip->enmUseTscDelta = SUPGIPUSETSCDELTA_NOT_APPLICABLE;
1536 pGip->cCpus = (uint16_t)cCpus;
1537 pGip->cPages = (uint16_t)(cbGip / PAGE_SIZE);
1538 pGip->u32UpdateHz = uUpdateHz;
1539 pGip->u32UpdateIntervalNS = uUpdateIntervalNS;
1540 pGip->fGetGipCpu = SUPGIPGETCPU_APIC_ID;
1541 RTCpuSetEmpty(&pGip->OnlineCpuSet);
1542 RTCpuSetEmpty(&pGip->PresentCpuSet);
1543 RTMpGetSet(&pGip->PossibleCpuSet);
1544 pGip->cOnlineCpus = RTMpGetOnlineCount();
1545 pGip->cPresentCpus = RTMpGetPresentCount();
1546 pGip->cPossibleCpus = RTMpGetCount();
1547 pGip->idCpuMax = RTMpGetMaxCpuId();
1548 for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromApicId); i++)
1549 pGip->aiCpuFromApicId[i] = UINT16_MAX;
1550 for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); i++)
1551 pGip->aiCpuFromCpuSetIdx[i] = UINT16_MAX;
1552 for (i = 0; i < cCpus; i++)
1553 supdrvGipInitCpu(pDevExt, pGip, &pGip->aCPUs[i], u64NanoTS);
1554
1555 /*
1556 * Link it to the device extension.
1557 */
1558 pDevExt->pGip = pGip;
1559 pDevExt->HCPhysGip = HCPhys;
1560 pDevExt->cGipUsers = 0;
1561}
1562
1563
1564/**
1565 * Creates the GIP.
1566 *
1567 * @returns VBox status code.
1568 * @param pDevExt Instance data. GIP stuff may be updated.
1569 */
1570int VBOXCALL supdrvGipCreate(PSUPDRVDEVEXT pDevExt)
1571{
1572 PSUPGLOBALINFOPAGE pGip;
1573 RTHCPHYS HCPhysGip;
1574 uint32_t u32SystemResolution;
1575 uint32_t u32Interval;
1576 uint32_t u32MinInterval;
1577 uint32_t uMod;
1578 unsigned cCpus;
1579 int rc;
1580
1581 LogFlow(("supdrvGipCreate:\n"));
1582
1583 /* Assert order. */
1584 Assert(pDevExt->u32SystemTimerGranularityGrant == 0);
1585 Assert(pDevExt->GipMemObj == NIL_RTR0MEMOBJ);
1586 Assert(!pDevExt->pGipTimer);
1587
1588 /*
1589 * Check the CPU count.
1590 */
1591 cCpus = RTMpGetArraySize();
1592 if ( cCpus > RTCPUSET_MAX_CPUS
1593 || cCpus > 256 /* ApicId is used for the mappings */)
1594 {
1595 SUPR0Printf("VBoxDrv: Too many CPUs (%u) for the GIP (max %u)\n", cCpus, RT_MIN(RTCPUSET_MAX_CPUS, 256));
1596 return VERR_TOO_MANY_CPUS;
1597 }
1598
1599 /*
1600 * Allocate a contiguous set of pages with a default kernel mapping.
1601 */
1602 rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), false /*fExecutable*/);
1603 if (RT_FAILURE(rc))
1604 {
1605 OSDBGPRINT(("supdrvGipCreate: failed to allocate the GIP page. rc=%d\n", rc));
1606 return rc;
1607 }
1608 pGip = (PSUPGLOBALINFOPAGE)RTR0MemObjAddress(pDevExt->GipMemObj); AssertPtr(pGip);
1609 HCPhysGip = RTR0MemObjGetPagePhysAddr(pDevExt->GipMemObj, 0); Assert(HCPhysGip != NIL_RTHCPHYS);
1610
1611 /*
1612 * Allocate the TSC-delta sync struct on a separate cache line.
1613 */
1614 pDevExt->pvTscDeltaSync = RTMemAllocZ(sizeof(SUPTSCDELTASYNC) + 63);
1615 pDevExt->pTscDeltaSync = RT_ALIGN_PT(pDevExt->pvTscDeltaSync, 64, PSUPTSCDELTASYNC);
1616 Assert(RT_ALIGN_PT(pDevExt->pTscDeltaSync, 64, PSUPTSCDELTASYNC) == pDevExt->pTscDeltaSync);
1617
1618 /*
1619 * Find a reasonable update interval and initialize the structure.
1620 */
1621 supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
1622 /** @todo figure out why using a 100Ms interval upsets timekeeping in VMs.
1623 * See @bugref{6710}. */
1624 u32MinInterval = RT_NS_10MS;
1625 u32SystemResolution = RTTimerGetSystemGranularity();
1626 u32Interval = u32MinInterval;
1627 uMod = u32MinInterval % u32SystemResolution;
1628 if (uMod)
1629 u32Interval += u32SystemResolution - uMod;
1630
1631 supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval, cCpus);
1632
1633 if (RT_UNLIKELY( pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_ZERO_CLAIMED
1634 && pGip->u32Mode == SUPGIPMODE_ASYNC_TSC
1635 && !supdrvOSGetForcedAsyncTscMode(pDevExt)))
1636 {
1637 /* Basically, invariant Windows boxes, should never be detected as async (i.e. TSC-deltas should be 0). */
1638 OSDBGPRINT(("supdrvGipCreate: The TSC-deltas should be normalized by the host OS, but verifying shows it's not!\n"));
1639 return VERR_INTERNAL_ERROR_2;
1640 }
1641
1642 RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet);
1643 RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet);
1644#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1645 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1646 {
1647 /* Initialize TSC-delta measurement thread before executing any Mp event callbacks. */
1648 rc = supdrvTscDeltaThreadInit(pDevExt);
1649 }
1650#endif
1651 if (RT_SUCCESS(rc))
1652 {
1653 rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt);
1654 if (RT_SUCCESS(rc))
1655 {
1656 rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip);
1657 if (RT_SUCCESS(rc))
1658 {
1659#ifndef SUPDRV_USE_TSC_DELTA_THREAD
1660 uint16_t iCpu;
1661 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1662 {
1663 /*
1664 * Measure the TSC deltas now that we have MP notifications.
1665 */
1666 int cTries = 5;
1667 do
1668 {
1669 rc = supdrvMeasureInitialTscDeltas(pDevExt);
1670 if ( rc != VERR_TRY_AGAIN
1671 && rc != VERR_CPU_OFFLINE)
1672 break;
1673 } while (--cTries > 0);
1674 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
1675 Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta));
1676 }
1677 else
1678 {
1679 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
1680 AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
1681 }
1682#endif
1683 if (RT_SUCCESS(rc))
1684 {
1685 rc = supdrvGipInitMeasureTscFreq(pDevExt);
1686 if (RT_SUCCESS(rc))
1687 {
1688 /*
1689 * Create the timer.
1690 * If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
1691 */
1692 if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC)
1693 {
1694 rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL,
1695 supdrvGipAsyncTimer, pDevExt);
1696 if (rc == VERR_NOT_SUPPORTED)
1697 {
1698 OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
1699 pGip->u32Mode = SUPGIPMODE_SYNC_TSC;
1700 }
1701 }
1702 if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
1703 rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */,
1704 supdrvGipSyncAndInvariantTimer, pDevExt);
1705 if (RT_SUCCESS(rc))
1706 {
1707 /*
1708 * We're good.
1709 */
1710 Log(("supdrvGipCreate: %u ns interval.\n", u32Interval));
1711 supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
1712
1713 g_pSUPGlobalInfoPage = pGip;
1714 if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1715 supdrvGipInitAsyncRefineTscFreq(pDevExt);
1716 return VINF_SUCCESS;
1717 }
1718
1719 OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
1720 Assert(!pDevExt->pGipTimer);
1721 }
1722 else
1723 OSDBGPRINT(("supdrvGipCreate: supdrvGipInitMeasureTscFreq failed. rc=%Rrc\n", rc));
1724 }
1725 else
1726 OSDBGPRINT(("supdrvGipCreate: supdrvMeasureInitialTscDeltas failed. rc=%Rrc\n", rc));
1727 }
1728 else
1729 OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));
1730 }
1731 else
1732 OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));
1733 }
1734 else
1735 OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc));
1736
1737 supdrvGipDestroy(pDevExt); /* Releases timer frequency increase too. */
1738 return rc;
1739}
1740
1741
1742/**
1743 * Invalidates the GIP data upon termination.
1744 *
1745 * @param pGip Pointer to the read-write kernel mapping of the GIP.
1746 */
1747static void supdrvGipTerm(PSUPGLOBALINFOPAGE pGip)
1748{
1749 unsigned i;
1750 pGip->u32Magic = 0;
1751 for (i = 0; i < pGip->cCpus; i++)
1752 {
1753 pGip->aCPUs[i].u64NanoTS = 0;
1754 pGip->aCPUs[i].u64TSC = 0;
1755 pGip->aCPUs[i].iTSCHistoryHead = 0;
1756 pGip->aCPUs[i].u64TSCSample = 0;
1757 pGip->aCPUs[i].i64TSCDelta = INT64_MAX;
1758 }
1759}
1760
1761
1762/**
1763 * Terminates the GIP.
1764 *
1765 * @param pDevExt Instance data. GIP stuff may be updated.
1766 */
1767void VBOXCALL supdrvGipDestroy(PSUPDRVDEVEXT pDevExt)
1768{
1769 int rc;
1770#ifdef DEBUG_DARWIN_GIP
1771 OSDBGPRINT(("supdrvGipDestroy: pDevExt=%p pGip=%p pGipTimer=%p GipMemObj=%p\n", pDevExt,
1772 pDevExt->GipMemObj != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pDevExt->GipMemObj) : NULL,
1773 pDevExt->pGipTimer, pDevExt->GipMemObj));
1774#endif
1775
1776 /*
1777 * Stop receiving MP notifications before tearing anything else down.
1778 */
1779 RTMpNotificationDeregister(supdrvGipMpEvent, pDevExt);
1780
1781#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1782 /*
1783 * Terminate the TSC-delta measurement thread and resources.
1784 */
1785 supdrvTscDeltaTerm(pDevExt);
1786#endif
1787
1788 /*
1789 * Destroy the TSC-refinement one-shot timer.
1790 */
1791 if (pDevExt->pTscRefineTimer)
1792 {
1793 RTTimerDestroy(pDevExt->pTscRefineTimer);
1794 pDevExt->pTscRefineTimer = NULL;
1795 }
1796
1797 if (pDevExt->pvTscDeltaSync)
1798 {
1799 RTMemFree(pDevExt->pvTscDeltaSync);
1800 pDevExt->pTscDeltaSync = NULL;
1801 pDevExt->pvTscDeltaSync = NULL;
1802 }
1803
1804 /*
1805 * Invalid the GIP data.
1806 */
1807 if (pDevExt->pGip)
1808 {
1809 supdrvGipTerm(pDevExt->pGip);
1810 pDevExt->pGip = NULL;
1811 }
1812 g_pSUPGlobalInfoPage = NULL;
1813
1814 /*
1815 * Destroy the timer and free the GIP memory object.
1816 */
1817 if (pDevExt->pGipTimer)
1818 {
1819 rc = RTTimerDestroy(pDevExt->pGipTimer); AssertRC(rc);
1820 pDevExt->pGipTimer = NULL;
1821 }
1822
1823 if (pDevExt->GipMemObj != NIL_RTR0MEMOBJ)
1824 {
1825 rc = RTR0MemObjFree(pDevExt->GipMemObj, true /* free mappings */); AssertRC(rc);
1826 pDevExt->GipMemObj = NIL_RTR0MEMOBJ;
1827 }
1828
1829 /*
1830 * Finally, make sure we've release the system timer resolution request
1831 * if one actually succeeded and is still pending.
1832 */
1833 supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
1834}
1835
1836
1837
1838
1839/*
1840 *
1841 *
1842 * GIP Update Timer Related Code
1843 * GIP Update Timer Related Code
1844 * GIP Update Timer Related Code
1845 *
1846 *
1847 */
1848
1849
1850/**
1851 * Worker routine for supdrvGipUpdate() and supdrvGipUpdatePerCpu() that
1852 * updates all the per cpu data except the transaction id.
1853 *
1854 * @param pDevExt The device extension.
1855 * @param pGipCpu Pointer to the per cpu data.
1856 * @param u64NanoTS The current time stamp.
1857 * @param u64TSC The current TSC.
1858 * @param iTick The current timer tick.
1859 *
1860 * @remarks Can be called with interrupts disabled!
1861 */
1862static void supdrvGipDoUpdateCpu(PSUPDRVDEVEXT pDevExt, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS, uint64_t u64TSC, uint64_t iTick)
1863{
1864 uint64_t u64TSCDelta;
1865 uint32_t u32UpdateIntervalTSC;
1866 uint32_t u32UpdateIntervalTSCSlack;
1867 unsigned iTSCHistoryHead;
1868 uint64_t u64CpuHz;
1869 uint32_t u32TransactionId;
1870
1871 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1872 AssertPtrReturnVoid(pGip);
1873
1874 /* Delta between this and the previous update. */
1875 ASMAtomicUoWriteU32(&pGipCpu->u32PrevUpdateIntervalNS, (uint32_t)(u64NanoTS - pGipCpu->u64NanoTS));
1876
1877 /*
1878 * Update the NanoTS.
1879 */
1880 ASMAtomicWriteU64(&pGipCpu->u64NanoTS, u64NanoTS);
1881
1882 /*
1883 * Calc TSC delta.
1884 */
1885 u64TSCDelta = u64TSC - pGipCpu->u64TSC;
1886 ASMAtomicWriteU64(&pGipCpu->u64TSC, u64TSC);
1887
1888 /* We don't need to keep realculating the frequency when it's invariant. */
1889 if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1890 return;
1891
1892 if (u64TSCDelta >> 32)
1893 {
1894 u64TSCDelta = pGipCpu->u32UpdateIntervalTSC;
1895 pGipCpu->cErrors++;
1896 }
1897
1898 /*
1899 * On the 2nd and 3rd callout, reset the history with the current TSC
1900 * interval since the values entered by supdrvGipInit are totally off.
1901 * The interval on the 1st callout completely unreliable, the 2nd is a bit
1902 * better, while the 3rd should be most reliable.
1903 */
1904 u32TransactionId = pGipCpu->u32TransactionId;
1905 if (RT_UNLIKELY( ( u32TransactionId == 5
1906 || u32TransactionId == 7)
1907 && ( iTick == 2
1908 || iTick == 3) ))
1909 {
1910 unsigned i;
1911 for (i = 0; i < RT_ELEMENTS(pGipCpu->au32TSCHistory); i++)
1912 ASMAtomicUoWriteU32(&pGipCpu->au32TSCHistory[i], (uint32_t)u64TSCDelta);
1913 }
1914
1915 /*
1916 * Validate the NanoTS deltas between timer fires with an arbitrary threshold of 0.5%.
1917 * Wait until we have at least one full history since the above history reset. The
1918 * assumption is that the majority of the previous history values will be tolerable.
1919 * See @bugref{6710} comment #67.
1920 */
1921 if ( u32TransactionId > 23 /* 7 + (8 * 2) */
1922 && pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
1923 {
1924 uint32_t uNanoTsThreshold = pGip->u32UpdateIntervalNS / 200;
1925 if ( pGipCpu->u32PrevUpdateIntervalNS > pGip->u32UpdateIntervalNS + uNanoTsThreshold
1926 || pGipCpu->u32PrevUpdateIntervalNS < pGip->u32UpdateIntervalNS - uNanoTsThreshold)
1927 {
1928 uint32_t u32;
1929 u32 = pGipCpu->au32TSCHistory[0];
1930 u32 += pGipCpu->au32TSCHistory[1];
1931 u32 += pGipCpu->au32TSCHistory[2];
1932 u32 += pGipCpu->au32TSCHistory[3];
1933 u32 >>= 2;
1934 u64TSCDelta = pGipCpu->au32TSCHistory[4];
1935 u64TSCDelta += pGipCpu->au32TSCHistory[5];
1936 u64TSCDelta += pGipCpu->au32TSCHistory[6];
1937 u64TSCDelta += pGipCpu->au32TSCHistory[7];
1938 u64TSCDelta >>= 2;
1939 u64TSCDelta += u32;
1940 u64TSCDelta >>= 1;
1941 }
1942 }
1943
1944 /*
1945 * TSC History.
1946 */
1947 Assert(RT_ELEMENTS(pGipCpu->au32TSCHistory) == 8);
1948 iTSCHistoryHead = (pGipCpu->iTSCHistoryHead + 1) & 7;
1949 ASMAtomicWriteU32(&pGipCpu->iTSCHistoryHead, iTSCHistoryHead);
1950 ASMAtomicWriteU32(&pGipCpu->au32TSCHistory[iTSCHistoryHead], (uint32_t)u64TSCDelta);
1951
1952 /*
1953 * UpdateIntervalTSC = average of last 8,2,1 intervals depending on update HZ.
1954 *
1955 * On Windows, we have an occasional (but recurring) sour value that messed up
1956 * the history but taking only 1 interval reduces the precision overall.
1957 * However, this problem existed before the invariant mode was introduced.
1958 */
1959 if ( pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC
1960 || pGip->u32UpdateHz >= 1000)
1961 {
1962 uint32_t u32;
1963 u32 = pGipCpu->au32TSCHistory[0];
1964 u32 += pGipCpu->au32TSCHistory[1];
1965 u32 += pGipCpu->au32TSCHistory[2];
1966 u32 += pGipCpu->au32TSCHistory[3];
1967 u32 >>= 2;
1968 u32UpdateIntervalTSC = pGipCpu->au32TSCHistory[4];
1969 u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[5];
1970 u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[6];
1971 u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[7];
1972 u32UpdateIntervalTSC >>= 2;
1973 u32UpdateIntervalTSC += u32;
1974 u32UpdateIntervalTSC >>= 1;
1975
1976 /* Value chosen for a 2GHz Athlon64 running linux 2.6.10/11. */
1977 u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 14;
1978 }
1979 else if (pGip->u32UpdateHz >= 90)
1980 {
1981 u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
1982 u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[(iTSCHistoryHead - 1) & 7];
1983 u32UpdateIntervalTSC >>= 1;
1984
1985 /* value chosen on a 2GHz thinkpad running windows */
1986 u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 7;
1987 }
1988 else
1989 {
1990 u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
1991
1992 /* This value hasn't be checked yet.. waiting for OS/2 and 33Hz timers.. :-) */
1993 u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 6;
1994 }
1995 ASMAtomicWriteU32(&pGipCpu->u32UpdateIntervalTSC, u32UpdateIntervalTSC + u32UpdateIntervalTSCSlack);
1996
1997 /*
1998 * CpuHz.
1999 */
2000 u64CpuHz = ASMMult2xU32RetU64(u32UpdateIntervalTSC, RT_NS_1SEC);
2001 u64CpuHz /= pGip->u32UpdateIntervalNS;
2002 ASMAtomicWriteU64(&pGipCpu->u64CpuHz, u64CpuHz);
2003}
2004
2005
2006/**
2007 * Updates the GIP.
2008 *
2009 * @param pDevExt The device extension.
2010 * @param u64NanoTS The current nanosecond timesamp.
2011 * @param u64TSC The current TSC timesamp.
2012 * @param idCpu The CPU ID.
2013 * @param iTick The current timer tick.
2014 *
2015 * @remarks Can be called with interrupts disabled!
2016 */
2017static void supdrvGipUpdate(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC, RTCPUID idCpu, uint64_t iTick)
2018{
2019 /*
2020 * Determine the relevant CPU data.
2021 */
2022 PSUPGIPCPU pGipCpu;
2023 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2024 AssertPtrReturnVoid(pGip);
2025
2026 if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
2027 pGipCpu = &pGip->aCPUs[0];
2028 else
2029 {
2030 unsigned iCpu = pGip->aiCpuFromApicId[ASMGetApicId()];
2031 if (RT_UNLIKELY(iCpu >= pGip->cCpus))
2032 return;
2033 pGipCpu = &pGip->aCPUs[iCpu];
2034 if (RT_UNLIKELY(pGipCpu->idCpu != idCpu))
2035 return;
2036 }
2037
2038 /*
2039 * Start update transaction.
2040 */
2041 if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
2042 {
2043 /* this can happen on win32 if we're taking to long and there are more CPUs around. shouldn't happen though. */
2044 AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
2045 ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2046 pGipCpu->cErrors++;
2047 return;
2048 }
2049
2050 /*
2051 * Recalc the update frequency every 0x800th time.
2052 */
2053 if ( pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC /* cuz we're not recalculating the frequency on invariants hosts. */
2054 && !(pGipCpu->u32TransactionId & (GIP_UPDATEHZ_RECALC_FREQ * 2 - 2)))
2055 {
2056 if (pGip->u64NanoTSLastUpdateHz)
2057 {
2058#ifdef RT_ARCH_AMD64 /** @todo fix 64-bit div here to work on x86 linux. */
2059 uint64_t u64Delta = u64NanoTS - pGip->u64NanoTSLastUpdateHz;
2060 uint32_t u32UpdateHz = (uint32_t)((RT_NS_1SEC_64 * GIP_UPDATEHZ_RECALC_FREQ) / u64Delta);
2061 if (u32UpdateHz <= 2000 && u32UpdateHz >= 30)
2062 {
2063 /** @todo r=ramshankar: Changing u32UpdateHz might screw up TSC frequency
2064 * calculation on non-invariant hosts if it changes the history decision
2065 * taken in supdrvGipDoUpdateCpu(). */
2066 uint64_t u64Interval = u64Delta / GIP_UPDATEHZ_RECALC_FREQ;
2067 ASMAtomicWriteU32(&pGip->u32UpdateHz, u32UpdateHz);
2068 ASMAtomicWriteU32(&pGip->u32UpdateIntervalNS, (uint32_t)u64Interval);
2069 }
2070#endif
2071 }
2072 ASMAtomicWriteU64(&pGip->u64NanoTSLastUpdateHz, u64NanoTS | 1);
2073 }
2074
2075 /*
2076 * Update the data.
2077 */
2078 supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
2079
2080 /*
2081 * Complete transaction.
2082 */
2083 ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2084}
2085
2086
2087/**
2088 * Updates the per cpu GIP data for the calling cpu.
2089 *
2090 * @param pDevExt The device extension.
2091 * @param u64NanoTS The current nanosecond timesamp.
2092 * @param u64TSC The current TSC timesamp.
2093 * @param idCpu The CPU ID.
2094 * @param idApic The APIC id for the CPU index.
2095 * @param iTick The current timer tick.
2096 *
2097 * @remarks Can be called with interrupts disabled!
2098 */
2099static void supdrvGipUpdatePerCpu(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC,
2100 RTCPUID idCpu, uint8_t idApic, uint64_t iTick)
2101{
2102 uint32_t iCpu;
2103 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2104
2105 /*
2106 * Avoid a potential race when a CPU online notification doesn't fire on
2107 * the onlined CPU but the tick creeps in before the event notification is
2108 * run.
2109 */
2110 if (RT_UNLIKELY(iTick == 1))
2111 {
2112 iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
2113 if (pGip->aCPUs[iCpu].enmState == SUPGIPCPUSTATE_OFFLINE)
2114 supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
2115 }
2116
2117 iCpu = pGip->aiCpuFromApicId[idApic];
2118 if (RT_LIKELY(iCpu < pGip->cCpus))
2119 {
2120 PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
2121 if (pGipCpu->idCpu == idCpu)
2122 {
2123 /*
2124 * Start update transaction.
2125 */
2126 if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
2127 {
2128 AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
2129 ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2130 pGipCpu->cErrors++;
2131 return;
2132 }
2133
2134 /*
2135 * Update the data.
2136 */
2137 supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
2138
2139 /*
2140 * Complete transaction.
2141 */
2142 ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2143 }
2144 }
2145}
2146
2147
2148/**
2149 * Timer callback function for the sync and invariant GIP modes.
2150 *
2151 * @param pTimer The timer.
2152 * @param pvUser Opaque pointer to the device extension.
2153 * @param iTick The timer tick.
2154 */
2155static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2156{
2157 RTCCUINTREG uFlags;
2158 uint64_t u64TSC;
2159 uint64_t u64NanoTS;
2160 PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
2161 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2162
2163 uFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
2164 u64TSC = ASMReadTSC();
2165 u64NanoTS = RTTimeSystemNanoTS();
2166
2167 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
2168 {
2169 /*
2170 * The calculations in supdrvGipUpdate() is very timing sensitive and doesn't handle
2171 * missed timer ticks. So for now it is better to use a delta of 0 and have the TSC rate
2172 * affected a bit until we get proper TSC deltas than implementing options like
2173 * rescheduling the tick to be delivered on the right CPU or missing the tick entirely.
2174 *
2175 * The likely hood of this happening is really low. On Windows, Linux, and Solaris
2176 * timers fire on the CPU they were registered/started on. Darwin timers doesn't
2177 * necessarily (they are high priority threads waiting).
2178 */
2179 Assert(!ASMIntAreEnabled());
2180 supdrvTscDeltaApply(pGip, &u64TSC, ASMGetApicId(), NULL /* pfDeltaApplied */);
2181 }
2182
2183 supdrvGipUpdate(pDevExt, u64NanoTS, u64TSC, NIL_RTCPUID, iTick);
2184
2185 ASMSetFlags(uFlags);
2186
2187#ifdef SUPDRV_USE_TSC_DELTA_THREAD
2188 if ( pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED
2189 && !RTCpuSetIsEmpty(&pDevExt->TscDeltaCpuSet))
2190 {
2191 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
2192 if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
2193 || pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
2194 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
2195 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
2196 /** @todo Do the actual poking using -- RTThreadUserSignal() */
2197 }
2198#endif
2199}
2200
2201
2202/**
2203 * Timer callback function for async GIP mode.
2204 * @param pTimer The timer.
2205 * @param pvUser Opaque pointer to the device extension.
2206 * @param iTick The timer tick.
2207 */
2208static DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2209{
2210 RTCCUINTREG fOldFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
2211 PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
2212 RTCPUID idCpu = RTMpCpuId();
2213 uint64_t u64TSC = ASMReadTSC();
2214 uint64_t NanoTS = RTTimeSystemNanoTS();
2215
2216 /** @todo reset the transaction number and whatnot when iTick == 1. */
2217 if (pDevExt->idGipMaster == idCpu)
2218 supdrvGipUpdate(pDevExt, NanoTS, u64TSC, idCpu, iTick);
2219 else
2220 supdrvGipUpdatePerCpu(pDevExt, NanoTS, u64TSC, idCpu, ASMGetApicId(), iTick);
2221
2222 ASMSetFlags(fOldFlags);
2223}
2224
2225
2226
2227
2228/*
2229 *
2230 *
2231 * TSC Delta Measurements And Related Code
2232 * TSC Delta Measurements And Related Code
2233 * TSC Delta Measurements And Related Code
2234 *
2235 *
2236 */
2237
2238
2239/*
2240 * Select TSC delta measurement algorithm.
2241 */
2242#if 1
2243# define GIP_TSC_DELTA_METHOD_1
2244#else
2245# define GIP_TSC_DELTA_METHOD_2
2246#endif
2247
2248
2249#ifdef GIP_TSC_DELTA_METHOD_2
2250
2251/**
2252 * TSC delta measurment algorithm \#2 result entry.
2253 */
2254typedef struct SUPDRVTSCDELTAMETHOD2ENTRY
2255{
2256 uint32_t iSeqMine;
2257 uint32_t iSeqOther;
2258 uint64_t uTsc;
2259} SUPDRVTSCDELTAMETHOD2ENTRY;
2260
2261/**
2262 * TSC delta measurment algorithm \#2 Data.
2263 */
2264typedef struct SUPDRVTSCDELTAMETHOD2
2265{
2266 /** Padding to make sure the iCurSeqNo is in its own cache line.
2267 * ASSUMES cacheline sizes <= 128 bytes. */
2268 uint32_t au32CacheLinePaddingBefore[128 / sizeof(uint32_t)];
2269 /** The current sequence number of this worker. */
2270 uint32_t volatile iCurSeqNo;
2271 /** Padding to make sure the iCurSeqNo is in its own cache line.
2272 * ASSUMES cacheline sizes <= 128 bytes. */
2273 uint32_t au32CacheLinePaddingAfter[128 / sizeof(uint32_t) - 1];
2274 /** Result table. */
2275 SUPDRVTSCDELTAMETHOD2ENTRY aResults[96];
2276} SUPDRVTSCDELTAMETHOD2;
2277/** Pointer to the data for TSC delta mesurment algorithm \#2 .*/
2278typedef SUPDRVTSCDELTAMETHOD2 *PSUPDRVTSCDELTAMETHOD2;
2279
2280#endif /* GIP_TSC_DELTA_METHOD_2 */
2281
2282/**
2283 * Argument package/state passed by supdrvMeasureTscDeltaOne to the RTMpOn
2284 * callback worker.
2285 */
2286typedef struct SUPDRVGIPTSCDELTARGS
2287{
2288 PSUPDRVDEVEXT pDevExt;
2289 PSUPGIPCPU pWorker;
2290 PSUPGIPCPU pMaster;
2291 RTCPUID idMaster;
2292#ifdef GIP_TSC_DELTA_METHOD_2
2293 PSUPDRVTSCDELTAMETHOD2 pMasterData;
2294 PSUPDRVTSCDELTAMETHOD2 pWorkerData;
2295 uint32_t cHits;
2296 /*uint32_t cOffByOne;*/
2297 uint32_t iAttempt; /**< 1-base outer loop counter. */
2298 bool fLagMaster;
2299 bool fLagWorker;
2300#endif
2301} SUPDRVGIPTSCDELTARGS;
2302typedef SUPDRVGIPTSCDELTARGS *PSUPDRVGIPTSCDELTARGS;
2303
2304
2305#ifdef GIP_TSC_DELTA_METHOD_2
2306/*
2307 * TSC delta measurement algorithm \#2 configuration and code - Experimental!!
2308 */
2309# undef GIP_TSC_DELTA_LOOPS
2310# undef GIP_TSC_DELTA_READ_TIME_LOOPS
2311# undef GIP_TSC_DELTA_PRIMER_LOOPS
2312# define GIP_TSC_DELTA_LOOPS 17
2313# define GIP_TSC_DELTA_PRIMER_LOOPS 1
2314# define GIP_TSC_DELTA_READ_TIME_LOOPS GIP_TSC_DELTA_PRIMER_LOOPS /* no read-time-loops necessary */
2315
2316
2317static int supdrvTscDeltaMethod2Init(PSUPDRVGIPTSCDELTARGS pArgs)
2318{
2319 uint32_t const fFlags = /*RTMEMALLOCEX_FLAGS_ANY_CTX |*/ RTMEMALLOCEX_FLAGS_ZEROED;
2320 int rc = RTMemAllocEx(sizeof(*pArgs->pMasterData), 0, fFlags, (void **)&pArgs->pWorkerData);
2321 if (RT_SUCCESS(rc))
2322 rc = RTMemAllocEx(sizeof(*pArgs->pMasterData), 0, fFlags, (void **)&pArgs->pMasterData);
2323 return rc;
2324}
2325
2326
2327static void supdrvTscDeltaMethod2Term(PSUPDRVGIPTSCDELTARGS pArgs)
2328{
2329 RTMemFreeEx(pArgs->pMasterData, sizeof(*pArgs->pMasterData));
2330 RTMemFreeEx(pArgs->pWorkerData, sizeof(*pArgs->pWorkerData));
2331 /*SUPR0Printf("cHits=%d cOffByOne=%d m=%d w=%d\n", pArgs->cHits, pArgs->cOffByOne, pArgs->pMaster->idApic, pArgs->pWorker->idApic);*/
2332}
2333
2334
2335static void supdrvTscDeltaMethod2Looped(PSUPDRVGIPTSCDELTARGS pArgs, RTCPUID idCpu, unsigned iLoop)
2336{
2337 if (pArgs->idMaster == idCpu)
2338 {
2339 if (iLoop < GIP_TSC_DELTA_PRIMER_LOOPS)
2340 {
2341 if (iLoop == 0)
2342 pArgs->iAttempt++;
2343
2344 /* Lag during the priming to be nice to everyone.. */
2345 pArgs->fLagMaster = true;
2346 pArgs->fLagWorker = true;
2347 }
2348 else if (iLoop < (GIP_TSC_DELTA_LOOPS - GIP_TSC_DELTA_PRIMER_LOOPS) / 4)
2349 {
2350 /* 25 % of the body without lagging. */
2351 pArgs->fLagMaster = false;
2352 pArgs->fLagWorker = false;
2353 }
2354 else if (iLoop < (GIP_TSC_DELTA_LOOPS - GIP_TSC_DELTA_PRIMER_LOOPS) / 4 * 2)
2355 {
2356 /* 25 % of the body with both lagging. */
2357 pArgs->fLagMaster = true;
2358 pArgs->fLagWorker = true;
2359 }
2360 else
2361 {
2362 /* 50% of the body with alternating lag. */
2363 pArgs->fLagMaster = (iLoop & 1) == 0;
2364 pArgs->fLagWorker = (iLoop & 1) == 1;
2365 }
2366 }
2367}
2368
2369
2370/**
2371 * The core function of the 2nd TSC delta mesurment algorithm.
2372 *
2373 * The idea here is that we have the two CPUs execute the exact same code
2374 * collecting a largish set of TSC samples. The code has one data dependency on
2375 * the other CPU which intention it is to synchronize the execution as well as
2376 * help cross references the two sets of TSC samples (the sequence numbers).
2377 *
2378 * The @a fLag parameter is used to modify the execution a tiny bit on one or
2379 * both of the CPUs. When @a fLag differs between the CPUs, it is thought that
2380 * it will help with making the CPUs enter lock step execution occationally.
2381 *
2382 */
2383static void supdrvTscDeltaMethod2CollectData(PSUPDRVTSCDELTAMETHOD2 pMyData, uint32_t volatile *piOtherSeqNo, bool fLag)
2384{
2385 SUPDRVTSCDELTAMETHOD2ENTRY *pEntry = &pMyData->aResults[0];
2386 uint32_t cLeft = RT_ELEMENTS(pMyData->aResults);
2387
2388 ASMAtomicWriteU32(&pMyData->iCurSeqNo, 0);
2389 ASMSerializeInstruction();
2390 while (cLeft-- > 0)
2391 {
2392 uint64_t uTsc;
2393 uint32_t iSeqMine = ASMAtomicIncU32(&pMyData->iCurSeqNo);
2394 uint32_t iSeqOther = ASMAtomicReadU32(piOtherSeqNo);
2395 ASMCompilerBarrier();
2396 ASMSerializeInstruction(); /* Way better result than with ASMMemoryFenceSSE2() in this position! */
2397 uTsc = ASMReadTSC();
2398 ASMAtomicIncU32(&pMyData->iCurSeqNo);
2399 ASMCompilerBarrier();
2400 ASMSerializeInstruction();
2401 pEntry->iSeqMine = iSeqMine;
2402 pEntry->iSeqOther = iSeqOther;
2403 pEntry->uTsc = uTsc;
2404 pEntry++;
2405 ASMSerializeInstruction();
2406 if (fLag)
2407 ASMNopPause();
2408 }
2409}
2410
2411
2412static void supdrvTscDeltaMethod2ProcessDataSet(PSUPDRVGIPTSCDELTARGS pArgs, PSUPDRVTSCDELTAMETHOD2 pMyData,
2413 bool fIsMaster, uint32_t cResults,
2414 PSUPDRVTSCDELTAMETHOD2 pOtherData, int64_t iMasterTscDelta,
2415 int64_t volatile *piWorkerTscDelta)
2416{
2417 uint32_t cHits = 0;
2418#if 0
2419 uint32_t cOffByOne = 0;
2420#endif
2421 uint32_t idxResult = 0;
2422 int64_t iBestDelta = *piWorkerTscDelta;
2423
2424 if (cResults > RT_ELEMENTS(pMyData->aResults))
2425 cResults = RT_ELEMENTS(pMyData->aResults);
2426
2427 for (idxResult = 0; idxResult < cResults; idxResult++)
2428 {
2429 uint32_t idxOther = pMyData->aResults[idxResult].iSeqOther;
2430 if (idxOther & 1)
2431 {
2432 idxOther >>= 1;
2433 if (idxOther < RT_ELEMENTS(pOtherData->aResults))
2434 {
2435 if (pOtherData->aResults[idxOther].iSeqOther == pMyData->aResults[idxResult].iSeqMine)
2436 {
2437 int64_t iDelta;
2438 if (fIsMaster)
2439 iDelta = pOtherData->aResults[idxOther].uTsc
2440 - (pMyData->aResults[idxResult].uTsc - iMasterTscDelta);
2441 else
2442 iDelta = (pOtherData->aResults[idxResult].uTsc - iMasterTscDelta)
2443 - pMyData->aResults[idxOther].uTsc;
2444 if ( iDelta >= GIP_TSC_DELTA_INITIAL_MASTER_VALUE
2445 ? iDelta < iBestDelta
2446 : iDelta > iBestDelta || iBestDelta == INT64_MAX)
2447 iBestDelta = iDelta;
2448 cHits++;
2449 }
2450 }
2451 }
2452#if 0 /* Can be used to detect battles between threads on the same core. Decided to change the master instead. */
2453 else
2454 {
2455 idxOther >>= 1;
2456 if ( idxOther < RT_ELEMENTS(pOtherData->aResults)
2457 && pOtherData->aResults[idxOther].iSeqOther == pMyData->aResults[idxResult].iSeqMine)
2458 cOffByOne++;
2459 }
2460#endif
2461 }
2462
2463 if (cHits > 0)
2464 *piWorkerTscDelta = iBestDelta;
2465 pArgs->cHits += cHits;
2466#if 0
2467 pArgs->cOffByOne += cOffByOne;
2468#endif
2469}
2470
2471
2472static void supdrvTscDeltaMethod2ProcessDataOnMaster(PSUPDRVGIPTSCDELTARGS pArgs, bool fFinalLoop)
2473{
2474 supdrvTscDeltaMethod2ProcessDataSet(pArgs,
2475 pArgs->pMasterData,
2476 true /*fIsMaster*/,
2477 RT_ELEMENTS(pArgs->pMasterData->aResults),
2478 pArgs->pWorkerData,
2479 pArgs->pMaster->i64TSCDelta,
2480 &pArgs->pWorker->i64TSCDelta);
2481
2482 supdrvTscDeltaMethod2ProcessDataSet(pArgs,
2483 pArgs->pWorkerData,
2484 false /*fIsMaster*/,
2485 ASMAtomicReadU32(&pArgs->pWorkerData->iCurSeqNo) >> 1,
2486 pArgs->pMasterData,
2487 pArgs->pMaster->i64TSCDelta,
2488 &pArgs->pWorker->i64TSCDelta);
2489}
2490
2491#endif /* GIP_TSC_DELTA_METHOD_2 */
2492
2493
2494/**
2495 * Callback used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
2496 * and compute the delta between them.
2497 *
2498 * @param idCpu The CPU we are current scheduled on.
2499 * @param pvUser1 Pointer to a parameter package (SUPDRVGIPTSCDELTARGS).
2500 * @param pvUser2 Unused.
2501 *
2502 * @remarks Measuring TSC deltas between the CPUs is tricky because we need to
2503 * read the TSC at exactly the same time on both the master and the
2504 * worker CPUs. Due to DMA, bus arbitration, cache locality,
2505 * contention, SMI, pipelining etc. there is no guaranteed way of
2506 * doing this on x86 CPUs.
2507 *
2508 * GIP_TSC_DELTA_METHOD_1:
2509 * We ignore the first few runs of the loop in order to prime the
2510 * cache. Also, we need to be careful about using 'pause' instruction
2511 * in critical busy-wait loops in this code - it can cause undesired
2512 * behaviour with hyperthreading.
2513 *
2514 * We try to minimize the measurement error by computing the minimum
2515 * read time of the compare statement in the worker by taking TSC
2516 * measurements across it.
2517 *
2518 * It must be noted that the computed minimum read time is mostly to
2519 * eliminate huge deltas when the worker is too early and doesn't by
2520 * itself help produce more accurate deltas. We allow two times the
2521 * computed minimum as an arbibtrary acceptable threshold. Therefore,
2522 * it is still possible to get negative deltas where there are none
2523 * when the worker is earlier. As long as these occasional negative
2524 * deltas are lower than the time it takes to exit guest-context and
2525 * the OS to reschedule EMT on a different CPU we won't expose a TSC
2526 * that jumped backwards. It is because of the existence of the
2527 * negative deltas we don't recompute the delta with the master and
2528 * worker interchanged to eliminate the remaining measurement error.
2529 *
2530 * For GIP_TSC_DELTA_METHOD_2, see supdrvTscDeltaMethod2CollectData.
2531 */
2532static DECLCALLBACK(void) supdrvMeasureTscDeltaCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
2533{
2534 PSUPDRVGIPTSCDELTARGS pArgs = (PSUPDRVGIPTSCDELTARGS)pvUser1;
2535 PSUPDRVDEVEXT pDevExt = pArgs->pDevExt;
2536 PSUPGIPCPU pGipCpuWorker = pArgs->pWorker;
2537 PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
2538 RTCPUID idMaster = pArgs->idMaster;
2539 int cTriesLeft;
2540
2541 /* A bit of paranoia first. */
2542 if (!pGipCpuMaster || !pGipCpuWorker)
2543 return;
2544
2545 /* If the CPU isn't part of the measurement, return immediately. */
2546 if ( idCpu != idMaster
2547 && idCpu != pGipCpuWorker->idCpu)
2548 return;
2549
2550 /* If the IPRT API isn't concurrent safe, the master and worker wait for each other
2551 with a timeout to avoid deadlocking the entire system. */
2552 if (!RTMpOnAllIsConcurrentSafe())
2553 {
2554 /** @todo This was introduced for Windows, but since Windows doesn't use this
2555 * code path any longer (as DPC timeouts BSOD regardless of interrupts,
2556 * see @bugref{6710} comment 81), eventually phase it out. */
2557 uint64_t uTscNow;
2558 uint64_t uTscStart;
2559 uint64_t const cWaitTicks = 130000; /* Arbitrary value, can be tweaked later. */
2560
2561 ASMSerializeInstruction();
2562 uTscStart = ASMReadTSC();
2563 if (idCpu == idMaster)
2564 {
2565 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_PRESTART_MASTER);
2566 while (ASMAtomicReadU32(&pDevExt->pTscDeltaSync->u) != GIP_TSC_DELTA_SYNC_PRESTART_WORKER)
2567 {
2568 ASMSerializeInstruction();
2569 uTscNow = ASMReadTSC();
2570 if (uTscNow - uTscStart > cWaitTicks)
2571 {
2572 /* Set the worker delta to indicate failure, not the master. */
2573 ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, INT64_MAX);
2574 return;
2575 }
2576
2577 ASMNopPause();
2578 }
2579 }
2580 else
2581 {
2582 while (ASMAtomicReadU32(&pDevExt->pTscDeltaSync->u) != GIP_TSC_DELTA_SYNC_PRESTART_MASTER)
2583 {
2584 ASMSerializeInstruction();
2585 uTscNow = ASMReadTSC();
2586 if (uTscNow - uTscStart > cWaitTicks)
2587 {
2588 ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, INT64_MAX);
2589 return;
2590 }
2591
2592 ASMNopPause();
2593 }
2594 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_PRESTART_WORKER);
2595 }
2596 }
2597
2598 /*
2599 * ...
2600 */
2601 Assert(pGipCpuWorker->i64TSCDelta == INT64_MAX);
2602 cTriesLeft = 12;
2603 while (cTriesLeft-- > 0)
2604 {
2605 unsigned i;
2606 uint64_t uMinCmpReadTime = UINT64_MAX;
2607 for (i = 0; i < GIP_TSC_DELTA_LOOPS; i++)
2608 {
2609#ifdef GIP_TSC_DELTA_METHOD_2
2610 supdrvTscDeltaMethod2Looped(pArgs, idCpu, i);
2611#endif
2612 if (idCpu == idMaster)
2613 {
2614 /*
2615 * The master.
2616 */
2617 RTCCUINTREG uFlags;
2618 AssertMsg(pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD,
2619 ("%#llx idMaster=%#x idWorker=%#x (idGipMaster=%#x)\n",
2620 pGipCpuMaster->u64TSCSample, idMaster, pGipCpuWorker->idCpu, pDevExt->idGipMaster));
2621 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_START);
2622
2623 /* Disable interrupts only in the master for as short a period
2624 as possible, thanks again to Windows. See @bugref{6710} comment #73. */
2625 uFlags = ASMIntDisableFlags();
2626
2627 while (ASMAtomicReadU32(&pDevExt->pTscDeltaSync->u) == GIP_TSC_DELTA_SYNC_START)
2628 { /* nothing */ }
2629
2630#ifdef GIP_TSC_DELTA_METHOD_1
2631 do
2632 {
2633 ASMSerializeInstruction();
2634 ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, ASMReadTSC());
2635 } while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
2636
2637#elif defined(GIP_TSC_DELTA_METHOD_2)
2638 supdrvTscDeltaMethod2CollectData(pArgs->pMasterData, &pArgs->pWorkerData->iCurSeqNo, pArgs->fLagMaster);
2639#else
2640# error "tsc delta method not selected"
2641#endif
2642
2643 /* Sync up with worker. */
2644 ASMSetFlags(uFlags);
2645
2646 while (ASMAtomicReadU32(&pDevExt->pTscDeltaSync->u) != GIP_TSC_DELTA_SYNC_WORKER_DONE)
2647 { /* nothing */ }
2648
2649 /* Process the data. */
2650 if (i > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
2651 {
2652#ifdef GIP_TSC_DELTA_METHOD_1
2653 if (pGipCpuWorker->u64TSCSample != GIP_TSC_DELTA_RSVD)
2654 {
2655 int64_t iDelta = pGipCpuWorker->u64TSCSample
2656 - (pGipCpuMaster->u64TSCSample - pGipCpuMaster->i64TSCDelta);
2657 if ( iDelta >= GIP_TSC_DELTA_INITIAL_MASTER_VALUE
2658 ? iDelta < pGipCpuWorker->i64TSCDelta
2659 : iDelta > pGipCpuWorker->i64TSCDelta || pGipCpuWorker->i64TSCDelta == INT64_MAX)
2660 pGipCpuWorker->i64TSCDelta = iDelta;
2661 }
2662#elif defined(GIP_TSC_DELTA_METHOD_2)
2663 if (i > GIP_TSC_DELTA_PRIMER_LOOPS)
2664 supdrvTscDeltaMethod2ProcessDataOnMaster(pArgs, i == GIP_TSC_DELTA_LOOPS - 1);
2665#else
2666# error "tsc delta method not selected"
2667#endif
2668 }
2669
2670 /* Reset our TSC sample and tell the worker to move on. */
2671 ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, GIP_TSC_DELTA_RSVD);
2672 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_STOP);
2673 }
2674 else
2675 {
2676 /*
2677 * The worker.
2678 */
2679 uint64_t uTscWorker;
2680 uint64_t uTscWorkerFlushed;
2681 uint64_t uCmpReadTime;
2682
2683 ASMAtomicReadU64(&pGipCpuMaster->u64TSCSample); /* Warm the cache line. */
2684 while (ASMAtomicReadU32(&pDevExt->pTscDeltaSync->u) != GIP_TSC_DELTA_SYNC_START)
2685 { /* nothing */ }
2686 Assert(pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
2687 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_WORKER_READY);
2688
2689#ifdef GIP_TSC_DELTA_METHOD_1
2690 /*
2691 * Keep reading the TSC until we notice that the master has read his. Reading
2692 * the TSC -after- the master has updated the memory is way too late. We thus
2693 * compensate by trying to measure how long it took for the worker to notice
2694 * the memory flushed from the master.
2695 */
2696 do
2697 {
2698 ASMSerializeInstruction();
2699 uTscWorker = ASMReadTSC();
2700 } while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
2701 ASMSerializeInstruction();
2702 uTscWorkerFlushed = ASMReadTSC();
2703
2704 uCmpReadTime = uTscWorkerFlushed - uTscWorker;
2705 if (i > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
2706 {
2707 /* This is totally arbitrary a.k.a I don't like it but I have no better ideas for now. */
2708 if (uCmpReadTime < (uMinCmpReadTime << 1))
2709 {
2710 ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, uTscWorker);
2711 if (uCmpReadTime < uMinCmpReadTime)
2712 uMinCmpReadTime = uCmpReadTime;
2713 }
2714 else
2715 ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
2716 }
2717 else if (i > GIP_TSC_DELTA_PRIMER_LOOPS)
2718 {
2719 if (uCmpReadTime < uMinCmpReadTime)
2720 uMinCmpReadTime = uCmpReadTime;
2721 }
2722
2723#elif defined(GIP_TSC_DELTA_METHOD_2)
2724 supdrvTscDeltaMethod2CollectData(pArgs->pWorkerData, &pArgs->pMasterData->iCurSeqNo, pArgs->fLagWorker);
2725#else
2726# error "tsc delta method not selected"
2727#endif
2728
2729 /* Tell master we're done collecting our data. */
2730 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_WORKER_DONE);
2731
2732 /* Wait for the master to process the data. */
2733 while (ASMAtomicReadU32(&pDevExt->pTscDeltaSync->u) == GIP_TSC_DELTA_SYNC_WORKER_DONE)
2734 ASMNopPause();
2735 }
2736 }
2737
2738 /*
2739 * We must reset the worker TSC sample value in case it gets picked as a
2740 * GIP master later on (it's trashed above, naturally).
2741 */
2742 if (idCpu == idMaster)
2743 ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
2744
2745 /*
2746 * Success? If so, stop trying.
2747 */
2748 if (pGipCpuWorker->i64TSCDelta != INT64_MAX)
2749 {
2750 if (idCpu == idMaster)
2751 {
2752 RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuMaster->iCpuSet);
2753 RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuMaster->iCpuSet);
2754 }
2755 else
2756 {
2757 RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
2758 RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
2759 }
2760 break;
2761 }
2762 }
2763}
2764
2765
2766/**
2767 * Clears TSC delta related variables.
2768 *
2769 * Clears all TSC samples as well as the delta synchronization variable on the
2770 * all the per-CPU structs. Optionally also clears the per-cpu deltas too.
2771 *
2772 * @param pDevExt Pointer to the device instance data.
2773 * @param fClearDeltas Whether the deltas are also to be cleared.
2774 */
2775DECLINLINE(void) supdrvClearTscSamples(PSUPDRVDEVEXT pDevExt, bool fClearDeltas)
2776{
2777 unsigned iCpu;
2778 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2779 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
2780 {
2781 PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
2782 ASMAtomicWriteU64(&pGipCpu->u64TSCSample, GIP_TSC_DELTA_RSVD);
2783 if (fClearDeltas)
2784 ASMAtomicWriteS64(&pGipCpu->i64TSCDelta, INT64_MAX);
2785 }
2786 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_STOP);
2787}
2788
2789
2790/**
2791 * Measures the TSC delta between the master GIP CPU and one specified worker
2792 * CPU.
2793 *
2794 * @returns VBox status code.
2795 * @param pDevExt Pointer to the device instance data.
2796 * @param idxWorker The index of the worker CPU from the GIP's array of
2797 * CPUs.
2798 *
2799 * @remarks This must be called with preemption enabled!
2800 */
2801static int supdrvMeasureTscDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker)
2802{
2803 int rc;
2804 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2805 RTCPUID idMaster = pDevExt->idGipMaster;
2806 PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[idxWorker];
2807 PSUPGIPCPU pGipCpuMaster;
2808 uint32_t iGipCpuMaster;
2809
2810 /* Validate input a bit. */
2811 AssertReturn(pGip, VERR_INVALID_PARAMETER);
2812 Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
2813 Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2814
2815 /*
2816 * Don't attempt measuring the delta for the GIP master.
2817 */
2818 if (pGipCpuWorker->idCpu == idMaster)
2819 {
2820 if (pGipCpuWorker->i64TSCDelta == INT64_MAX) /* This shouldn't happen, but just in case. */
2821 ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
2822 return VINF_SUCCESS;
2823 }
2824
2825 /*
2826 * If the CPU has hyper-threading and the APIC IDs of the master and worker are adjacent,
2827 * try pick a different master. (This fudge only works with multi core systems.)
2828 * ASSUMES related threads have adjacent APIC IDs. ASSUMES two threads per core.
2829 */
2830 iGipCpuMaster = supdrvGipFindCpuIndexForCpuId(pGip, idMaster);
2831 AssertReturn(iGipCpuMaster < pGip->cCpus, VERR_INVALID_CPU_ID);
2832 pGipCpuMaster = &pGip->aCPUs[iGipCpuMaster];
2833 if ( (pGipCpuMaster->idApic & ~1) == (pGipCpuWorker->idApic & ~1)
2834 && ASMHasCpuId()
2835 && ASMIsValidStdRange(ASMCpuId_EAX(0))
2836 && (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_HTT)
2837 && pGip->cOnlineCpus > 2)
2838 {
2839 uint32_t i;
2840 for (i = 0; i < pGip->cCpus; i++)
2841 if ( i != iGipCpuMaster
2842 && i != idxWorker
2843 && pGip->aCPUs[i].enmState == SUPGIPCPUSTATE_ONLINE
2844 && pGip->aCPUs[i].i64TSCDelta != INT64_MAX
2845 && pGip->aCPUs[i].idCpu != NIL_RTCPUID
2846 && pGip->aCPUs[i].idCpu != idMaster /* paranoia starts here... */
2847 && pGip->aCPUs[i].idCpu != pGipCpuWorker->idCpu
2848 && pGip->aCPUs[i].idApic != pGipCpuWorker->idApic
2849 && pGip->aCPUs[i].idApic != pGipCpuMaster->idApic)
2850 {
2851 iGipCpuMaster = i;
2852 pGipCpuMaster = &pGip->aCPUs[i];
2853 idMaster = pGipCpuMaster->idCpu;
2854 break;
2855 }
2856 }
2857
2858 /*
2859 * Set the master TSC as the initiator. This serializes delta measurments.
2860 */
2861 while (!ASMAtomicCmpXchgU32(&pDevExt->idTscDeltaInitiator, idMaster, NIL_RTCPUID))
2862 {
2863 /*
2864 * Sleep here rather than spin as there is a parallel measurement
2865 * being executed and that can take a good while to be done.
2866 */
2867 RTThreadSleep(1);
2868 }
2869
2870 if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpuWorker->iCpuSet))
2871 {
2872 /*
2873 * Initialize data package for the RTMpOnAll callback.
2874 */
2875 SUPDRVGIPTSCDELTARGS Args;
2876 RT_ZERO(Args);
2877 Args.pWorker = pGipCpuWorker;
2878 Args.pMaster = pGipCpuMaster;
2879 Args.idMaster = idMaster;
2880 Args.pDevExt = pDevExt;
2881#ifdef GIP_TSC_DELTA_METHOD_1
2882 rc = VINF_SUCCESS;
2883#elif defined(GIP_TSC_DELTA_METHOD_2)
2884 rc = supdrvTscDeltaMethod2Init(&Args);
2885#else
2886# error "huh?"
2887#endif
2888 if (RT_SUCCESS(rc))
2889 {
2890 /*
2891 * Fire TSC-read workers on all CPUs but only synchronize between master
2892 * and one worker to ease memory contention.
2893 */
2894 ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, INT64_MAX);
2895 ASMAtomicWriteU32(&pDevExt->pTscDeltaSync->u, GIP_TSC_DELTA_SYNC_STOP);
2896
2897 rc = RTMpOnAll(supdrvMeasureTscDeltaCallback, &Args, NULL);
2898 if (RT_SUCCESS(rc))
2899 {
2900 if (RT_LIKELY(pGipCpuWorker->i64TSCDelta != INT64_MAX))
2901 {
2902 /*
2903 * Work the TSC delta applicability rating. It starts
2904 * optimistic in supdrvGipInit, we downgrade it here.
2905 */
2906 SUPGIPUSETSCDELTA enmRating;
2907 if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO
2908 || pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO)
2909 enmRating = SUPGIPUSETSCDELTA_NOT_ZERO;
2910 else if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO
2911 || pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO)
2912 enmRating = SUPGIPUSETSCDELTA_ROUGHLY_ZERO;
2913 else
2914 enmRating = SUPGIPUSETSCDELTA_PRACTICALLY_ZERO;
2915 if (pGip->enmUseTscDelta < enmRating)
2916 {
2917 AssertCompile(sizeof(pGip->enmUseTscDelta) == sizeof(uint32_t));
2918 ASMAtomicWriteU32((uint32_t volatile *)&pGip->enmUseTscDelta, enmRating);
2919 }
2920 }
2921 else
2922 rc = VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED;
2923 }
2924 }
2925
2926#ifdef GIP_TSC_DELTA_METHOD_2
2927 supdrvTscDeltaMethod2Term(&Args);
2928#endif
2929 }
2930 else
2931 rc = VERR_CPU_OFFLINE;
2932
2933 ASMAtomicWriteU32(&pDevExt->idTscDeltaInitiator, NIL_RTCPUID);
2934 return rc;
2935}
2936
2937
2938/**
2939 * Performs the initial measurements of the TSC deltas between CPUs.
2940 *
2941 * This is called by supdrvGipCreate or triggered by it if threaded.
2942 *
2943 * @returns VBox status code.
2944 * @param pDevExt Pointer to the device instance data.
2945 *
2946 * @remarks Must be called only after supdrvGipInitOnCpu() as this function uses
2947 * idCpu, GIP's online CPU set which are populated in
2948 * supdrvGipInitOnCpu().
2949 */
2950static int supdrvMeasureInitialTscDeltas(PSUPDRVDEVEXT pDevExt)
2951{
2952 PSUPGIPCPU pGipCpuMaster;
2953 unsigned iCpu;
2954 unsigned iOddEven;
2955 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2956 uint32_t idxMaster = UINT32_MAX;
2957 int rc = VINF_SUCCESS;
2958 uint32_t cMpOnOffEvents = ASMAtomicReadU32(&pDevExt->cMpOnOffEvents);
2959
2960 Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
2961
2962 /*
2963 * Pick the first CPU online as the master TSC and make it the new GIP master based
2964 * on the APIC ID.
2965 *
2966 * Technically we can simply use "idGipMaster" but doing this gives us master as CPU 0
2967 * in most cases making it nicer/easier for comparisons. It is safe to update the GIP
2968 * master as this point since the sync/async timer isn't created yet.
2969 */
2970 supdrvClearTscSamples(pDevExt, true /* fClearDeltas */);
2971 for (iCpu = 0; iCpu < RT_ELEMENTS(pGip->aiCpuFromApicId); iCpu++)
2972 {
2973 uint16_t idxCpu = pGip->aiCpuFromApicId[iCpu];
2974 if (idxCpu != UINT16_MAX)
2975 {
2976 PSUPGIPCPU pGipCpu = &pGip->aCPUs[idxCpu];
2977 if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpu->iCpuSet))
2978 {
2979 idxMaster = idxCpu;
2980 pGipCpu->i64TSCDelta = GIP_TSC_DELTA_INITIAL_MASTER_VALUE;
2981 break;
2982 }
2983 }
2984 }
2985 AssertReturn(idxMaster != UINT32_MAX, VERR_CPU_NOT_FOUND);
2986 pGipCpuMaster = &pGip->aCPUs[idxMaster];
2987 ASMAtomicWriteSize(&pDevExt->idGipMaster, pGipCpuMaster->idCpu);
2988
2989 /*
2990 * If there is only a single CPU online we have nothing to do.
2991 */
2992 if (pGip->cOnlineCpus <= 1)
2993 {
2994 AssertReturn(pGip->cOnlineCpus > 0, VERR_INTERNAL_ERROR_5);
2995 return VINF_SUCCESS;
2996 }
2997
2998 /*
2999 * Loop thru the GIP CPU array and get deltas for each CPU (except the
3000 * master). We do the CPUs with the even numbered APIC IDs first so that
3001 * we've got alternative master CPUs to pick from on hyper-threaded systems.
3002 */
3003 for (iOddEven = 0; iOddEven < 2; iOddEven++)
3004 {
3005 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
3006 {
3007 PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
3008 if ( iCpu != idxMaster
3009 && (iOddEven > 0 || (pGipCpuWorker->idApic & 1) == 0)
3010 && RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
3011 {
3012 rc = supdrvMeasureTscDeltaOne(pDevExt, iCpu);
3013 if (RT_FAILURE(rc))
3014 {
3015 SUPR0Printf("supdrvMeasureTscDeltaOne failed. rc=%d CPU[%u].idCpu=%u Master[%u].idCpu=%u\n", rc, iCpu,
3016 pGipCpuWorker->idCpu, idxMaster, pDevExt->idGipMaster, pGipCpuMaster->idCpu);
3017 break;
3018 }
3019
3020 if (ASMAtomicReadU32(&pDevExt->cMpOnOffEvents) != cMpOnOffEvents)
3021 {
3022 SUPR0Printf("One or more CPUs transitioned between online & offline states. I'm confused, retry...\n");
3023 rc = VERR_TRY_AGAIN;
3024 break;
3025 }
3026 }
3027 }
3028 }
3029
3030 return rc;
3031}
3032
3033
3034#ifdef SUPDRV_USE_TSC_DELTA_THREAD
3035
3036/**
3037 * Switches the TSC-delta measurement thread into the butchered state.
3038 *
3039 * @returns VBox status code.
3040 * @param pDevExt Pointer to the device instance data.
3041 * @param fSpinlockHeld Whether the TSC-delta spinlock is held or not.
3042 * @param pszFailed An error message to log.
3043 * @param rcFailed The error code to exit the thread with.
3044 */
3045static int supdrvTscDeltaThreadButchered(PSUPDRVDEVEXT pDevExt, bool fSpinlockHeld, const char *pszFailed, int rcFailed)
3046{
3047 if (!fSpinlockHeld)
3048 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3049
3050 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Butchered;
3051 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3052 OSDBGPRINT(("supdrvTscDeltaThreadButchered: %s. rc=%Rrc\n", rcFailed));
3053 return rcFailed;
3054}
3055
3056
3057/**
3058 * The TSC-delta measurement thread.
3059 *
3060 * @returns VBox status code.
3061 * @param hThread The thread handle.
3062 * @param pvUser Opaque pointer to the device instance data.
3063 */
3064static DECLCALLBACK(int) supdrvTscDeltaThread(RTTHREAD hThread, void *pvUser)
3065{
3066 PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
3067 bool fInitialMeasurement = true;
3068 uint32_t cConsecutiveTimeouts = 0;
3069 int rc = VERR_INTERNAL_ERROR_2;
3070 for (;;)
3071 {
3072 /*
3073 * Switch on the current state.
3074 */
3075 SUPDRVTSCDELTATHREADSTATE enmState;
3076 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3077 enmState = pDevExt->enmTscDeltaThreadState;
3078 switch (enmState)
3079 {
3080 case kTscDeltaThreadState_Creating:
3081 {
3082 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
3083 rc = RTSemEventSignal(pDevExt->hTscDeltaEvent);
3084 if (RT_FAILURE(rc))
3085 return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
3086 /* fall thru */
3087 }
3088
3089 case kTscDeltaThreadState_Listening:
3090 {
3091 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3092
3093 /* Simple adaptive timeout. */
3094 if (cConsecutiveTimeouts++ == 10)
3095 {
3096 if (pDevExt->cMsTscDeltaTimeout == 1) /* 10 ms */
3097 pDevExt->cMsTscDeltaTimeout = 10;
3098 else if (pDevExt->cMsTscDeltaTimeout == 10) /* +100 ms */
3099 pDevExt->cMsTscDeltaTimeout = 100;
3100 else if (pDevExt->cMsTscDeltaTimeout == 100) /* +1000 ms */
3101 pDevExt->cMsTscDeltaTimeout = 500;
3102 cConsecutiveTimeouts = 0;
3103 }
3104 rc = RTThreadUserWait(pDevExt->hTscDeltaThread, pDevExt->cMsTscDeltaTimeout);
3105 if ( RT_FAILURE(rc)
3106 && rc != VERR_TIMEOUT)
3107 return supdrvTscDeltaThreadButchered(pDevExt, false /* fSpinlockHeld */, "RTThreadUserWait", rc);
3108 RTThreadUserReset(pDevExt->hTscDeltaThread);
3109 break;
3110 }
3111
3112 case kTscDeltaThreadState_WaitAndMeasure:
3113 {
3114 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Measuring;
3115 rc = RTSemEventSignal(pDevExt->hTscDeltaEvent); /* (Safe on windows as long as spinlock isn't IRQ safe.) */
3116 if (RT_FAILURE(rc))
3117 return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
3118 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3119 pDevExt->cMsTscDeltaTimeout = 1;
3120 RTThreadSleep(10);
3121 /* fall thru */
3122 }
3123
3124 case kTscDeltaThreadState_Measuring:
3125 {
3126 cConsecutiveTimeouts = 0;
3127 if (fInitialMeasurement)
3128 {
3129 int cTries = 8;
3130 int cMsWaitPerTry = 10;
3131 fInitialMeasurement = false;
3132 do
3133 {
3134 rc = supdrvMeasureInitialTscDeltas(pDevExt);
3135 if ( RT_SUCCESS(rc)
3136 || ( RT_FAILURE(rc)
3137 && rc != VERR_TRY_AGAIN
3138 && rc != VERR_CPU_OFFLINE))
3139 {
3140 break;
3141 }
3142 RTThreadSleep(cMsWaitPerTry);
3143 } while (cTries-- > 0);
3144 }
3145 else
3146 {
3147 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
3148 unsigned iCpu;
3149
3150 /* Measure TSC-deltas only for the CPUs that are in the set. */
3151 rc = VINF_SUCCESS;
3152 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
3153 {
3154 PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
3155 if ( pGipCpuWorker->i64TSCDelta == INT64_MAX
3156 && RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
3157 {
3158 rc |= supdrvMeasureTscDeltaOne(pDevExt, iCpu);
3159 }
3160 }
3161 }
3162 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3163 if (pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
3164 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
3165 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3166 Assert(rc != VERR_NOT_AVAILABLE); /* VERR_NOT_AVAILABLE is used as the initial value. */
3167 ASMAtomicWriteS32(&pDevExt->rcTscDelta, rc);
3168 break;
3169 }
3170
3171 case kTscDeltaThreadState_Terminating:
3172 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Destroyed;
3173 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3174 return VINF_SUCCESS;
3175
3176 case kTscDeltaThreadState_Butchered:
3177 default:
3178 return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "Invalid state", VERR_INVALID_STATE);
3179 }
3180 }
3181
3182 return rc;
3183}
3184
3185
3186/**
3187 * Waits for the TSC-delta measurement thread to respond to a state change.
3188 *
3189 * @returns VINF_SUCCESS on success, VERR_TIMEOUT if it doesn't respond in time,
3190 * other error code on internal error.
3191 *
3192 * @param pThis Pointer to the grant service instance data.
3193 * @param enmCurState The current state.
3194 * @param enmNewState The new state we're waiting for it to enter.
3195 */
3196static int supdrvTscDeltaThreadWait(PSUPDRVDEVEXT pDevExt, SUPDRVTSCDELTATHREADSTATE enmCurState,
3197 SUPDRVTSCDELTATHREADSTATE enmNewState)
3198{
3199 /*
3200 * Wait a short while for the expected state transition.
3201 */
3202 int rc;
3203 RTSemEventWait(pDevExt->hTscDeltaEvent, RT_MS_1SEC);
3204 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3205 if (pDevExt->enmTscDeltaThreadState == enmNewState)
3206 {
3207 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3208 rc = VINF_SUCCESS;
3209 }
3210 else if (pDevExt->enmTscDeltaThreadState == enmCurState)
3211 {
3212 /*
3213 * Wait longer if the state has not yet transitioned to the one we want.
3214 */
3215 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3216 rc = RTSemEventWait(pDevExt->hTscDeltaEvent, 50 * RT_MS_1SEC);
3217 if ( RT_SUCCESS(rc)
3218 || rc == VERR_TIMEOUT)
3219 {
3220 /*
3221 * Check the state whether we've succeeded.
3222 */
3223 SUPDRVTSCDELTATHREADSTATE enmState;
3224 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3225 enmState = pDevExt->enmTscDeltaThreadState;
3226 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3227 if (enmState == enmNewState)
3228 rc = VINF_SUCCESS;
3229 else if (enmState == enmCurState)
3230 {
3231 rc = VERR_TIMEOUT;
3232 OSDBGPRINT(("supdrvTscDeltaThreadWait: timed out state transition. enmState=%d enmNewState=%d\n", enmState,
3233 enmNewState));
3234 }
3235 else
3236 {
3237 rc = VERR_INTERNAL_ERROR;
3238 OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d, expected %d\n", enmCurState,
3239 enmState, enmNewState));
3240 }
3241 }
3242 else
3243 OSDBGPRINT(("supdrvTscDeltaThreadWait: RTSemEventWait failed. rc=%Rrc\n", rc));
3244 }
3245 else
3246 {
3247 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3248 OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d\n", enmCurState, enmNewState));
3249 rc = VERR_INTERNAL_ERROR;
3250 }
3251
3252 return rc;
3253}
3254
3255
3256/**
3257 * Waits for TSC-delta measurements to be completed for all online CPUs.
3258 *
3259 * @returns VBox status code.
3260 * @param pDevExt Pointer to the device instance data.
3261 */
3262static int supdrvTscDeltaThreadWaitForOnlineCpus(PSUPDRVDEVEXT pDevExt)
3263{
3264 int cTriesLeft = 5;
3265 int cMsTotalWait;
3266 int cMsWaited = 0;
3267 int cMsWaitGranularity = 1;
3268
3269 PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
3270 AssertReturn(pGip, VERR_INVALID_POINTER);
3271
3272 if (RT_UNLIKELY(pDevExt->hTscDeltaThread == NIL_RTTHREAD))
3273 return VERR_THREAD_NOT_WAITABLE;
3274
3275 cMsTotalWait = RT_MIN(pGip->cPresentCpus + 10, 200);
3276 while (cTriesLeft-- > 0)
3277 {
3278 if (RTCpuSetIsEqual(&pDevExt->TscDeltaObtainedCpuSet, &pGip->OnlineCpuSet))
3279 return VINF_SUCCESS;
3280 RTThreadSleep(cMsWaitGranularity);
3281 cMsWaited += cMsWaitGranularity;
3282 if (cMsWaited >= cMsTotalWait)
3283 break;
3284 }
3285
3286 return VERR_TIMEOUT;
3287}
3288
3289
3290/**
3291 * Terminates the actual thread running supdrvTscDeltaThread().
3292 *
3293 * This is an internal worker function for supdrvTscDeltaThreadInit() and
3294 * supdrvTscDeltaTerm().
3295 *
3296 * @param pDevExt Pointer to the device instance data.
3297 */
3298static void supdrvTscDeltaThreadTerminate(PSUPDRVDEVEXT pDevExt)
3299{
3300 int rc;
3301 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3302 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Terminating;
3303 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3304 RTThreadUserSignal(pDevExt->hTscDeltaThread);
3305 rc = RTThreadWait(pDevExt->hTscDeltaThread, 50 * RT_MS_1SEC, NULL /* prc */);
3306 if (RT_FAILURE(rc))
3307 {
3308 /* Signal a few more times before giving up. */
3309 int cTriesLeft = 5;
3310 while (--cTriesLeft > 0)
3311 {
3312 RTThreadUserSignal(pDevExt->hTscDeltaThread);
3313 rc = RTThreadWait(pDevExt->hTscDeltaThread, 2 * RT_MS_1SEC, NULL /* prc */);
3314 if (rc != VERR_TIMEOUT)
3315 break;
3316 }
3317 }
3318}
3319
3320
3321/**
3322 * Initializes and spawns the TSC-delta measurement thread.
3323 *
3324 * A thread is required for servicing re-measurement requests from events like
3325 * CPUs coming online, suspend/resume etc. as it cannot be done synchronously
3326 * under all contexts on all OSs.
3327 *
3328 * @returns VBox status code.
3329 * @param pDevExt Pointer to the device instance data.
3330 *
3331 * @remarks Must only be called -after- initializing GIP and setting up MP
3332 * notifications!
3333 */
3334static int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt)
3335{
3336 int rc;
3337 Assert(pDevExt->pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
3338 rc = RTSpinlockCreate(&pDevExt->hTscDeltaSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "VBoxTscSpnLck");
3339 if (RT_SUCCESS(rc))
3340 {
3341 rc = RTSemEventCreate(&pDevExt->hTscDeltaEvent);
3342 if (RT_SUCCESS(rc))
3343 {
3344 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Creating;
3345 pDevExt->cMsTscDeltaTimeout = 1;
3346 rc = RTThreadCreate(&pDevExt->hTscDeltaThread, supdrvTscDeltaThread, pDevExt, 0 /* cbStack */,
3347 RTTHREADTYPE_DEFAULT, RTTHREADFLAGS_WAITABLE, "VBoxTscThread");
3348 if (RT_SUCCESS(rc))
3349 {
3350 rc = supdrvTscDeltaThreadWait(pDevExt, kTscDeltaThreadState_Creating, kTscDeltaThreadState_Listening);
3351 if (RT_SUCCESS(rc))
3352 {
3353 ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
3354 return rc;
3355 }
3356
3357 OSDBGPRINT(("supdrvTscDeltaInit: supdrvTscDeltaThreadWait failed. rc=%Rrc\n", rc));
3358 supdrvTscDeltaThreadTerminate(pDevExt);
3359 }
3360 else
3361 OSDBGPRINT(("supdrvTscDeltaInit: RTThreadCreate failed. rc=%Rrc\n", rc));
3362 RTSemEventDestroy(pDevExt->hTscDeltaEvent);
3363 pDevExt->hTscDeltaEvent = NIL_RTSEMEVENT;
3364 }
3365 else
3366 OSDBGPRINT(("supdrvTscDeltaInit: RTSemEventCreate failed. rc=%Rrc\n", rc));
3367 RTSpinlockDestroy(pDevExt->hTscDeltaSpinlock);
3368 pDevExt->hTscDeltaSpinlock = NIL_RTSPINLOCK;
3369 }
3370 else
3371 OSDBGPRINT(("supdrvTscDeltaInit: RTSpinlockCreate failed. rc=%Rrc\n", rc));
3372
3373 return rc;
3374}
3375
3376
3377/**
3378 * Terminates the TSC-delta measurement thread and cleanup.
3379 *
3380 * @param pDevExt Pointer to the device instance data.
3381 */
3382static void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt)
3383{
3384 if ( pDevExt->hTscDeltaSpinlock != NIL_RTSPINLOCK
3385 && pDevExt->hTscDeltaEvent != NIL_RTSEMEVENT)
3386 {
3387 supdrvTscDeltaThreadTerminate(pDevExt);
3388 }
3389
3390 if (pDevExt->hTscDeltaSpinlock != NIL_RTSPINLOCK)
3391 {
3392 RTSpinlockDestroy(pDevExt->hTscDeltaSpinlock);
3393 pDevExt->hTscDeltaSpinlock = NIL_RTSPINLOCK;
3394 }
3395
3396 if (pDevExt->hTscDeltaEvent != NIL_RTSEMEVENT)
3397 {
3398 RTSemEventDestroy(pDevExt->hTscDeltaEvent);
3399 pDevExt->hTscDeltaEvent = NIL_RTSEMEVENT;
3400 }
3401
3402 ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
3403}
3404
3405
3406#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
3407
3408
3409/**
3410 * Service a TSC-delta measurement request.
3411 *
3412 * @returns VBox status code.
3413 * @param pDevExt Pointer to the device instance data.
3414 * @param pSession The support driver session.
3415 * @param pReq Pointer to the TSC-delta measurement request.
3416 */
3417int VBOXCALL supdrvIOCtl_TscDeltaMeasure(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCDELTAMEASURE pReq)
3418{
3419 PSUPGLOBALINFOPAGE pGip;
3420 RTCPUID idCpuWorker;
3421 int rc;
3422 int16_t cTries;
3423 RTMSINTERVAL cMsWaitRetry;
3424 uint16_t iCpu;
3425
3426 /*
3427 * Validate.
3428 */
3429 AssertPtr(pDevExt); AssertPtr(pSession); AssertPtr(pReq); /* paranoia^2 */
3430 if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
3431 return VERR_WRONG_ORDER;
3432 pGip = pDevExt->pGip;
3433 AssertReturn(pGip, VERR_INTERNAL_ERROR_2);
3434
3435 idCpuWorker = pReq->u.In.idCpu;
3436 if (idCpuWorker == NIL_RTCPUID)
3437 return VERR_INVALID_CPU_ID;
3438 cTries = RT_MAX(pReq->u.In.cRetries + 1, 10);
3439 cMsWaitRetry = RT_MAX(pReq->u.In.cMsWaitRetry, 5);
3440
3441 /*
3442 * The request is a noop if the TSC delta isn't being used.
3443 */
3444 pGip = pDevExt->pGip;
3445 if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED)
3446 return VINF_SUCCESS;
3447
3448 rc = VERR_CPU_NOT_FOUND;
3449 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
3450 {
3451 PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
3452 if (pGipCpuWorker->idCpu == idCpuWorker)
3453 {
3454 if ( pGipCpuWorker->i64TSCDelta != INT64_MAX
3455 && !pReq->u.In.fForce)
3456 return VINF_SUCCESS;
3457
3458#ifdef SUPDRV_USE_TSC_DELTA_THREAD
3459 if (pReq->u.In.fAsync)
3460 {
3461 /** @todo Async. doesn't implement options like retries, waiting. We'll need
3462 * to pass those options to the thread somehow and implement it in the
3463 * thread. Check if anyone uses/needs fAsync before implementing this. */
3464 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3465 RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
3466 if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
3467 || pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
3468 {
3469 pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
3470 }
3471 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3472 RTThreadUserSignal(pDevExt->hTscDeltaThread);
3473 return VINF_SUCCESS;
3474 }
3475
3476 /*
3477 * If a TSC-delta measurement request is already being serviced by the thread,
3478 * wait 'cTries' times if a retry-timeout is provided, otherwise bail as busy.
3479 */
3480 while (cTries-- > 0)
3481 {
3482 SUPDRVTSCDELTATHREADSTATE enmState;
3483 RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
3484 enmState = pDevExt->enmTscDeltaThreadState;
3485 RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
3486
3487 if ( enmState == kTscDeltaThreadState_Measuring
3488 || enmState == kTscDeltaThreadState_WaitAndMeasure)
3489 {
3490 if ( !cTries
3491 || !cMsWaitRetry)
3492 return VERR_SUPDRV_TSC_DELTA_MEASUREMENT_BUSY;
3493 if (cMsWaitRetry)
3494 RTThreadSleep(cMsWaitRetry);
3495 }
3496 }
3497 cTries = RT_MAX(pReq->u.In.cRetries + 1, 10);
3498#endif
3499
3500 while (cTries-- > 0)
3501 {
3502 rc = supdrvMeasureTscDeltaOne(pDevExt, iCpu);
3503 if (RT_SUCCESS(rc))
3504 {
3505 Assert(pGipCpuWorker->i64TSCDelta != INT64_MAX);
3506 break;
3507 }
3508
3509 if (cMsWaitRetry)
3510 RTThreadSleep(cMsWaitRetry);
3511 }
3512
3513 break;
3514 }
3515 }
3516 return rc;
3517}
3518
3519
3520/**
3521 * Reads TSC with delta applied.
3522 *
3523 * Will try to resolve delta value INT64_MAX before applying it. This is the
3524 * main purpose of this function, to handle the case where the delta needs to be
3525 * determined.
3526 *
3527 * @returns VBox status code.
3528 * @param pDevExt Pointer to the device instance data.
3529 * @param pSession The support driver session.
3530 * @param pReq Pointer to the TSC-read request.
3531 */
3532int VBOXCALL supdrvIOCtl_TscRead(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCREAD pReq)
3533{
3534 PSUPGLOBALINFOPAGE pGip;
3535 int rc;
3536
3537 /*
3538 * Validate. We require the client to have mapped GIP (no asserting on
3539 * ring-3 preconditions).
3540 */
3541 AssertPtr(pDevExt); AssertPtr(pReq); AssertPtr(pSession); /* paranoia^2 */
3542 if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
3543 return VERR_WRONG_ORDER;
3544 pGip = pDevExt->pGip;
3545 AssertReturn(pGip, VERR_INTERNAL_ERROR_2);
3546
3547 /*
3548 * We're usually here because we need to apply delta, but we shouldn't be
3549 * upset if the GIP is some different mode.
3550 */
3551 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
3552 {
3553 uint32_t cTries = 0;
3554 for (;;)
3555 {
3556 /*
3557 * Start by gathering the data, using CLI for disabling preemption
3558 * while we do that.
3559 */
3560 RTCCUINTREG uFlags = ASMIntDisableFlags();
3561 int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
3562 int iGipCpu;
3563 if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
3564 && (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
3565 {
3566 int64_t i64Delta = pGip->aCPUs[iGipCpu].i64TSCDelta;
3567 pReq->u.Out.idApic = pGip->aCPUs[iGipCpu].idApic;
3568 pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
3569 ASMSetFlags(uFlags);
3570
3571 /*
3572 * If we're lucky we've got a delta, but no predicitions here
3573 * as this I/O control is normally only used when the TSC delta
3574 * is set to INT64_MAX.
3575 */
3576 if (i64Delta != INT64_MAX)
3577 {
3578 pReq->u.Out.u64AdjustedTsc -= i64Delta;
3579 rc = VINF_SUCCESS;
3580 break;
3581 }
3582
3583 /* Give up after a few times. */
3584 if (cTries >= 4)
3585 {
3586 rc = VWRN_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED;
3587 break;
3588 }
3589
3590 /* Need to measure the delta an try again. */
3591 rc = supdrvMeasureTscDeltaOne(pDevExt, iGipCpu);
3592 Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX || RT_FAILURE_NP(rc));
3593 }
3594 else
3595 {
3596 /* This really shouldn't happen. */
3597 AssertMsgFailed(("idCpu=%#x iCpuSet=%#x (%d)\n", RTMpCpuId(), iCpuSet, iCpuSet));
3598 pReq->u.Out.idApic = ASMGetApicId();
3599 pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
3600 ASMSetFlags(uFlags);
3601 rc = VERR_INTERNAL_ERROR_5; /** @todo change to warning. */
3602 break;
3603 }
3604 }
3605 }
3606 else
3607 {
3608 /*
3609 * No delta to apply. Easy. Deal with preemption the lazy way.
3610 */
3611 RTCCUINTREG uFlags = ASMIntDisableFlags();
3612 int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
3613 int iGipCpu;
3614 if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
3615 && (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
3616 pReq->u.Out.idApic = pGip->aCPUs[iGipCpu].idApic;
3617 else
3618 pReq->u.Out.idApic = ASMGetApicId();
3619 pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
3620 ASMSetFlags(uFlags);
3621 rc = VINF_SUCCESS;
3622 }
3623
3624 return rc;
3625}
3626
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette