VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/time/timesupref.h

Last change on this file was 98103, checked in by vboxsync, 16 months ago

Copyright year updates by scm.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 18.6 KB
Line 
1/* $Id: timesupref.h 98103 2023-01-17 14:15:46Z vboxsync $ */
2/** @file
3 * IPRT - Time using SUPLib, the C Code Template.
4 */
5
6/*
7 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/**
39 * The C reference implementation of the assembly routines.
40 *
41 * Calculate NanoTS using the information in the global information page (GIP)
42 * which the support library (SUPLib) exports.
43 *
44 * This function guarantees that the returned timestamp is later (in time) than
45 * any previous calls in the same thread.
46 *
47 * @remark The way the ever increasing time guarantee is currently implemented means
48 * that if you call this function at a frequency higher than 1GHz you're in for
49 * trouble. We currently assume that no idiot will do that for real life purposes.
50 *
51 * @returns Nanosecond timestamp.
52 * @param pData Pointer to the data structure.
53 * @param pExtra Where to return extra time info. Optional.
54 */
55RTDECL(uint64_t) rtTimeNanoTSInternalRef(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra)
56{
57#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
58 PSUPGIPCPU pGipCpuAttemptedTscRecalibration = NULL;
59#endif
60 AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
61
62 for (;;)
63 {
64#ifndef IN_RING3 /* This simplifies and improves everything. */
65 RTCCUINTREG const uFlags = ASMIntDisableFlags();
66#endif
67
68 /*
69 * Check that the GIP is sane and that the premises for this worker function
70 * hasn't changed (CPU onlined with bad delta or missing features).
71 */
72 PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
73 if ( RT_LIKELY(pGip)
74 && RT_LIKELY(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
75#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
76 && RT_LIKELY(pGip->enmUseTscDelta >= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
77#else
78 && RT_LIKELY(pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO)
79#endif
80#if defined(IN_RING3) && TMPL_GET_CPU_METHOD != 0
81 && RT_LIKELY(pGip->fGetGipCpu & TMPL_GET_CPU_METHOD)
82#endif
83 )
84 {
85 /*
86 * Resolve pGipCpu if needed. If the instruction is serializing, we
87 * read the transaction id first if possible.
88 */
89#if TMPL_MODE == TMPL_MODE_ASYNC || TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
90# if defined(IN_RING0)
91 uint32_t const iCpuSet = RTMpCurSetIndex();
92 uint16_t const iGipCpu = iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
93 ? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
94# elif defined(IN_RC)
95 uint32_t const iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet;
96 uint16_t const iGipCpu = iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
97 ? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
98# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID
99# if TMPL_MODE != TMPL_MODE_ASYNC
100 uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
101# endif
102 uint8_t const idApic = ASMGetApicId();
103 uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
104# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_0B
105# if TMPL_MODE != TMPL_MODE_ASYNC
106 uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
107# endif
108 uint32_t const idApic = ASMGetApicIdExt0B();
109 uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
110# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_8000001E
111# if TMPL_MODE != TMPL_MODE_ASYNC
112 uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
113# endif
114 uint32_t const idApic = ASMGetApicIdExt8000001E();
115 uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
116# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
117 || TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
118# if TMPL_MODE != TMPL_MODE_ASYNC
119 uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
120# endif
121 uint32_t uAux;
122 ASMReadTscWithAux(&uAux);
123# if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS
124 uint16_t const iCpuSet = uAux & (RTCPUSET_MAX_CPUS - 1);
125# else
126 uint16_t iCpuSet = 0;
127 uint16_t offGipCpuGroup = pGip->aoffCpuGroup[(uAux >> 8) & UINT8_MAX];
128 if (offGipCpuGroup < pGip->cPages * PAGE_SIZE)
129 {
130 PSUPGIPCPUGROUP pGipCpuGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offGipCpuGroup);
131 if ( (uAux & UINT8_MAX) < pGipCpuGroup->cMaxMembers
132 && pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX] != -1)
133 iCpuSet = pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX];
134 }
135# endif
136 uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
137# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS
138 uint16_t const cbLim = ASMGetIdtrLimit();
139 uint16_t const iCpuSet = (cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8)) & (RTCPUSET_MAX_CPUS - 1);
140 uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
141# else
142# error "What?"
143# endif
144 if (RT_LIKELY(iGipCpu < pGip->cCpus))
145 {
146 PSUPGIPCPU pGipCpu = &pGip->aCPUs[iGipCpu];
147#else
148 {
149#endif
150 /*
151 * Get the transaction ID if necessary and we haven't already
152 * read it before a serializing instruction above. We can skip
153 * this for ASYNC_TSC mode in ring-0 and raw-mode context since
154 * we disable interrupts.
155 */
156#if TMPL_MODE == TMPL_MODE_ASYNC && defined(IN_RING3)
157 uint32_t const u32TransactionId = pGipCpu->u32TransactionId;
158 ASMCompilerBarrier();
159 TMPL_READ_FENCE();
160#elif TMPL_MODE != TMPL_MODE_ASYNC \
161 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID \
162 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B \
163 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E \
164 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
165 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
166 uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
167 ASMCompilerBarrier();
168 TMPL_READ_FENCE();
169#endif
170
171 /*
172 * Gather all the data we need. The mess at the end is to make
173 * sure all loads are done before we recheck the transaction ID
174 * without triggering serializing twice.
175 */
176 uint32_t u32NanoTSFactor0 = pGip->u32UpdateIntervalNS;
177#if TMPL_MODE == TMPL_MODE_ASYNC
178 uint32_t u32UpdateIntervalTSC = pGipCpu->u32UpdateIntervalTSC;
179 uint64_t u64NanoTS = pGipCpu->u64NanoTS;
180 uint64_t u64TSC = pGipCpu->u64TSC;
181#else
182 uint32_t u32UpdateIntervalTSC = pGip->aCPUs[0].u32UpdateIntervalTSC;
183 uint64_t u64NanoTS = pGip->aCPUs[0].u64NanoTS;
184 uint64_t u64TSC = pGip->aCPUs[0].u64TSC;
185# if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
186 int64_t i64TscDelta = pGipCpu->i64TSCDelta;
187# endif
188#endif
189 uint64_t u64PrevNanoTS = ASMAtomicUoReadU64(pData->pu64Prev);
190#if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
191 || TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
192 ASMCompilerBarrier();
193 uint32_t uAux2;
194 uint64_t u64Delta = ASMReadTscWithAux(&uAux2); /* serializing */
195#else
196 uint64_t u64Delta = ASMReadTSC();
197 ASMCompilerBarrier();
198# if TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID /* getting APIC will serialize */ \
199 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B \
200 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E \
201 && (defined(IN_RING3) || TMPL_MODE != TMPL_MODE_ASYNC)
202 TMPL_READ_FENCE(); /* Expensive (~30 ticks). Would like convincing argumentation that let us remove it. */
203# endif
204#endif
205
206 /*
207 * Check that we didn't change CPU.
208 */
209#if defined(IN_RING3) && ( TMPL_MODE == TMPL_MODE_ASYNC || TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA )
210# if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID
211 if (RT_LIKELY(ASMGetApicId() == idApic))
212# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_0B
213 if (RT_LIKELY(ASMGetApicIdExt0B() == idApic))
214# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_8000001E
215 if (RT_LIKELY(ASMGetApicIdExt8000001E() == idApic))
216# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
217 || TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
218 if (RT_LIKELY(uAux2 == uAux))
219# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS
220 if (RT_LIKELY(ASMGetIdtrLimit() == cbLim))
221# endif
222#endif
223 {
224 /*
225 * Check the transaction ID (see above for R0/RC + ASYNC).
226 */
227#if defined(IN_RING3) || TMPL_MODE != TMPL_MODE_ASYNC
228# if TMPL_MODE == TMPL_MODE_ASYNC
229 if (RT_LIKELY(pGipCpu->u32TransactionId == u32TransactionId && !(u32TransactionId & 1) ))
230# else
231 if (RT_LIKELY(pGip->aCPUs[0].u32TransactionId == u32TransactionId && !(u32TransactionId & 1) ))
232# endif
233#endif
234 {
235
236 /*
237 * Apply the TSC delta. If the delta is invalid and the
238 * execution allows it, try trigger delta recalibration.
239 */
240#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
241 if (RT_LIKELY( i64TscDelta != INT64_MAX
242 || pGipCpu == pGipCpuAttemptedTscRecalibration))
243#endif
244 {
245#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
246# ifndef IN_RING3
247 if (RT_LIKELY(i64TscDelta != INT64_MAX))
248# endif
249 u64Delta -= i64TscDelta;
250#endif
251
252 /*
253 * Bingo! We've got a consistent set of data.
254 */
255#ifndef IN_RING3
256 ASMSetFlags(uFlags);
257#endif
258
259 if (pExtra)
260 pExtra->uTSCValue = u64Delta;
261
262 /*
263 * Calc NanoTS delta.
264 */
265 u64Delta -= u64TSC;
266 if (RT_LIKELY(u64Delta <= u32UpdateIntervalTSC))
267 { /* MSVC branch hint, probably pointless. */ }
268 else
269 {
270 /*
271 * We've expired the interval, cap it. If we're here for the 2nd
272 * time without any GIP update in-between, the checks against
273 * *pu64Prev below will force 1ns stepping.
274 */
275 ASMAtomicIncU32(&pData->cExpired);
276 u64Delta = u32UpdateIntervalTSC;
277 }
278#if !defined(_MSC_VER) || !defined(RT_ARCH_X86) /* GCC makes very pretty code from these two inline calls, while MSC cannot. */
279 u64Delta = ASMMult2xU32RetU64((uint32_t)u64Delta, u32NanoTSFactor0);
280 u64Delta = ASMDivU64ByU32RetU32(u64Delta, u32UpdateIntervalTSC);
281#else
282 __asm
283 {
284 mov eax, dword ptr [u64Delta]
285 mul dword ptr [u32NanoTSFactor0]
286 div dword ptr [u32UpdateIntervalTSC]
287 mov dword ptr [u64Delta], eax
288 xor edx, edx
289 mov dword ptr [u64Delta + 4], edx
290 }
291#endif
292
293 /*
294 * Calculate the time and compare it with the previously returned value.
295 */
296 u64NanoTS += u64Delta;
297 uint64_t u64DeltaPrev = u64NanoTS - u64PrevNanoTS;
298 if (RT_LIKELY( u64DeltaPrev > 0
299 && u64DeltaPrev < UINT64_C(86000000000000) /* 24h */))
300 { /* Frequent - less than 24h since last call. */ }
301 else if (RT_LIKELY( (int64_t)u64DeltaPrev <= 0
302 && (int64_t)u64DeltaPrev + u32NanoTSFactor0 * 2 >= 0))
303 {
304 /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
305 ASMAtomicIncU32(&pData->c1nsSteps);
306 u64NanoTS = u64PrevNanoTS + 1;
307 }
308 else if (!u64PrevNanoTS)
309 /* We're resuming (see TMVirtualResume). */;
310 else
311 {
312 /* Something has gone bust, if negative offset it's real bad. */
313 ASMAtomicIncU32(&pData->cBadPrev);
314 pData->pfnBad(pData, u64NanoTS, u64DeltaPrev, u64PrevNanoTS);
315 }
316
317 /*
318 * Attempt updating the previous value, provided we're still ahead of it.
319 *
320 * There is no point in recalculating u64NanoTS because we got preempted or if
321 * we raced somebody while the GIP was updated, since these are events
322 * that might occur at any point in the return path as well.
323 */
324 if (RT_LIKELY(ASMAtomicCmpXchgU64(pData->pu64Prev, u64NanoTS, u64PrevNanoTS)))
325 return u64NanoTS;
326
327 ASMAtomicIncU32(&pData->cUpdateRaces);
328 for (int cTries = 25; cTries > 0; cTries--)
329 {
330 u64PrevNanoTS = ASMAtomicReadU64(pData->pu64Prev);
331 if (u64PrevNanoTS >= u64NanoTS)
332 break;
333 if (ASMAtomicCmpXchgU64(pData->pu64Prev, u64NanoTS, u64PrevNanoTS))
334 break;
335 ASMNopPause();
336 }
337 return u64NanoTS;
338 }
339
340#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
341 /*
342 * Call into the support driver to try make it recalculate the delta. We
343 * remember which GIP CPU structure we're probably working on so we won't
344 * end up in a loop if the driver for some reason cannot get the job done.
345 */
346 else /* else is unecessary, but helps checking the preprocessor spaghetti. */
347 {
348 pGipCpuAttemptedTscRecalibration = pGipCpu;
349 uint64_t u64TscTmp;
350 uint16_t idApicUpdate;
351 int rc = SUPR3ReadTsc(&u64TscTmp, &idApicUpdate);
352 if (RT_SUCCESS(rc) && idApicUpdate < RT_ELEMENTS(pGip->aiCpuFromApicId))
353 {
354 uint32_t iUpdateGipCpu = pGip->aiCpuFromApicId[idApicUpdate];
355 if (iUpdateGipCpu < pGip->cCpus)
356 pGipCpuAttemptedTscRecalibration = &pGip->aCPUs[iUpdateGipCpu];
357 }
358 }
359#endif
360 }
361 }
362
363 /*
364 * No joy must try again.
365 */
366#ifdef _MSC_VER
367# pragma warning(disable: 4702)
368#endif
369#ifndef IN_RING3
370 ASMSetFlags(uFlags);
371#endif
372 ASMNopPause();
373 continue;
374 }
375
376#if TMPL_MODE == TMPL_MODE_ASYNC || TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
377 /*
378 * We've got a bad CPU or APIC index of some kind.
379 */
380 else /* else is unecessary, but helps checking the preprocessor spaghetti. */
381 {
382# ifndef IN_RING3
383 ASMSetFlags(uFlags);
384# endif
385# if defined(IN_RING0) \
386 || defined(IN_RC) \
387 || ( TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID \
388 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B /*?*/ \
389 && TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E /*?*/)
390 return pData->pfnBadCpuIndex(pData, pExtra, UINT16_MAX-1, iCpuSet, iGipCpu);
391# else
392 return pData->pfnBadCpuIndex(pData, pExtra, idApic, UINT16_MAX-1, iGipCpu);
393# endif
394 }
395#endif
396 }
397
398 /*
399 * Something changed in the GIP config or it was unmapped, figure out
400 * the right worker function to use now.
401 */
402#ifndef IN_RING3
403 ASMSetFlags(uFlags);
404#endif
405 return pData->pfnRediscover(pData, pExtra);
406 }
407}
408
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use