VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/time/timesupA.mac

Last change on this file was 98103, checked in by vboxsync, 16 months ago

Copyright year updates by scm.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 23.6 KB
Line 
1; $Id: timesupA.mac 98103 2023-01-17 14:15:46Z vboxsync $
2;; @file
3; IPRT - Time using SUPLib, the Assembly Code Template.
4;
5
6;
7; Copyright (C) 2006-2023 Oracle and/or its affiliates.
8;
9; This file is part of VirtualBox base platform packages, as
10; available from https://www.virtualbox.org.
11;
12; This program is free software; you can redistribute it and/or
13; modify it under the terms of the GNU General Public License
14; as published by the Free Software Foundation, in version 3 of the
15; License.
16;
17; This program is distributed in the hope that it will be useful, but
18; WITHOUT ANY WARRANTY; without even the implied warranty of
19; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20; General Public License for more details.
21;
22; You should have received a copy of the GNU General Public License
23; along with this program; if not, see <https://www.gnu.org/licenses>.
24;
25; The contents of this file may alternatively be used under the terms
26; of the Common Development and Distribution License Version 1.0
27; (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28; in the VirtualBox distribution, in which case the provisions of the
29; CDDL are applicable instead of those of the GPL.
30;
31; You may elect to license modified versions of this file under the
32; terms and conditions of either the GPL or the CDDL or both.
33;
34; SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35;
36
37%ifdef RT_ARCH_X86
38;;
39; The x86 assembly implementation of the assembly routines.
40;
41; @returns Nanosecond timestamp.
42; @param pData Pointer to the nanosecond timestamp data.
43;
44BEGINPROC rtTimeNanoTSInternalAsm
45 ;
46 ; Variable definitions.
47 ;
48%define pData [ebp + 08h]
49%define u64RetNanoTS_Hi [ebp - 04h]
50%define u64RetNanoTS [ebp - 08h]
51%define u32UpdateIntervalNS [ebp - 0ch]
52%define u32UpdateIntervalTSC [ebp - 10h]
53%define u64TSC_Hi [ebp - 14h]
54%define u64TSC [ebp - 18h]
55%define u64CurNanoTS_Hi [ebp - 1ch]
56%define u64CurNanoTS [ebp - 20h]
57%define u64PrevNanoTS_Hi [ebp - 24h]
58%define u64PrevNanoTS [ebp - 28h]
59%define u32TransactionId [ebp - 2ch]
60%define u32ApicIdPlus [ebp - 30h]
61%define TmpVar [ebp - 34h]
62%define SavedEBX [ebp - 38h]
63%define SavedEDI [ebp - 3ch]
64%define SavedESI [ebp - 40h]
65
66 ;
67 ; Prolog.
68 ;
69 push ebp
70 mov ebp, esp
71 sub esp, 40h
72 mov SavedEBX, ebx
73 mov SavedEDI, edi
74 mov SavedESI, esi
75
76
77 ;;
78 ;; Read the GIP data and the previous value.
79 ;;
80.ReadGip:
81
82
83 ;
84 ; Load pGip.
85 ;
86%ifdef IMPORTED_SUPLIB
87 %ifdef IN_RING0
88 mov esi, IMP(g_SUPGlobalInfoPage)
89 %else
90 mov esi, IMP(g_pSUPGlobalInfoPage)
91 mov esi, [esi]
92 %endif
93%else
94 mov esi, [NAME(g_pSUPGlobalInfoPage)]
95%endif
96 or esi, esi
97 jz .Rediscover
98 cmp dword [esi + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
99 jne .Rediscover
100
101 ;
102 ; Calc pGipCPU, setting u32ApicIdPlus if necessary.
103 ;
104%ifdef NEED_APIC_ID
105 ; u8ApicId = ASMGetApicId();
106 mov eax, 1
107 cpuid ; expensive
108 %ifdef NEED_TRANSACTION_ID
109 mov u32ApicIdPlus, ebx
110 %endif
111 ; pGipCpu/pGipCpuDelta = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
112 shr ebx, 24
113 movzx ebx, word [esi + ebx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
114 mov eax, SUPGIPCPU_size
115 mul ebx
116 lea edi, [esi + eax + SUPGLOBALINFOPAGE.aCPUs] ; edi == &pGip->aCPU[u8ApicId];
117%endif
118
119%ifdef NEED_TRANSACTION_ID
120 ;
121 ; Serialized loading of u32TransactionId.
122 ;
123 %ifdef ASYNC_GIP
124 mov ebx, [edi + SUPGIPCPU.u32TransactionId]
125 %else
126 mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
127 %endif
128 mov u32TransactionId, ebx
129 %ifdef USE_LFENCE
130 lfence
131 %else
132 lock xor dword TmpVar, 0
133 %endif
134%endif
135
136 ;
137 ; Load the data and TSC with delta applied.
138 ;
139 mov eax, [esi + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
140 mov u32UpdateIntervalNS, eax
141%ifdef ASYNC_GIP ; esi is now free.
142 mov edx, [edi + SUPGIPCPU.u32UpdateIntervalTSC]
143%else
144 mov edx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
145%endif
146 mov u32UpdateIntervalTSC, edx
147
148 rdtsc
149%ifdef WITH_TSC_DELTA
150 cmp dword [edi + SUPGIPCPU.i64TSCDelta], 0xffffffff
151 je .TscDeltaPossiblyInvalid
152.TscDeltaValid:
153 sub eax, dword [edi + SUPGIPCPU.i64TSCDelta]
154 sbb edx, dword [edi + SUPGIPCPU.i64TSCDelta + 4]
155.TscDeltaNotValid: ; edi is now free.
156%endif
157
158%ifdef ASYNC_GIP
159 mov ecx, [edi + SUPGIPCPU.u64NanoTS]
160 mov esi, [edi + SUPGIPCPU.u64NanoTS + 4]
161%else
162 mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
163 mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS + 4]
164%endif
165 mov u64CurNanoTS, ecx
166 mov u64CurNanoTS_Hi, ebx
167%ifdef ASYNC_GIP
168 mov ecx, [edi + SUPGIPCPU.u64TSC]
169 mov ebx, [edi + SUPGIPCPU.u64TSC + 4]
170%else
171 mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC]
172 mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC + 4]
173%endif
174 mov u64TSC, ecx
175 mov u64TSC_Hi, ebx
176
177 ; u64PrevNanoTS = ASMAtomicReadU64(pu64Prev);
178 ; This serializes load/save. And with the dependency on the
179 ; RDTSC result, we try to make sure it has completed as well.
180%ifdef ASYNC_GIP
181 mov esi, pData
182 mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
183%else
184 mov edi, pData
185 mov edi, [esi + RTTIMENANOTSDATA.pu64Prev]
186%endif
187 mov ebx, eax
188 mov ecx, edx
189%ifdef ASYNC_GIP
190 lock cmpxchg8b [esi]
191%else
192 lock cmpxchg8b [edi]
193%endif
194 mov u64PrevNanoTS, eax
195 mov u64PrevNanoTS_Hi, edx
196
197%undef SAVED_u64RetNanoTS
198%ifdef NEED_TRANSACTION_ID
199 ;
200 ; Check that the GIP and CPU didn't change.
201 ; We've already serialized all the loads and stores at this point.
202 ;
203 %ifdef NEED_APIC_ID
204 mov u64RetNanoTS, ebx
205 mov u64RetNanoTS_Hi, ecx
206 %define SAVED_u64RetNanoTS
207 mov eax, 1
208 cpuid
209 cmp u32ApicIdPlus, ebx
210 jne .ReadGip
211 %endif
212 %ifdef ASYNC_GIP
213 mov esi, [edi + SUPGIPCPU.u32TransactionId]
214 %else
215 mov esi, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
216 %endif
217 cmp esi, u32TransactionId
218 jne .ReadGip
219 test esi, 1
220 jnz .ReadGip
221%endif ; NEED_TRANSACTION_ID
222%ifdef SAVED_u64RetNanoTS
223 mov ebx, u64RetNanoTS
224 mov ecx, u64RetNanoTS_Hi
225%endif
226
227 ;;
228 ;; Calc the timestamp.
229 ;;
230 ; u64RetNanoTS -= u64TSC;
231 sub ebx, u64TSC
232 sbb ecx, u64TSC_Hi
233
234 ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
235 or ecx, ecx
236 jnz .OverFlow
237 cmp ebx, u32UpdateIntervalTSC
238 ja .OverFlow
239 mov eax, ebx
240.ContinueCalcs: ; eax <= u32UpdateIntervalTSC
241 mul dword u32UpdateIntervalNS
242 div dword u32UpdateIntervalTSC
243 xor edx, edx
244
245 ; u64RetNanoTS += u64CurNanoTS;
246 add eax, u64CurNanoTS
247 adc edx, u64CurNanoTS_Hi
248
249 ;;
250 ;; Compare it with the previous one.
251 ;;
252 ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
253 ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
254 ;; @todo optimize this compare (/me too tired).
255 mov ecx, u64PrevNanoTS_Hi
256 mov ebx, u64PrevNanoTS
257 cmp edx, ecx
258 ja .Compare2
259 jb .DeltaPrevTooBig
260 cmp eax, ebx
261 jbe .DeltaPrevTooBig
262
263.Compare2:
264 add ebx, 0x6F736000
265 adc ecx, 0x00004E37
266 cmp edx, ecx
267 jb .CompareDone
268 ja .DeltaPrevTooBig
269 cmp eax, ebx
270 jae .DeltaPrevTooBig
271.CompareDone:
272
273
274 ;;
275 ;; Update the previous value with the u64RetNanoTS value.
276 ;;
277.Update:
278 ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
279 mov ebx, eax
280 mov ecx, edx
281 mov esi, pData
282 mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
283 mov eax, u64PrevNanoTS
284 mov edx, u64PrevNanoTS_Hi
285 lock cmpxchg8b [esi]
286 jnz .UpdateFailed
287
288.Updated:
289 mov eax, ebx
290 mov edx, ecx
291
292.Done:
293 mov esi, SavedESI
294 mov edi, SavedEDI
295 mov ebx, SavedEBX
296 leave
297 ret
298
299
300 ;;
301 ;; We've expired the interval, cap it. If we're here for the 2nd
302 ;; time without any GIP update in-between, the checks against
303 ;; pData->u64Prev below will force 1ns stepping.
304 ;;
305.OverFlow:
306 ; u64Delta = u32UpdateIntervalTSC;
307 mov esi, pData
308 inc dword [esi + RTTIMENANOTSDATA.cExpired]
309 mov eax, u32UpdateIntervalTSC
310 jmp .ContinueCalcs
311
312
313 ;;
314 ;; u64DeltaPrev >= 24h
315 ;;
316 ;; eax:edx = u64RetNanoTS (to be adjusted)
317 ;;
318.DeltaPrevTooBig:
319 ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
320 mov ebx, eax
321 sub ebx, u64PrevNanoTS
322 mov ecx, edx
323 sbb ecx, u64PrevNanoTS_Hi ; ebx:ecx = u64DeltaPrev
324
325 ; else if ( (int64_t)u64DeltaPrev <= 0
326 ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
327 ; {
328 ; /* Occasional - u64RetNanoTS is in the recent 'past' relative the previous call. */
329 ; pData->c1nsSteps++;
330 ; u64RetNanoTS = u64PrevNanoTS + 1;
331 ; }
332 mov esi, u32UpdateIntervalNS
333 cmp ecx, 0
334 jl .PrevNotZero2ndTest
335 jg .DeltaPrevNotInRecentPast
336 cmp ebx, 0
337 ja .DeltaPrevNotInRecentPast
338
339.PrevNotZero2ndTest:
340 add esi, esi ; ASSUMES: u32UpdateIntervalNS * 2 <= 32-bit.
341 xor edi, edi
342 add esi, ebx
343 adc edi, ecx
344 test edi, edi
345 js .DeltaPrevNotInRecentPast
346
347.DeltaPrevInRecentPast:
348 mov esi, pData
349 inc dword [esi + RTTIMENANOTSDATA.c1nsSteps]
350 mov eax, u64PrevNanoTS
351 mov edx, u64PrevNanoTS_Hi
352 add eax, 1
353 adc edx, 0
354 jmp .Update
355
356.DeltaPrevNotInRecentPast:
357 ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume). */
358 ; /* do nothing */;
359 cmp dword u64PrevNanoTS, 0
360 jne .DeltaPrevNotZero
361 cmp dword u64PrevNanoTS_Hi, 0
362 jne .DeltaPrevNotZero
363 jmp .Update
364
365.DeltaPrevNotZero:
366 ; else
367 ; {
368 ; /* Something has gone bust, if negative offset it's real bad. */
369 ; rtTimeNanoTSInternalBitch(pVM,
370 ; }
371
372 ; call C function that does the bitching.
373 mov u64RetNanoTS, eax
374 mov u64RetNanoTS_Hi, edx
375
376 mov edi, u64PrevNanoTS_Hi
377 mov esi, u64PrevNanoTS
378 push edi
379 push esi ; 4 - u64PrevNanoTS
380 push ecx
381 push ebx ; 3 - u64DeltaPrev
382 push edx
383 push eax ; 2 - u64RetNanoTS
384 mov eax, pData
385 push eax ; 1 - pData
386 call dword [eax + RTTIMENANOTSDATA.pfnBad]
387 add esp, 4*7
388
389 mov eax, u64RetNanoTS
390 mov edx, u64RetNanoTS_Hi
391 jmp .Update
392
393
394 ;;
395 ;; Attempt updating the previous value, provided we're still ahead of it.
396 ;;
397 ;; There is no point in recalculating u64NanoTS because we got preempted or if
398 ;; we raced somebody while the GIP was updated, since these are events
399 ;; that might occur at any point in the return path as well.
400 ;;
401 ;; eax:edx = *pData->u64Prev
402 ;; ebx:ecx = u64RetNanoTS
403 ;;
404 ALIGNCODE(16)
405.UpdateFailed:
406 mov edi, pData
407 lock inc dword [edi + RTTIMENANOTSDATA.cUpdateRaces]
408 ; for (i = 0; i < 10; i++)
409 mov edi, 10
410.UpdateLoop:
411 ; if (u64PrevNanoTS >= u64NanoTS)
412 ; break;
413 cmp edx, ecx
414 jg .Updated
415 jne .UpdateLoopLess
416 cmp eax, ebx
417 jae .Updated
418.UpdateLoopLess:
419 ; retry
420 lock cmpxchg8b [esi]
421 jz .Updated
422 dec edi
423 jnz .UpdateLoop
424 jmp .Updated
425
426
427 ;;
428 ;; The GIP is seemingly invalid, redo the discovery.
429 ;;
430.Rediscover:
431 mov eax, pData
432 push eax
433 call [eax + RTTIMENANOTSDATA.pfnRediscover]
434 add esp, 4h
435 jmp .Done
436
437
438%ifdef WITH_TSC_DELTA
439 ;;
440 ;; Unlikely branch for when we think the TSC delta might be invalid.
441 ;;
442.TscDeltaPossiblyInvalid:
443 cmp dword [edi + SUPGIPCPU.i64TSCDelta + 4], 0x7fffffff
444 jne .TscDeltaValid
445 jmp .TscDeltaNotValid
446%endif
447
448 ;
449 ; Cleanup variables
450 ;
451%undef pData
452%undef u64Delta_Hi
453%undef u64Delta
454%undef u32UpdateIntervalNS
455%undef u32UpdateIntervalTSC
456%undef u64TSC_Hi
457%undef u64TSC
458%undef u64NanoTS_Hi
459%undef u64NanoTS
460%undef u64PrevNanoTS_Hi
461%undef u64PrevNanoTS
462%undef u32TransactionId
463%undef u8ApicId
464
465%else ; AMD64
466
467;;
468; The AMD64 assembly implementation of the assembly routines.
469;
470; @returns Nanosecond timestamp.
471; @param pData gcc:rdi msc:rcx Pointer to the nanosecond timestamp data.
472;
473BEGINPROC rtTimeNanoTSInternalAsm
474 ;
475 ; Define variables and stack frame.
476 ;
477%define SavedRBX [rbp - 08h]
478%define SavedR12 [rbp - 10h]
479%define SavedR13 [rbp - 18h]
480%define SavedRDI [rbp - 20h]
481%define SavedRSI [rbp - 28h]
482%define TmpVar [rbp - 30h]
483%define TmpVar2 [rbp - 38h]
484%ifdef NEED_TRANSACTION_ID
485 %ifdef NEED_APIC_ID
486 %define SavedR14 [rbp - 40h]
487 %define SavedR15 [rbp - 48h]
488 %endif
489%endif
490
491%define pData rdi
492
493%ifdef ASYNC_GIP
494 %define u64TSC rsi
495 %define pGip rsi
496 %ifdef NEED_APIC_ID
497 %define pGipCPU r8
498 %endif
499%else
500 %define u64TSC r8
501 %define pGip rsi
502 %ifdef NEED_APIC_ID
503 %define pGipCPU r8
504 %endif
505%endif
506%define u32TransactionId r9d
507%define u64CurNanoTS r10
508%define u64PrevNanoTS r11 ; not parameter register
509%define u32UpdateIntervalTSC r12d
510%define u32UpdateIntervalTSC_64 r12
511%define u32UpdateIntervalNS r13d
512%define u32UpdateIntervalNS_64 r13
513%undef u64SavedRetNanoTS
514%undef u32ApicIdPlus
515%ifdef NEED_TRANSACTION_ID
516 %ifdef NEED_APIC_ID
517 %define u64SavedRetNanoTS r14
518 %define u32ApicIdPlus r15d
519 %endif
520%endif
521
522 ;
523 ; The prolog.
524 ;
525 push rbp
526 mov rbp, rsp
527%ifdef ASM_CALL64_MSC
528 sub rsp, 50h+20h
529%else
530 sub rsp, 50h
531%endif
532 mov SavedRBX, rbx
533 mov SavedR12, r12
534 mov SavedR13, r13
535%ifdef ASM_CALL64_MSC
536 mov SavedRDI, rdi
537 mov SavedRSI, rsi
538 mov pData, rcx
539%else
540 ;mov pData, rdi - already in rdi.
541%endif
542%ifdef SavedR14
543 mov SavedR14, r14
544%endif
545%ifdef SavedR15
546 mov SavedR15, r15
547%endif
548
549
550 ;;
551 ;; Data fetch loop.
552 ;; We take great pain ensuring that data consistency here.
553 ;;
554.ReadGip:
555
556 ;
557 ; Load pGip - finding the GIP is fun...
558 ;
559%ifdef RT_OS_WINDOWS
560 %ifdef IMPORTED_SUPLIB
561 %ifdef IN_RING0
562 mov rax, qword IMP(g_SUPGlobalInfoPage)
563 mov pGip, rax
564 %else
565 mov pGip, [IMP(g_pSUPGlobalInfoPage) wrt rip]
566 mov pGip, [pGip]
567 %endif
568 %else
569 mov pGip, [NAME(g_pSUPGlobalInfoPage) wrt rip]
570 %endif
571%else
572 %ifdef IN_RING0
573 mov rax, qword NAME(g_SUPGlobalInfoPage)
574 mov pGip, rax
575 %else
576 mov pGip, [rel NAME(g_pSUPGlobalInfoPage) wrt ..gotpcrel]
577 mov pGip, [pGip]
578 %endif
579%endif
580 or pGip, pGip
581 jz .Rediscover
582 cmp dword [pGip + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
583 jne .Rediscover
584
585 ;
586 ; pGipCPU, setting u32ApicIdPlus if necessary.
587 ;
588%ifdef NEED_APIC_ID
589 ; u8ApicId = ASMGetApicId();
590 mov eax, 1
591 cpuid ; expensive
592 %ifdef NEED_TRANSACTION_ID
593 mov u32ApicIdPlus, ebx
594 %endif
595 ; pGipCPU = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
596 shr ebx, 24
597 movzx eax, word [pGip + rbx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
598 imul eax, SUPGIPCPU_size
599 lea pGipCPU, [pGip + rax + SUPGLOBALINFOPAGE.aCPUs]
600%endif
601
602%ifdef NEED_TRANSACTION_ID
603 ;
604 ; Serialized loading of u32TransactionId.
605 ;
606 %ifdef ASYNC_GIP
607 mov u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
608 %else
609 mov u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
610 %endif
611 %ifdef USE_LFENCE
612 lfence
613 %else
614 lock xor dword TmpVar, 0
615 %endif
616%endif
617
618 ;
619 ; Load the data and TSC.
620 ;
621 mov u32UpdateIntervalNS, [pGip + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
622%ifdef ASYNC_GIP
623 mov u32UpdateIntervalTSC, [pGipCPU + SUPGIPCPU.u32UpdateIntervalTSC]
624%else
625 mov u32UpdateIntervalTSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
626%endif
627
628 rdtsc
629 mov u64PrevNanoTS, [pData + RTTIMENANOTSDATA.pu64Prev]
630 mov u64PrevNanoTS, [u64PrevNanoTS]
631 shl rdx, 32
632 or rax, rdx ; rax is u64RetNanoTS.
633%ifdef WITH_TSC_DELTA
634 mov rdx, [pGipCPU + SUPGIPCPU.i64TSCDelta]
635 mov u64CurNanoTS, 0x7fffffffffffffff ; INT64_MAX - temporarily borrowing u64CurNanoTS
636 cmp rdx, u64CurNanoTS
637 je .TscDeltaNotValid
638 sub rax, rdx
639.TscDeltaNotValid:
640%endif
641%ifdef u64SavedRetNanoTS ; doing this here may save a tick or so?
642 mov u64SavedRetNanoTS, rax
643%endif
644
645%ifdef ASYNC_GIP
646 mov u64CurNanoTS, [pGipCPU + SUPGIPCPU.u64NanoTS]
647 mov u64TSC, [pGipCPU + SUPGIPCPU.u64TSC] ; transhes pGIP!
648%else
649 mov u64CurNanoTS, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
650 mov u64TSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC] ; trashes pGipCPU!
651%endif
652
653
654%ifdef NEED_TRANSACTION_ID
655 ;
656 ; Check that the GIP and CPU didn't change.
657 ;
658 ; It is crucial that the rdtsc instruction has completed before
659 ; we check the transaction id. The LOCK prefixed instruction with
660 ; dependency on the RDTSC result should do the trick, I think.
661 ; CPUID is serializing, so the async path is safe by default.
662 ;
663 %ifdef NEED_APIC_ID
664 mov eax, 1
665 cpuid
666 cmp u32ApicIdPlus, ebx
667 jne .ReadGip
668 %else
669 lock xor qword TmpVar, rax
670 %endif
671 %ifdef ASYNC_GIP
672 cmp u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
673 %else
674 cmp u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
675 %endif
676 jne .ReadGip
677 test u32TransactionId, 1
678 jnz .ReadGip
679 %ifdef u64SavedRetNanoTS
680 mov rax, u64SavedRetNanoTS ; rax is u64RetNanoTS.
681 %endif
682%endif ; NEED_TRANSACTION_ID
683
684
685 ;;
686 ;; Calc the timestamp.
687 ;;
688 ; u64RetNanoTS -= u64TSC;
689 sub rax, u64TSC
690 xor edx, edx
691
692 ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
693 cmp rax, u32UpdateIntervalTSC_64
694 ja .OverFlow
695.ContinueCalcs: ; edx = 0; eax <= u32UpdateIntervalTSC
696 mul u32UpdateIntervalNS
697 div u32UpdateIntervalTSC
698
699 ; u64RetNanoTS += u64CurNanoTS;
700 add rax, u64CurNanoTS
701
702
703 ;;
704 ;; Compare it with the previous one.
705 ;;
706 ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
707 ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
708 ; /* Frequent - less than 24h since last call. */;
709 cmp rax, u64PrevNanoTS
710 jbe .DeltaPrevTooBig
711 mov ecx, 5
712 shl rcx, 44 ; close enough
713 add rcx, u64PrevNanoTS
714 cmp rax, rcx
715 jae .DeltaPrevTooBig
716
717
718 ;;
719 ;; Update the previous value.
720 ;;
721.Update:
722 ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
723 mov rbx, [pData + RTTIMENANOTSDATA.pu64Prev]
724 mov rcx, rax
725 mov rax, u64PrevNanoTS
726 lock cmpxchg [rbx], rcx
727 jnz .UpdateFailed
728
729.Updated:
730 mov rax, rcx
731
732.Done:
733 mov rbx, SavedRBX
734 mov r12, SavedR12
735 mov r13, SavedR13
736%ifdef SavedR14
737 mov r14, SavedR14
738%endif
739%ifdef SavedR15
740 mov r15, SavedR15
741%endif
742%ifdef ASM_CALL64_MSC
743 mov rdi, SavedRDI
744 mov rsi, SavedRSI
745%endif
746 leave
747 ret
748
749
750 ;;
751 ;; We've expired the interval, cap it. If we're here for the 2nd
752 ;; time without any GIP update in-between, the checks against
753 ;; pData->u64Prev below will force 1ns stepping.
754 ;;
755ALIGNCODE(16)
756.OverFlow:
757 ; u64RetNanoTS = u32UpdateIntervalTSC;
758 inc dword [pData + RTTIMENANOTSDATA.cExpired]
759 mov eax, u32UpdateIntervalTSC
760 jmp .ContinueCalcs
761
762
763 ;;
764 ;; u64DeltaPrev >= 24h
765 ;;
766 ;; rax = u64RetNanoTS (to be adjusted)
767 ;;
768ALIGNCODE(16)
769.DeltaPrevTooBig:
770 ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
771 mov rbx, rax
772 sub rbx, u64PrevNanoTS
773
774 ; else if ( (int64_t)u64DeltaPrev <= 0
775 ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
776 ; {
777 ; /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
778 ; pData->c1nsSteps++;
779 ; u64RetNanoTS = u64PrevNanoTS + 1;
780 ; }
781 test rbx, rbx
782 jg .DeltaPrevNotInRecentPast
783
784 lea rdx, [u32UpdateIntervalNS_64 + u32UpdateIntervalNS_64]
785 add rdx, rbx
786 js .DeltaPrevNotInRecentPast
787
788 ; body
789 inc dword [pData + RTTIMENANOTSDATA.c1nsSteps]
790 lea rax, [u64PrevNanoTS + 1]
791 jmp .Update
792
793 ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume) / first call. */
794 ; /* do nothing */;
795.DeltaPrevNotInRecentPast:
796 or u64PrevNanoTS, u64PrevNanoTS
797 jz .Update
798
799 ; else
800 ; {
801 ; /* Something has gone bust, if negative offset it's real bad. */
802 ; rtTimeNanoTSInternalBitch(pVM,
803 ; }
804
805 ; call C function that does the bitching.
806 mov TmpVar, rax
807 mov TmpVar2, pData
808
809%ifdef ASM_CALL64_MSC
810 mov rcx, pData ; param 1 - pData
811 mov rdx, rax ; param 2 - u64RetNanoTS
812 mov r8, rbx ; param 3 - u64DeltaPrev
813 mov r9, u64PrevNanoTS ; param 4 - u64PrevNanoTS
814%else
815 ;mov rdi, pData - already in rdi; param 1 - pData
816 mov rsi, rax ; param 2 - u64RetNanoTS
817 mov rdx, rbx ; param 3 - u64DeltaPrev
818 mov rcx, u64PrevNanoTS ; param 4 - u64PrevNanoTS
819%endif
820 call qword [pData + RTTIMENANOTSDATA.pfnBad]
821
822 mov rax, TmpVar
823 mov pData, TmpVar2
824 jmp .Update
825
826
827 ;;
828 ;; Attempt updating the previous value, provided we're still ahead of it.
829 ;;
830 ;; There is no point in recalculating u64NanoTS because we got preempted or if
831 ;; we raced somebody while the GIP was updated, since these are events
832 ;; that might occur at any point in the return path as well.
833 ;;
834 ;; rax = *pData->u64Prev;
835 ;; rcx = u64RetNanoTS
836 ;;
837ALIGNCODE(16)
838.UpdateFailed:
839 lock inc dword [pData + RTTIMENANOTSDATA.cUpdateRaces]
840 ; for (i = 0; i < 10; i++)
841 mov edx, 10
842.UpdateLoop:
843 ; if (u64PrevNanoTS >= u64RetNanoTS)
844 ; break;
845 cmp rax, rcx
846 jge .Updated
847.UpdateLoopLess:
848 ; retry
849 lock cmpxchg [rbx], rcx
850 jz .Updated
851 dec edx
852 jnz .UpdateLoop
853 jmp .Updated
854
855
856 ;;
857 ;; The GIP is seemingly invalid, redo the discovery.
858 ;;
859.Rediscover:
860%ifdef ASM_CALL64_MSC
861 mov rcx, pData
862%else
863 ; mov rdi, pData - already in rdi
864%endif
865 call [pData + RTTIMENANOTSDATA.pfnRediscover]
866 jmp .Done
867
868
869 ;
870 ; Cleanup variables
871 ;
872%undef SavedRBX
873%undef SavedR12
874%undef SavedR13
875%undef SavedR14
876%undef SavedR15
877%undef SavedRDI
878%undef SavedRSI
879%undef pData
880%undef TmpVar
881%undef u64TSC
882%undef pGip
883%undef pGipCPU
884%undef u32TransactionId
885%undef u64CurNanoTS
886%undef u64PrevNanoTS
887%undef u32UpdateIntervalTSC
888%undef u32UpdateIntervalTSC_64
889%undef u32UpdateIntervalNS
890%undef u64SavedRetNanoTS
891%undef u32ApicIdPlus
892
893%endif ; AMD64
894ENDPROC rtTimeNanoTSInternalAsm
895
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use