Index: /trunk/include/VBox/err.h =================================================================== --- /trunk/include/VBox/err.h (revision 87791) +++ /trunk/include/VBox/err.h (revision 87792) @@ -1031,4 +1031,28 @@ /** Invalid value for cVirtualTicking. */ #define VERR_TM_VIRTUAL_TICKING_IPE (-2211) +/** Max timer limit reached. */ +#define VERR_TM_TOO_MANY_TIMERS (-2212) +/** Invalid timer queue number. */ +#define VERR_TM_INVALID_TIMER_QUEUE (-2213) +/** The timer queue is not longer allowed to grow. */ +#define VERR_TM_TIMER_QUEUE_CANNOT_GROW (-2214) +/** TM internal processing error \#1. */ +#define VERR_TM_IPE_1 (-2291) +/** TM internal processing error \#2. */ +#define VERR_TM_IPE_2 (-2292) +/** TM internal processing error \#3. */ +#define VERR_TM_IPE_3 (-2293) +/** TM internal processing error \#4. */ +#define VERR_TM_IPE_4 (-2294) +/** TM internal processing error \#5. */ +#define VERR_TM_IPE_5 (-2295) +/** TM internal processing error \#6. */ +#define VERR_TM_IPE_6 (-2296) +/** TM internal processing error \#7. */ +#define VERR_TM_IPE_7 (-2297) +/** TM internal processing error \#8. */ +#define VERR_TM_IPE_8 (-2298) +/** TM internal processing error \#9. */ +#define VERR_TM_IPE_9 (-2299) /** @} */ Index: /trunk/include/VBox/vmm/gvm.h =================================================================== --- /trunk/include/VBox/vmm/gvm.h (revision 87791) +++ /trunk/include/VBox/vmm/gvm.h (revision 87792) @@ -255,9 +255,17 @@ } dbgfr0; + union + { +#if defined(VMM_INCLUDED_SRC_include_TMInternal_h) && defined(IN_RING0) + TMR0PERVM s; +#endif + uint8_t padding[128]; + } tmr0; + /** Padding so aCpus starts on a page boundrary. */ #ifdef VBOX_WITH_NEM_R0 - uint8_t abPadding2[4096*2 - 64 - 256 - 256 - 1024 - 256 - 64 - 2176 - 640 - 512 - 64 - 1024 - sizeof(PGVMCPU) * VMM_MAX_CPU_COUNT]; + uint8_t abPadding2[4096*2 - 64 - 256 - 256 - 1024 - 256 - 64 - 2176 - 640 - 512 - 64 - 1024 - 128 - sizeof(PGVMCPU) * VMM_MAX_CPU_COUNT]; #else - uint8_t abPadding2[4096*2 - 64 - 256 - 256 - 1024 - 64 - 2176 - 640 - 512 - 64 - 1024 - sizeof(PGVMCPU) * VMM_MAX_CPU_COUNT]; + uint8_t abPadding2[4096*2 - 64 - 256 - 256 - 1024 - 64 - 2176 - 640 - 512 - 64 - 1024 - 128 - sizeof(PGVMCPU) * VMM_MAX_CPU_COUNT]; #endif Index: /trunk/include/VBox/vmm/gvm.mac =================================================================== --- /trunk/include/VBox/vmm/gvm.mac (revision 87791) +++ /trunk/include/VBox/vmm/gvm.mac (revision 87792) @@ -82,4 +82,6 @@ alignb 64 .dbgfr0 resb 1024 + alignb 64 + .tmr0 resb 128 times ((($ + VMM_MAX_CPU_COUNT * RTR0PTR_CB + 4095) & ~4095) - ($ + VMM_MAX_CPU_COUNT * RTR0PTR_CB)) resb 1 Index: /trunk/include/VBox/vmm/tm.h =================================================================== --- /trunk/include/VBox/vmm/tm.h (revision 87791) +++ /trunk/include/VBox/vmm/tm.h (revision 87792) @@ -262,5 +262,4 @@ -#ifdef IN_RING3 /** @defgroup grp_tm_r3 The TM Host Context Ring-3 API * @{ @@ -296,5 +295,13 @@ VMMR3_INT_DECL(bool) TMR3CpuTickIsFixedRateMonotonic(PVM pVM, bool fWithParavirtEnabled); /** @} */ -#endif /* IN_RING3 */ + + +/** @defgroup grp_tm_r0 The TM Host Context Ring-0 API + * @{ + */ +VMMR0_INT_DECL(void) TMR0InitPerVMData(PGVM pGVM); +VMMR0_INT_DECL(void) TMR0CleanupVM(PGVM pGVM); +VMMR0_INT_DECL(int) TMR0TimerQueueGrow(PGVM pGVM, uint32_t idxQueue, uint32_t cMinTimers); +/** @} */ Index: /trunk/include/VBox/vmm/vm.h =================================================================== --- /trunk/include/VBox/vmm/vm.h (revision 87791) +++ /trunk/include/VBox/vmm/vm.h (revision 87792) @@ -1364,5 +1364,5 @@ struct TM s; #endif - uint8_t padding[7872]; /* multiple of 64 */ + uint8_t padding[9152]; /* multiple of 64 */ } tm; @@ -1463,5 +1463,5 @@ /** Padding for aligning the structure size on a page boundrary. */ - uint8_t abAlignment2[3672 - sizeof(PVMCPUR3) * VMM_MAX_CPU_COUNT]; + uint8_t abAlignment2[2392 - sizeof(PVMCPUR3) * VMM_MAX_CPU_COUNT]; /* ---- end small stuff ---- */ Index: /trunk/include/VBox/vmm/vm.mac =================================================================== --- /trunk/include/VBox/vmm/vm.mac (revision 87791) +++ /trunk/include/VBox/vmm/vm.mac (revision 87792) @@ -131,5 +131,5 @@ .em resb 256 .nem resb 128 - .tm resb 7872 + .tm resb 9152 .dbgf resb 2432 .ssm resb 128 Index: /trunk/include/VBox/vmm/vmm.h =================================================================== --- /trunk/include/VBox/vmm/vmm.h (revision 87791) +++ /trunk/include/VBox/vmm/vmm.h (revision 87792) @@ -424,9 +424,6 @@ VMMR0_DO_IOM_SYNC_STATS_INDICES, - /** Official call we use for testing Ring-0 APIs. */ - VMMR0_DO_TESTS = 704, - /** Call DBGFR0TraceCreateReqHandler. */ - VMMR0_DO_DBGF_TRACER_CREATE = 710, + VMMR0_DO_DBGF_TRACER_CREATE = 704, /** Call DBGFR0TraceCallReqHandler. */ VMMR0_DO_DBGF_TRACER_CALL_REQ_HANDLER, @@ -439,4 +436,10 @@ /** Call DBGFR0BpOwnerInitReqHandler(). */ VMMR0_DO_DBGF_BP_OWNER_INIT, + + /** Grow a timer queue. */ + VMMR0_DO_TM_GROW_TIMER_QUEUE = 768, + + /** Official call we use for testing Ring-0 APIs. */ + VMMR0_DO_TESTS = 2048, /** The usual 32-bit type blow up. */ Index: /trunk/src/VBox/VMM/Makefile.kmk =================================================================== --- /trunk/src/VBox/VMM/Makefile.kmk (revision 87791) +++ /trunk/src/VBox/VMM/Makefile.kmk (revision 87792) @@ -504,4 +504,5 @@ VMMR0/PGMR0Pool.cpp \ VMMR0/PGMR0SharedPage.cpp \ + VMMR0/TMR0.cpp \ VMMR0/VMMR0.cpp \ VMMRZ/CPUMRZ.cpp \ Index: /trunk/src/VBox/VMM/VMMAll/TMAll.cpp =================================================================== --- /trunk/src/VBox/VMM/VMMAll/TMAll.cpp (revision 87791) +++ /trunk/src/VBox/VMM/VMMAll/TMAll.cpp (revision 87792) @@ -360,4 +360,9 @@ DECLINLINE(void) tmSchedule(PVMCC pVM, PTMTIMER pTimer) { + TMCLOCK const enmClock = pTimer->enmClock; + AssertReturnVoid((unsigned)enmClock < TMCLOCK_MAX); + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[enmClock]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, enmClock, pQueue); + if ( VM_IS_EMT(pVM) && RT_SUCCESS(TM_TRY_LOCK_TIMERS(pVM))) @@ -365,5 +370,5 @@ STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a); Log3(("tmSchedule: tmTimerQueueSchedule\n")); - tmTimerQueueSchedule(pVM, &pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock]); + tmTimerQueueSchedule(pVM, pQueueCC, pQueue); #ifdef VBOX_STRICT tmTimerQueuesSanityChecks(pVM, "tmSchedule"); @@ -404,23 +409,25 @@ * Links the timer onto the scheduling queue. * - * @param pQueue The timer queue the timer belongs to. - * @param pTimer The timer. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. + * @param pTimer The timer. * * @todo FIXME: Look into potential race with the thread running the queues * and stuff. */ -DECLINLINE(void) tmTimerLinkSchedule(PTMTIMERQUEUE pQueue, PTMTIMER pTimer) -{ - Assert(!pTimer->offScheduleNext); - const int32_t offHeadNew = (intptr_t)pTimer - (intptr_t)pQueue; - int32_t offHead; +DECLINLINE(void) tmTimerLinkSchedule(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) +{ + Assert(pTimer->idxScheduleNext == UINT32_MAX); + const uint32_t idxHeadNew = pTimer - &pQueueCC->paTimers[0]; + AssertReturnVoid(idxHeadNew < pQueueCC->cTimersAlloc); + + uint32_t idxHead; do { - offHead = pQueue->offSchedule; - if (offHead) - pTimer->offScheduleNext = ((intptr_t)pQueue + offHead) - (intptr_t)pTimer; - else - pTimer->offScheduleNext = 0; - } while (!ASMAtomicCmpXchgS32(&pQueue->offSchedule, offHeadNew, offHead)); + idxHead = pQueue->idxSchedule; + Assert(idxHead == UINT32_MAX || idxHead < pQueueCC->cTimersAlloc); + pTimer->idxScheduleNext = idxHead; + } while (!ASMAtomicCmpXchgU32(&pQueue->idxSchedule, idxHeadNew, idxHead)); } @@ -438,7 +445,10 @@ DECLINLINE(bool) tmTimerTryWithLink(PVMCC pVM, PTMTIMER pTimer, TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld) { + TMCLOCK const enmClock = pTimer->enmClock; + AssertReturn((unsigned)enmClock < TMCLOCK_MAX, false); if (tmTimerTry(pTimer, enmStateNew, enmStateOld)) { - tmTimerLinkSchedule(&pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock], pTimer); + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[enmClock]; + tmTimerLinkSchedule(TM_GET_TIMER_QUEUE_CC(pVM, enmClock, pQueue), pQueue, pTimer); return true; } @@ -451,5 +461,7 @@ * * @param pVM The cross context VM structure. - * @param pQueue The queue. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. * @param pTimer The timer. * @param u64Expire The timer expiration time. @@ -457,36 +469,37 @@ * @remarks Called while owning the relevant queue lock. */ -DECL_FORCE_INLINE(void) tmTimerQueueLinkActive(PVMCC pVM, PTMTIMERQUEUE pQueue, PTMTIMER pTimer, uint64_t u64Expire) -{ - Assert(!pTimer->offNext); - Assert(!pTimer->offPrev); +DECL_FORCE_INLINE(void) tmTimerQueueLinkActive(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, + PTMTIMER pTimer, uint64_t u64Expire) +{ + Assert(pTimer->idxNext == UINT32_MAX); + Assert(pTimer->idxPrev == UINT32_MAX); Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE || pTimer->enmClock != TMCLOCK_VIRTUAL_SYNC); /* (active is not a stable state) */ RT_NOREF(pVM); - PTMTIMER pCur = TMTIMER_GET_HEAD(pQueue); + PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue); if (pCur) { - for (;; pCur = TMTIMER_GET_NEXT(pCur)) + for (;; pCur = tmTimerGetNext(pQueueCC, pCur)) { if (pCur->u64Expire > u64Expire) { - const PTMTIMER pPrev = TMTIMER_GET_PREV(pCur); - TMTIMER_SET_NEXT(pTimer, pCur); - TMTIMER_SET_PREV(pTimer, pPrev); + const PTMTIMER pPrev = tmTimerGetPrev(pQueueCC, pCur); + tmTimerSetNext(pQueueCC, pTimer, pCur); + tmTimerSetPrev(pQueueCC, pTimer, pPrev); if (pPrev) - TMTIMER_SET_NEXT(pPrev, pTimer); + tmTimerSetNext(pQueueCC, pPrev, pTimer); else { - TMTIMER_SET_HEAD(pQueue, pTimer); + tmTimerQueueSetHead(pQueueCC, pQueue, pTimer); ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire); DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive head", pTimer->szName); } - TMTIMER_SET_PREV(pCur, pTimer); + tmTimerSetPrev(pQueueCC, pCur, pTimer); return; } - if (!pCur->offNext) + if (pCur->idxNext == UINT32_MAX) { - TMTIMER_SET_NEXT(pCur, pTimer); - TMTIMER_SET_PREV(pTimer, pCur); + tmTimerSetNext(pQueueCC, pCur, pTimer); + tmTimerSetPrev(pQueueCC, pTimer, pCur); DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive tail", pTimer->szName); return; @@ -496,5 +509,5 @@ else { - TMTIMER_SET_HEAD(pQueue, pTimer); + tmTimerQueueSetHead(pQueueCC, pQueue, pTimer); ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire); DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive empty", pTimer->szName); @@ -508,10 +521,12 @@ * * @param pVM The cross context VM structure. - * @param pQueue The timer queue. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. * @param pTimer The timer that needs scheduling. * * @remarks Called while owning the lock. */ -DECLINLINE(void) tmTimerQueueScheduleOne(PVMCC pVM, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) +DECLINLINE(void) tmTimerQueueScheduleOne(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) { Assert(pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC); @@ -533,5 +548,5 @@ if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE, TMTIMERSTATE_PENDING_RESCHEDULE))) break; /* retry */ - tmTimerQueueUnlinkActive(pVM, pQueue, pTimer); + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); RT_FALL_THRU(); @@ -540,8 +555,8 @@ */ case TMTIMERSTATE_PENDING_SCHEDULE: - Assert(!pTimer->offNext); Assert(!pTimer->offPrev); + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX); if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, TMTIMERSTATE_PENDING_SCHEDULE))) break; /* retry */ - tmTimerQueueLinkActive(pVM, pQueue, pTimer, pTimer->u64Expire); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, pTimer->u64Expire); return; @@ -552,5 +567,5 @@ if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, TMTIMERSTATE_PENDING_STOP))) break; /* retry */ - tmTimerQueueUnlinkActive(pVM, pQueue, pTimer); + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); RT_FALL_THRU(); @@ -559,5 +574,5 @@ */ case TMTIMERSTATE_PENDING_STOP_SCHEDULE: - Assert(!pTimer->offNext); Assert(!pTimer->offPrev); + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX); if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_PENDING_STOP_SCHEDULE))) break; @@ -576,5 +591,5 @@ case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: - tmTimerLinkSchedule(pQueue, pTimer); + tmTimerLinkSchedule(pQueueCC, pQueue, pTimer); STAM_COUNTER_INC(&pVM->tm.s.CTX_SUFF_Z(StatPostponed)); return; @@ -601,9 +616,11 @@ * * @param pVM The cross context VM structure. - * @param pQueue The queue to schedule. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3) data of the queue to schedule. + * @param pQueue The shared queue data of the queue to schedule. * * @remarks Called while owning the lock. */ -void tmTimerQueueSchedule(PVMCC pVM, PTMTIMERQUEUE pQueue) +void tmTimerQueueSchedule(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue) { TM_ASSERT_TIMER_LOCK_OWNERSHIP(pVM); @@ -613,17 +630,16 @@ * Dequeue the scheduling list and iterate it. */ - int32_t offNext = ASMAtomicXchgS32(&pQueue->offSchedule, 0); - Log2(("tmTimerQueueSchedule: pQueue=%p:{.enmClock=%d, offNext=%RI32, .u64Expired=%'RU64}\n", pQueue, pQueue->enmClock, offNext, pQueue->u64Expire)); - if (!offNext) - return; - PTMTIMER pNext = (PTMTIMER)((intptr_t)pQueue + offNext); - while (pNext) - { + uint32_t idxNext = ASMAtomicXchgU32(&pQueue->idxSchedule, UINT32_MAX); + Log2(("tmTimerQueueSchedule: pQueue=%p:{.enmClock=%d, idxNext=%RI32, .u64Expired=%'RU64}\n", pQueue, pQueue->enmClock, idxNext, pQueue->u64Expire)); + while (idxNext != UINT32_MAX) + { + AssertBreak(idxNext < pQueueCC->cTimersAlloc); + /* - * Unlink the head timer and find the next one. + * Unlink the head timer and take down the index of the next one. */ - PTMTIMER pTimer = pNext; - pNext = pNext->offScheduleNext ? (PTMTIMER)((intptr_t)pNext + pNext->offScheduleNext) : NULL; - pTimer->offScheduleNext = 0; + PTMTIMER pTimer = &pQueueCC->paTimers[idxNext]; + idxNext = pTimer->idxScheduleNext; + pTimer->idxScheduleNext = UINT32_MAX; /* @@ -632,7 +648,7 @@ Log2(("tmTimerQueueSchedule: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, .szName=%s}\n", pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->szName)); - tmTimerQueueScheduleOne(pVM, pQueue, pTimer); + tmTimerQueueScheduleOne(pVM, pQueueCC, pQueue, pTimer); Log2(("tmTimerQueueSchedule: %p: new %s\n", pTimer, tmTimerState(pTimer->enmState))); - } /* foreach timer in current schedule batch. */ + } Log2(("tmTimerQueueSchedule: u64Expired=%'RU64\n", pQueue->u64Expire)); } @@ -648,109 +664,123 @@ * @remarks Called while owning the lock. */ -void tmTimerQueuesSanityChecks(PVM pVM, const char *pszWhere) +void tmTimerQueuesSanityChecks(PVMCC pVM, const char *pszWhere) { TM_ASSERT_TIMER_LOCK_OWNERSHIP(pVM); - /* - * Check the linking of the active lists. - */ - bool fHaveVirtualSyncLock = false; - for (int i = 0; i < TMCLOCK_MAX; i++) - { - PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[i]; - Assert((int)pQueue->enmClock == i); - if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC) + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue); + Assert(pQueue->enmClock == (TMCLOCK)idxQueue); + + int rc = PDMCritSectRwTryEnterShared(&pQueue->AllocLock); + if (RT_SUCCESS(rc)) { - if (PDMCritSectTryEnter(&pVM->tm.s.VirtualSyncLock) != VINF_SUCCESS) - continue; - fHaveVirtualSyncLock = true; - } - PTMTIMER pPrev = NULL; - for (PTMTIMER pCur = TMTIMER_GET_HEAD(pQueue); pCur; pPrev = pCur, pCur = TMTIMER_GET_NEXT(pCur)) - { - AssertMsg((int)pCur->enmClock == i, ("%s: %d != %d\n", pszWhere, pCur->enmClock, i)); - AssertMsg(TMTIMER_GET_PREV(pCur) == pPrev, ("%s: %p != %p\n", pszWhere, TMTIMER_GET_PREV(pCur), pPrev)); - TMTIMERSTATE enmState = pCur->enmState; - switch (enmState) + if ( pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC + || PDMCritSectTryEnter(&pVM->tm.s.VirtualSyncLock) == VINF_SUCCESS) { - case TMTIMERSTATE_ACTIVE: - AssertMsg( !pCur->offScheduleNext - || pCur->enmState != TMTIMERSTATE_ACTIVE, - ("%s: %RI32\n", pszWhere, pCur->offScheduleNext)); - break; - case TMTIMERSTATE_PENDING_STOP: - case TMTIMERSTATE_PENDING_RESCHEDULE: - case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: - break; - default: - AssertMsgFailed(("%s: Invalid state enmState=%d %s\n", pszWhere, enmState, tmTimerState(enmState))); - break; - } - } - } - - -# ifdef IN_RING3 - /* - * Do the big list and check that active timers all are in the active lists. - */ - PTMTIMERR3 pPrev = NULL; - for (PTMTIMERR3 pCur = pVM->tm.s.pCreated; pCur; pPrev = pCur, pCur = pCur->pBigNext) - { - Assert(pCur->pBigPrev == pPrev); - Assert((unsigned)pCur->enmClock < (unsigned)TMCLOCK_MAX); - - TMTIMERSTATE enmState = pCur->enmState; - switch (enmState) - { - case TMTIMERSTATE_ACTIVE: - case TMTIMERSTATE_PENDING_STOP: - case TMTIMERSTATE_PENDING_RESCHEDULE: - case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: - if (fHaveVirtualSyncLock || pCur->enmClock != TMCLOCK_VIRTUAL_SYNC) + /* Check the linking of the active lists. */ + PTMTIMER pPrev = NULL; + for (PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue); + pCur; + pPrev = pCur, pCur = tmTimerGetNext(pQueueCC, pCur)) { - PTMTIMERR3 pCurAct = TMTIMER_GET_HEAD(&pVM->tm.s.CTX_SUFF(paTimerQueues)[pCur->enmClock]); - Assert(pCur->offPrev || pCur == pCurAct); - while (pCurAct && pCurAct != pCur) - pCurAct = TMTIMER_GET_NEXT(pCurAct); - Assert(pCurAct == pCur); - } - break; - - case TMTIMERSTATE_PENDING_SCHEDULE: - case TMTIMERSTATE_PENDING_STOP_SCHEDULE: - case TMTIMERSTATE_STOPPED: - case TMTIMERSTATE_EXPIRED_DELIVER: - if (fHaveVirtualSyncLock || pCur->enmClock != TMCLOCK_VIRTUAL_SYNC) - { - Assert(!pCur->offNext); - Assert(!pCur->offPrev); - for (PTMTIMERR3 pCurAct = TMTIMER_GET_HEAD(&pVM->tm.s.CTX_SUFF(paTimerQueues)[pCur->enmClock]); - pCurAct; - pCurAct = TMTIMER_GET_NEXT(pCurAct)) + AssertMsg(pCur->enmClock == (TMCLOCK)idxQueue, ("%s: %d != %u\n", pszWhere, pCur->enmClock, idxQueue)); + AssertMsg(tmTimerGetPrev(pQueueCC, pCur) == pPrev, ("%s: %p != %p\n", pszWhere, tmTimerGetPrev(pQueueCC, pCur), pPrev)); + TMTIMERSTATE enmState = pCur->enmState; + switch (enmState) { - Assert(pCurAct != pCur); - Assert(TMTIMER_GET_NEXT(pCurAct) != pCur); - Assert(TMTIMER_GET_PREV(pCurAct) != pCur); + case TMTIMERSTATE_ACTIVE: + AssertMsg( pCur->idxScheduleNext == UINT32_MAX + || pCur->enmState != TMTIMERSTATE_ACTIVE, + ("%s: %RI32\n", pszWhere, pCur->idxScheduleNext)); + break; + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + break; + default: + AssertMsgFailed(("%s: Invalid state enmState=%d %s\n", pszWhere, enmState, tmTimerState(enmState))); + break; } } - break; - - /* ignore */ - case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: - break; - - /* shouldn't get here! */ - case TMTIMERSTATE_EXPIRED_GET_UNLINK: - case TMTIMERSTATE_DESTROY: - default: - AssertMsgFailed(("Invalid state enmState=%d %s\n", enmState, tmTimerState(enmState))); - break; + +# ifdef IN_RING3 + /* Go thru all the timers and check that the active ones all are in the active lists. */ + uint32_t idxTimer = pQueue->cTimersAlloc; + uint32_t cFree = 0; + while (idxTimer-- > 0) + { + PTMTIMER const pTimer = &pQueue->paTimers[idxTimer]; + TMTIMERSTATE const enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_FREE: + cFree++; + break; + + case TMTIMERSTATE_ACTIVE: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + { + PTMTIMERR3 pCurAct = tmTimerQueueGetHead(pQueueCC, pQueue); + Assert(pTimer->idxPrev != UINT32_MAX || pTimer == pCurAct); + while (pCurAct && pCurAct != pTimer) + pCurAct = tmTimerGetNext(pQueueCC, pCurAct); + Assert(pCurAct == pTimer); + break; + } + + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_EXPIRED_DELIVER: + { + Assert(pTimer->idxNext == UINT32_MAX); + Assert(pTimer->idxPrev == UINT32_MAX); + for (PTMTIMERR3 pCurAct = tmTimerQueueGetHead(pQueueCC, pQueue); + pCurAct; + pCurAct = tmTimerGetNext(pQueueCC, pCurAct)) + { + Assert(pCurAct != pTimer); + Assert(tmTimerGetNext(pQueueCC, pCurAct) != pTimer); + Assert(tmTimerGetPrev(pQueueCC, pCurAct) != pTimer); + } + break; + } + + /* ignore */ + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + break; + + case TMTIMERSTATE_INVALID: + Assert(idxTimer == 0); + break; + + /* shouldn't get here! */ + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_DESTROY: + default: + AssertMsgFailed(("Invalid state enmState=%d %s\n", enmState, tmTimerState(enmState))); + break; + } + + /* Check the handle value. */ + if (enmState > TMTIMERSTATE_INVALID && enmState < TMTIMERSTATE_DESTROY) + { + Assert((pTimer->hSelf & TMTIMERHANDLE_TIMER_IDX_MASK) == idxTimer); + Assert(((pTimer->hSelf >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) & TMTIMERHANDLE_QUEUE_IDX_SMASK) == idxQueue); + } + } + Assert(cFree == pQueue->cTimersFree); +# endif /* IN_RING3 */ + + if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC) + PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock); + } + PDMCritSectRwLeaveShared(&pQueue->AllocLock); } } -# endif /* IN_RING3 */ - - if (fHaveVirtualSyncLock) - PDMCritSectLeave(&pVM->tm.s.VirtualSyncLock); } #endif /* !VBOX_STRICT */ @@ -883,5 +913,5 @@ * Check for TMCLOCK_VIRTUAL expiration. */ - const uint64_t u64Expire1 = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire); + const uint64_t u64Expire1 = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire); const int64_t i64Delta1 = u64Expire1 - u64Now; if (i64Delta1 <= 0) @@ -906,5 +936,5 @@ */ uint64_t u64VirtualSyncNow; - uint64_t u64Expire2 = ASMAtomicUoReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + uint64_t u64Expire2 = ASMAtomicUoReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)) { @@ -915,5 +945,5 @@ && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp) && u64VirtualSyncNow == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) - && u64Expire2 == ASMAtomicUoReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire))) + && u64Expire2 == ASMAtomicUoReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire))) { u64VirtualSyncNow = u64Now - u64VirtualSyncNow; @@ -960,5 +990,5 @@ fCatchUp = ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp); off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync); - u64Expire2 = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + u64Expire2 = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); if (fCatchUp) { @@ -971,5 +1001,5 @@ && u32Pct == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage) && off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) - && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire) + && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire) && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp) && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)) @@ -996,5 +1026,5 @@ } else if ( off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) - && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire) + && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire) && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp) && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)) @@ -1171,9 +1201,10 @@ static int tmTimerSetOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t u64Expire) { - Assert(!pTimer->offPrev); - Assert(!pTimer->offNext); + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE); TMCLOCK const enmClock = pTimer->enmClock; + AssertReturn((unsigned)enmClock < TMCLOCK_MAX, VERR_TM_IPE_2); /* @@ -1193,5 +1224,6 @@ * Link the timer into the active list. */ - tmTimerQueueLinkActive(pVM, &pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock], pTimer, u64Expire); + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[enmClock]; + tmTimerQueueLinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, enmClock, pQueue), pQueue, pTimer, u64Expire); STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetOpt); @@ -1220,6 +1252,7 @@ AssertRCReturn(rc, rc); - PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC]; - TMTIMERSTATE enmState = pTimer->enmState; + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue); + TMTIMERSTATE const enmState = pTimer->enmState; switch (enmState) { @@ -1235,5 +1268,5 @@ pTimer->u64Expire = u64Expire; TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE); - tmTimerQueueLinkActive(pVM, pQueue, pTimer, u64Expire); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); rc = VINF_SUCCESS; break; @@ -1241,7 +1274,7 @@ case TMTIMERSTATE_ACTIVE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStActive); - tmTimerQueueUnlinkActive(pVM, pQueue, pTimer); + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); pTimer->u64Expire = u64Expire; - tmTimerQueueLinkActive(pVM, pQueue, pTimer, u64Expire); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); rc = VINF_SUCCESS; break; @@ -1356,6 +1389,6 @@ if (tmTimerTryWithLink(pVM, pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState)) { - Assert(!pTimer->offPrev); - Assert(!pTimer->offNext); + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); pTimer->u64Expire = u64Expire; TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE); @@ -1478,6 +1511,6 @@ static int tmTimerSetRelativeOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now) { - Assert(!pTimer->offPrev); - Assert(!pTimer->offNext); + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE); @@ -1486,4 +1519,5 @@ */ TMCLOCK const enmClock = pTimer->enmClock; + AssertReturn((unsigned)enmClock < (unsigned)TMCLOCK_MAX, VERR_TM_IPE_2); uint64_t const u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now); pTimer->u64Expire = u64Expire; @@ -1494,5 +1528,6 @@ */ DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerSetRelativeOptimizedStart", pTimer->szName); - tmTimerQueueLinkActive(pVM, &pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock], pTimer, u64Expire); + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[enmClock]; + tmTimerQueueLinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, enmClock, pQueue), pQueue, pTimer, u64Expire); STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeOpt); @@ -1530,6 +1565,7 @@ /* Update the timer. */ - PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC]; - TMTIMERSTATE enmState = pTimer->enmState; + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue); + TMTIMERSTATE const enmState = pTimer->enmState; switch (enmState) { @@ -1542,5 +1578,5 @@ pTimer->u64Expire = u64Expire; TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE); - tmTimerQueueLinkActive(pVM, pQueue, pTimer, u64Expire); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); rc = VINF_SUCCESS; break; @@ -1548,7 +1584,7 @@ case TMTIMERSTATE_ACTIVE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStActive); - tmTimerQueueUnlinkActive(pVM, pQueue, pTimer); + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); pTimer->u64Expire = u64Expire; - tmTimerQueueLinkActive(pVM, pQueue, pTimer, u64Expire); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); rc = VINF_SUCCESS; break; @@ -1679,6 +1715,6 @@ if (tmTimerTryWithLink(pVM, pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState)) { - Assert(!pTimer->offPrev); - Assert(!pTimer->offNext); + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, enmClock, pu64Now); Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [EXP/STOP]\n", @@ -1879,13 +1915,15 @@ /* Update the timer state. */ - PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC]; - TMTIMERSTATE enmState = pTimer->enmState; + TMTIMERSTATE const enmState = pTimer->enmState; switch (enmState) { case TMTIMERSTATE_ACTIVE: - tmTimerQueueUnlinkActive(pVM, pQueue, pTimer); + { + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; + tmTimerQueueUnlinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue), pQueue, pTimer); TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED); rc = VINF_SUCCESS; break; + } case TMTIMERSTATE_EXPIRED_DELIVER: @@ -2554,4 +2592,5 @@ AssertCompile(TMTIMERSTATE_##state == (num)); \ return #num "-" #state + CASE( 0,INVALID); CASE( 1,STOPPED); CASE( 2,ACTIVE); @@ -2580,5 +2619,5 @@ * @param pVM The cross context VM structure. */ -static uint32_t tmGetFrequencyHint(PVM pVM) +static uint32_t tmGetFrequencyHint(PVMCC pVM) { /* @@ -2599,8 +2638,9 @@ */ uMaxHzHint = 0; - for (int i = 0; i < TMCLOCK_MAX; i++) + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) { - PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[i]; - for (PTMTIMER pCur = TMTIMER_GET_HEAD(pQueue); pCur; pCur = TMTIMER_GET_NEXT(pCur)) + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PTMTIMERQUEUECC pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue); + for (PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue); pCur; pCur = tmTimerGetNext(pQueueCC, pCur)) { uint32_t uHzHint = ASMAtomicUoReadU32(&pCur->uHzHint); @@ -2624,4 +2664,5 @@ case TMTIMERSTATE_DESTROY: case TMTIMERSTATE_FREE: + case TMTIMERSTATE_INVALID: break; /* no default, want gcc warnings when adding more states. */ Index: /trunk/src/VBox/VMM/VMMAll/TMAllVirtual.cpp =================================================================== --- /trunk/src/VBox/VMM/VMMAll/TMAllVirtual.cpp (revision 87791) +++ /trunk/src/VBox/VMM/VMMAll/TMAllVirtual.cpp (revision 87792) @@ -279,7 +279,7 @@ if ( !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER) && !pVM->tm.s.fRunningQueues - && ( pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64 + && ( pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64 || ( pVM->tm.s.fVirtualSyncTicking - && pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64 - pVM->tm.s.offVirtualSync + && pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64 - pVM->tm.s.offVirtualSync ) ) @@ -416,5 +416,5 @@ } - uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); if (pnsAbsDeadline) *pnsAbsDeadline = u64Expire; /* Always return the unadjusted absolute deadline, or HM will waste time going @@ -520,5 +520,5 @@ } - uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); if (pnsAbsDeadline) *pnsAbsDeadline = u64Expire; @@ -597,5 +597,5 @@ PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu); if ( !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER) - && pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64) + && pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64) { Log5(("TMAllVirtual(%u): FF: 0 -> 1\n", __LINE__)); @@ -635,5 +635,5 @@ { off = u64 - off; - uint64_t const u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + uint64_t const u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); if (off < u64Expire) { @@ -767,5 +767,5 @@ u64 -= off; /** @todo u64VirtualSyncLast */ - uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); if (pnsAbsDeadline) *pnsAbsDeadline = u64Expire; @@ -928,5 +928,5 @@ { /** @todo Try use ASMAtomicUoReadU64 instead. */ - uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire); + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); return u64Expire == uDeadlineVersion; } Index: /trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp =================================================================== --- /trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp (revision 87791) +++ /trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp (revision 87792) @@ -910,4 +910,5 @@ PDMR0InitPerVMData(pGVM); IOMR0InitPerVMData(pGVM); + TMR0InitPerVMData(pGVM); if (RT_SUCCESS(rc) && RT_SUCCESS(rc2)) { @@ -1306,4 +1307,5 @@ DBGFR0CleanupVM(pGVM); PGMR0CleanupVM(pGVM); + TMR0CleanupVM(pGVM); AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */ Index: /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp =================================================================== --- /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp (revision 87791) +++ /trunk/src/VBox/VMM/VMMR0/VMMR0.cpp (revision 87792) @@ -2310,4 +2310,16 @@ /* + * TM requests. + */ + case VMMR0_DO_TM_GROW_TIMER_QUEUE: + { + if (pReqHdr || idCpu == NIL_VMCPUID) + return VERR_INVALID_PARAMETER; + rc = TMR0TimerQueueGrow(pGVM, RT_HI_U32(u64Arg), RT_LO_U32(u64Arg)); + VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING); + break; + } + + /* * For profiling. */ @@ -2390,5 +2402,5 @@ if ( pVM != NULL && pGVM != NULL - && pVM == pGVM /** @todo drop pGVM */ + && pVM == pGVM /** @todo drop pVM or pGVM */ && idCpu < pGVM->cCpus && pGVM->pSession == pSession Index: /trunk/src/VBox/VMM/VMMR3/TM.cpp =================================================================== --- /trunk/src/VBox/VMM/VMMR3/TM.cpp (revision 87791) +++ /trunk/src/VBox/VMM/VMMR3/TM.cpp (revision 87792) @@ -152,4 +152,5 @@ #include #include +#include #include #include @@ -175,4 +176,7 @@ static DECLCALLBACK(int) tmR3Save(PVM pVM, PSSMHANDLE pSSM); static DECLCALLBACK(int) tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass); +#ifdef VBOX_WITH_STATISTICS +static void tmR3TimerQueueRegisterStats(PVM pVM, PTMTIMERQUEUE pQueue, uint32_t cTimers); +#endif static DECLCALLBACK(void) tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick); static void tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue); @@ -187,6 +191,6 @@ static DECLCALLBACK(void) tmR3InfoCpuLoad(PVM pVM, PCDBGFINFOHLP pHlp, int cArgs, char **papszArgs); static DECLCALLBACK(VBOXSTRICTRC) tmR3CpuTickParavirtDisable(PVM pVM, PVMCPU pVCpu, void *pvData); -static const char * tmR3GetTSCModeName(PVM pVM); -static const char * tmR3GetTSCModeNameEx(TMTSCMODE enmMode); +static const char *tmR3GetTSCModeName(PVM pVM); +static const char *tmR3GetTSCModeNameEx(TMTSCMODE enmMode); @@ -212,21 +216,23 @@ * Init the structure. */ - void *pv; - int rc = MMHyperAlloc(pVM, sizeof(pVM->tm.s.paTimerQueuesR3[0]) * TMCLOCK_MAX, 0, MM_TAG_TM, &pv); - AssertRCReturn(rc, rc); - pVM->tm.s.paTimerQueuesR3 = (PTMTIMERQUEUE)pv; - pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pv); - pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pv); - - pVM->tm.s.offVM = RT_UOFFSETOF(VM, tm.s); pVM->tm.s.idTimerCpu = pVM->cCpus - 1; /* The last CPU. */ - pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].enmClock = TMCLOCK_VIRTUAL; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].u64Expire = INT64_MAX; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].enmClock = TMCLOCK_VIRTUAL_SYNC; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].u64Expire = INT64_MAX; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].enmClock = TMCLOCK_REAL; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].u64Expire = INT64_MAX; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].enmClock = TMCLOCK_TSC; - pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].u64Expire = INT64_MAX; + + strcpy(pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].szName, "virtual"); + strcpy(pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].szName, "virtual_sync"); /* Underscore is for STAM ordering issue. */ + strcpy(pVM->tm.s.aTimerQueues[TMCLOCK_REAL].szName, "real"); + strcpy(pVM->tm.s.aTimerQueues[TMCLOCK_TSC].szName, "tsc"); + + for (uint32_t i = 0; i < RT_ELEMENTS(pVM->tm.s.aTimerQueues); i++) + { + Assert(pVM->tm.s.aTimerQueues[i].szName[0] != '\0'); + pVM->tm.s.aTimerQueues[i].enmClock = (TMCLOCK)i; + pVM->tm.s.aTimerQueues[i].u64Expire = INT64_MAX; + pVM->tm.s.aTimerQueues[i].idxActive = UINT32_MAX; + pVM->tm.s.aTimerQueues[i].idxSchedule = UINT32_MAX; + pVM->tm.s.aTimerQueues[i].idxFreeHint = 1; + int rc = PDMR3CritSectRwInit(pVM, &pVM->tm.s.aTimerQueues[i].AllocLock, RT_SRC_POS, + "TM queue %s", pVM->tm.s.aTimerQueues[i].szName); + AssertLogRelRCReturn(rc, rc); + } /* @@ -243,5 +249,5 @@ RTHCPHYS HCPhysGIP; - rc = SUPR3GipGetPhys(&HCPhysGIP); + int rc = SUPR3GipGetPhys(&HCPhysGIP); AssertMsgRCReturn(rc, ("Failed to get GIP physical address!\n"), rc); @@ -1111,4 +1117,35 @@ pVM->tm.s.fTSCModeSwitchAllowed &= tmR3HasFixedTSC(pVM) && GIMIsEnabled(pVM) && !VM_IS_RAW_MODE_ENABLED(pVM); LogRel(("TM: TMR3InitFinalize: fTSCModeSwitchAllowed=%RTbool\n", pVM->tm.s.fTSCModeSwitchAllowed)); + + /* + * Grow the virtual & real timer tables so we've got sufficient + * space for dynamically created timers. We cannot allocate more + * after ring-0 init completes. + */ + static struct { uint32_t idxQueue, cExtra; } s_aExtra[] = { {TMCLOCK_VIRTUAL, 128}, {TMCLOCK_REAL, 32} }; + for (uint32_t i = 0; i < RT_ELEMENTS(s_aExtra); i++) + { + uint32_t const cExtra = s_aExtra[i].cExtra; + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[s_aExtra[i].idxQueue]; + if (s_aExtra[i].cExtra > pQueue->cTimersFree) + { + uint32_t cTimersAlloc = pQueue->cTimersAlloc + s_aExtra[i].cExtra - pQueue->cTimersFree; + rc = VMMR3CallR0Emt(pVM, VMMGetCpu(pVM), VMMR0_DO_TM_GROW_TIMER_QUEUE, + RT_MAKE_U64(cTimersAlloc, s_aExtra[i].idxQueue), NULL); + AssertLogRelMsgReturn(RT_SUCCESS(rc), ("rc=%Rrc cTimersAlloc=%u %s\n", rc, cTimersAlloc, pQueue->szName), rc); + } + } + +#ifdef VBOX_WITH_STATISTICS + /* + * Register timer statistics now that we've fixed the timer table sizes. + */ + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + pVM->tm.s.aTimerQueues[idxQueue].fCannotGrow = true; + tmR3TimerQueueRegisterStats(pVM, &pVM->tm.s.aTimerQueues[idxQueue], UINT32_MAX); + } +#endif + return rc; } @@ -1127,10 +1164,7 @@ LogFlow(("TMR3Relocate\n")); - pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pVM->tm.s.paTimerQueuesR3); - if (VM_IS_RAW_MODE_ENABLED(pVM)) { pVM->tm.s.pvGIPRC = MMHyperR3ToRC(pVM, pVM->tm.s.pvGIPR3); - pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pVM->tm.s.paTimerQueuesR3); pVM->tm.s.VirtualGetRawDataRC.pu64Prev += offDelta; pVM->tm.s.VirtualGetRawDataRC.pfnBad += offDelta; @@ -1153,5 +1187,4 @@ VMM_INT_DECL(int) TMR3Term(PVM pVM) { - AssertMsg(pVM->tm.s.offVM, ("bad init order!\n")); if (pVM->tm.s.pTimer) { @@ -1206,5 +1239,5 @@ */ for (int i = 0; i < TMCLOCK_MAX; i++) - tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[i]); + tmTimerQueueSchedule(pVM, &pVM->tm.s.aTimerQueues[i], &pVM->tm.s.aTimerQueues[i]); #ifdef VBOX_STRICT tmTimerQueuesSanityChecks(pVM, "TMR3Reset"); @@ -1493,4 +1526,64 @@ +#ifdef VBOX_WITH_STATISTICS + +/** + * Register statistics for a timer. + * + * @param pVM The cross context VM structure. + * @param pTimer The timer to register statistics for. + */ +static void tmR3TimerRegisterStats(PVM pVM, PTMTIMER pTimer) +{ + STAMR3RegisterF(pVM, &pTimer->StatTimer, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, + tmR3TimerClockName(pTimer), "/TM/Timers/%s", pTimer->szName); + STAMR3RegisterF(pVM, &pTimer->StatCritSectEnter, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, + "", "/TM/Timers/%s/CritSectEnter", pTimer->szName); + STAMR3RegisterF(pVM, &pTimer->StatGet, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, + "", "/TM/Timers/%s/Get", pTimer->szName); + STAMR3RegisterF(pVM, &pTimer->StatSetAbsolute, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, + "", "/TM/Timers/%s/SetAbsolute", pTimer->szName); + STAMR3RegisterF(pVM, &pTimer->StatSetRelative, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, + "", "/TM/Timers/%s/SetRelative", pTimer->szName); + STAMR3RegisterF(pVM, &pTimer->StatStop, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, + "", "/TM/Timers/%s/Stop", pTimer->szName); +} + + +/** + * Deregister the statistics for a timer. + */ +static void tmR3TimerDeregisterStats(PVM pVM, PTMTIMER pTimer) +{ + char szPrefix[128]; + size_t cchPrefix = RTStrPrintf(szPrefix, sizeof(szPrefix), "/TM/Timers/%s/", pTimer->szName); + STAMR3DeregisterByPrefix(pVM->pUVM, szPrefix); + szPrefix[cchPrefix - 1] = '\0'; + STAMR3Deregister(pVM->pUVM, szPrefix); +} + + +/** + * Register statistics for all allocated timers in a queue. + * + * @param pVM The cross context VM structure. + * @param pTimer The timer to register statistics for. + * @param cTimers Number of timers to consider (in growth scenario). + */ +static void tmR3TimerQueueRegisterStats(PVM pVM, PTMTIMERQUEUE pQueue, uint32_t cTimers) +{ + uint32_t idxTimer = RT_MIN(cTimers, pQueue->cTimersAlloc); + while (idxTimer-- > 0) + { + PTMTIMER pTimer = &pQueue->paTimers[idxTimer]; + TMTIMERSTATE enmState = pTimer->enmState; + if (enmState > TMTIMERSTATE_INVALID && enmState < TMTIMERSTATE_DESTROY) + tmR3TimerRegisterStats(pVM, pTimer); + } +} + +#endif /* VBOX_WITH_STATISTICS */ + + /** * Internal TMR3TimerCreate worker. @@ -1505,5 +1598,5 @@ static int tmr3TimerCreate(PVM pVM, TMCLOCK enmClock, uint32_t fFlags, const char *pszName, PPTMTIMERR3 ppTimer) { - PTMTIMERR3 pTimer; + PTMTIMER pTimer; /* @@ -1511,6 +1604,8 @@ */ VM_ASSERT_EMT(pVM); + AssertReturn((fFlags & (TMTIMER_FLAGS_RING0 | TMTIMER_FLAGS_NO_RING0)) != (TMTIMER_FLAGS_RING0 | TMTIMER_FLAGS_NO_RING0), VERR_INVALID_FLAGS); + AssertPtrReturn(pszName, VERR_INVALID_POINTER); size_t const cchName = strlen(pszName); @@ -1518,34 +1613,75 @@ AssertMsgReturn(cchName > 2, ("Too short timer name: %s\n", pszName), VERR_INVALID_NAME); + AssertMsgReturn(enmClock >= TMCLOCK_REAL && enmClock < TMCLOCK_MAX, + ("%d\n", enmClock), VERR_INVALID_PARAMETER); + AssertReturn(enmClock != TMCLOCK_TSC, VERR_NOT_SUPPORTED); + if (enmClock == TMCLOCK_VIRTUAL_SYNC) + VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_WRONG_ORDER); + + /* + * Exclusively lock the queue. + * + * Note! This means that it is not possible to allocate timers from a timer callback. + */ + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[enmClock]; + int rc = PDMCritSectRwEnterExcl(&pQueue->AllocLock, VERR_IGNORED); + AssertRCReturn(rc, rc); + /* * Allocate the timer. */ - if (pVM->tm.s.pFree && VM_IS_EMT(pVM)) - { - pTimer = pVM->tm.s.pFree; - pVM->tm.s.pFree = pTimer->pBigNext; - Log3(("TM: Recycling timer %p, new free head %p.\n", pTimer, pTimer->pBigNext)); - } - else - pTimer = NULL; - - if (!pTimer) - { - int rc = MMHyperAlloc(pVM, sizeof(*pTimer), 0, MM_TAG_TM, (void **)&pTimer); - if (RT_FAILURE(rc)) - return rc; - Log3(("TM: Allocated new timer %p\n", pTimer)); - } + if (!pQueue->cTimersFree) + { + AssertReturn(!pQueue->fCannotGrow, VERR_TM_TIMER_QUEUE_CANNOT_GROW); + uint32_t cTimersAlloc = pQueue->cTimersAlloc + 64; + Assert(cTimersAlloc < _32K); + rc = VMMR3CallR0Emt(pVM, VMMGetCpu(pVM), VMMR0_DO_TM_GROW_TIMER_QUEUE, + RT_MAKE_U64(cTimersAlloc, (uint64_t)(pQueue - &pVM->tm.s.aTimerQueues[0])), NULL); + AssertLogRelRCReturnStmt(rc, PDMCritSectRwLeaveExcl(&pQueue->AllocLock), rc); + AssertReturnStmt(pQueue->cTimersAlloc >= cTimersAlloc, PDMCritSectRwLeaveExcl(&pQueue->AllocLock), VERR_TM_IPE_3); + } + + /* Scan the array for free timers. */ + pTimer = NULL; + PTMTIMER const paTimers = pQueue->paTimers; + uint32_t const cTimersAlloc = pQueue->cTimersAlloc; + uint32_t idxTimer = pQueue->idxFreeHint; + for (uint32_t iScan = 0; iScan < 2; iScan++) + { + while (idxTimer < cTimersAlloc) + { + if (paTimers[idxTimer].enmState == TMTIMERSTATE_FREE) + { + pTimer = &paTimers[idxTimer]; + pQueue->idxFreeHint = idxTimer + 1; + break; + } + idxTimer++; + } + if (pTimer != NULL) + break; + idxTimer = 1; + } + AssertLogRelMsgReturnStmt(pTimer != NULL, ("cTimersFree=%u cTimersAlloc=%u enmClock=%s\n", pQueue->cTimersFree, + pQueue->cTimersAlloc, pQueue->szName), + PDMCritSectRwLeaveExcl(&pQueue->AllocLock), VERR_INTERNAL_ERROR_3); + pQueue->cTimersFree -= 1; /* * Initialize it. */ + Assert(idxTimer != 0); + Assert(idxTimer <= TMTIMERHANDLE_TIMER_IDX_MASK); + pTimer->hSelf = idxTimer + | ((uintptr_t)(pQueue - &pVM->tm.s.aTimerQueues[0]) << TMTIMERHANDLE_QUEUE_IDX_SHIFT); + Assert(!(pTimer->hSelf & TMTIMERHANDLE_RANDOM_MASK)); + pTimer->hSelf |= (RTRandU64() & TMTIMERHANDLE_RANDOM_MASK); + pTimer->u64Expire = 0; pTimer->enmClock = enmClock; - pTimer->hSelf = (TMTIMERHANDLE)pTimer; pTimer->enmState = TMTIMERSTATE_STOPPED; - pTimer->offScheduleNext = 0; - pTimer->offNext = 0; - pTimer->offPrev = 0; + pTimer->idxScheduleNext = UINT32_MAX; + pTimer->idxNext = UINT32_MAX; + pTimer->idxPrev = UINT32_MAX; pTimer->fFlags = fFlags; pTimer->uHzHint = 0; @@ -1555,32 +1691,18 @@ pTimer->szName[cchName] = '\0'; - /* insert into the list of created timers. */ +#ifdef VBOX_STRICT TM_LOCK_TIMERS(pVM); - pTimer->pBigPrev = NULL; - pTimer->pBigNext = pVM->tm.s.pCreated; - pVM->tm.s.pCreated = pTimer; - if (pTimer->pBigNext) - pTimer->pBigNext->pBigPrev = pTimer; -#ifdef VBOX_STRICT tmTimerQueuesSanityChecks(pVM, "tmR3TimerCreate"); + TM_UNLOCK_TIMERS(pVM); #endif - TM_UNLOCK_TIMERS(pVM); + + PDMCritSectRwLeaveExcl(&pQueue->AllocLock); #ifdef VBOX_WITH_STATISTICS /* - * Register statistics. - */ - STAMR3RegisterF(pVM, &pTimer->StatTimer, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, - tmR3TimerClockName(pTimer), "/TM/Timers/%s", pTimer->szName); - STAMR3RegisterF(pVM, &pTimer->StatCritSectEnter, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, - "", "/TM/Timers/%s/CritSectEnter", pTimer->szName); - STAMR3RegisterF(pVM, &pTimer->StatGet, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, - "", "/TM/Timers/%s/Get", pTimer->szName); - STAMR3RegisterF(pVM, &pTimer->StatSetAbsolute, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, - "", "/TM/Timers/%s/SetAbsolute", pTimer->szName); - STAMR3RegisterF(pVM, &pTimer->StatSetRelative, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, - "", "/TM/Timers/%s/SetRelative", pTimer->szName); - STAMR3RegisterF(pVM, &pTimer->StatStop, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, - "", "/TM/Timers/%s/Stop", pTimer->szName); + * Only register statistics if we're passed the no-realloc point. + */ + if (pQueue->fCannotGrow) + tmR3TimerRegisterStats(pVM, pTimer); #endif @@ -1762,7 +1884,7 @@ static int tmR3TimerDestroy(PVMCC pVM, PTMTIMER pTimer) { - Assert((unsigned)pTimer->enmClock < (unsigned)TMCLOCK_MAX); - - PTMTIMERQUEUE pQueue = &pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock]; + AssertReturn((unsigned)pTimer->enmClock < (unsigned)TMCLOCK_MAX, VERR_INTERNAL_ERROR_3); + + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[pTimer->enmClock]; bool fActive = false; bool fPending = false; @@ -1776,4 +1898,5 @@ * like create does. All the work is done here. */ + PDMCritSectRwEnterExcl(&pQueue->AllocLock, VERR_IGNORED); TM_LOCK_TIMERS(pVM); for (int cRetries = 1000;; cRetries--) @@ -1814,8 +1937,12 @@ AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->szName)); TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveExcl(&pQueue->AllocLock); + + AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName), + VERR_TM_UNSTABLE_STATE); if (!RTThreadYield()) RTThreadSleep(1); - AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName), - VERR_TM_UNSTABLE_STATE); + + PDMCritSectRwEnterExcl(&pQueue->AllocLock, VERR_IGNORED); TM_LOCK_TIMERS(pVM); continue; @@ -1827,4 +1954,5 @@ case TMTIMERSTATE_DESTROY: TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveExcl(&pQueue->AllocLock); AssertLogRelMsgFailedReturn(("pTimer=%p %s\n", pTimer, tmTimerState(enmState)), VERR_TM_INVALID_STATE); @@ -1832,4 +1960,5 @@ AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName)); TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveExcl(&pQueue->AllocLock); return VERR_TM_UNKNOWN_STATE; } @@ -1845,6 +1974,10 @@ AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->szName)); TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveExcl(&pQueue->AllocLock); + AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName), VERR_TM_UNSTABLE_STATE); + + PDMCritSectRwEnterExcl(&pQueue->AllocLock, VERR_IGNORED); TM_LOCK_TIMERS(pVM); } @@ -1855,17 +1988,17 @@ if (fActive) { - const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer); - const PTMTIMER pNext = TMTIMER_GET_NEXT(pTimer); + const PTMTIMER pPrev = tmTimerGetPrev(pQueue, pTimer); + const PTMTIMER pNext = tmTimerGetNext(pQueue, pTimer); if (pPrev) - TMTIMER_SET_NEXT(pPrev, pNext); + tmTimerSetPrev(pQueue, pPrev, pNext); else { - TMTIMER_SET_HEAD(pQueue, pNext); + tmTimerQueueSetHead(pQueue, pQueue, pNext); pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX; } if (pNext) - TMTIMER_SET_PREV(pNext, pPrev); - pTimer->offNext = 0; - pTimer->offPrev = 0; + tmTimerSetPrev(pQueue, pNext, pPrev); + pTimer->idxNext = UINT32_MAX; + pTimer->idxPrev = UINT32_MAX; } @@ -1877,38 +2010,27 @@ Log3(("TMR3TimerDestroy: tmTimerQueueSchedule\n")); STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a); - Assert(pQueue->offSchedule); - tmTimerQueueSchedule(pVM, pQueue); + Assert(pQueue->idxSchedule < pQueue->cTimersAlloc); + tmTimerQueueSchedule(pVM, pQueue, pQueue); STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a); } +#ifdef VBOX_WITH_STATISTICS /* * Deregister statistics. */ -#ifdef VBOX_WITH_STATISTICS - char szPrefix[128]; - RTStrPrintf(szPrefix, sizeof(szPrefix), "/TM/Timers/%s", pTimer->szName); - STAMR3DeregisterByPrefix(pVM->pUVM, szPrefix); + tmR3TimerDeregisterStats(pVM, pTimer); #endif /* - * Ready to move the timer from the created list and onto the free list. - */ - Assert(!pTimer->offNext); Assert(!pTimer->offPrev); Assert(!pTimer->offScheduleNext); - - /* unlink from created list */ - if (pTimer->pBigPrev) - pTimer->pBigPrev->pBigNext = pTimer->pBigNext; - else - pVM->tm.s.pCreated = pTimer->pBigNext; - if (pTimer->pBigNext) - pTimer->pBigNext->pBigPrev = pTimer->pBigPrev; - pTimer->pBigNext = 0; - pTimer->pBigPrev = 0; - - /* free */ - Log2(("TM: Inserting %p into the free list ahead of %p!\n", pTimer, pVM->tm.s.pFree)); + * Change it to free state and update the queue accordingly. + */ + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX); Assert(pTimer->idxScheduleNext == UINT32_MAX); + TM_SET_STATE(pTimer, TMTIMERSTATE_FREE); - pTimer->pBigNext = pVM->tm.s.pFree; - pVM->tm.s.pFree = pTimer; + + pQueue->cTimersFree += 1; + uint32_t idxTimer = (uint32_t)(pTimer - pQueue->paTimers); + if (idxTimer < pQueue->idxFreeHint) + pQueue->idxFreeHint = idxTimer; #ifdef VBOX_STRICT @@ -1916,4 +2038,5 @@ #endif TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveExcl(&pQueue->AllocLock); return VINF_SUCCESS; } @@ -1951,18 +2074,26 @@ return VERR_INVALID_PARAMETER; - TM_LOCK_TIMERS(pVM); - PTMTIMER pCur = pVM->tm.s.pCreated; - while (pCur) - { - PTMTIMER pDestroy = pCur; - pCur = pDestroy->pBigNext; - if ( pDestroy->enmType == TMTIMERTYPE_DEV - && pDestroy->u.Dev.pDevIns == pDevIns) + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + uint32_t idxTimer = pQueue->cTimersAlloc; + while (idxTimer-- > 0) { - int rc = tmR3TimerDestroy(pVM, pDestroy); - AssertRC(rc); + PTMTIMER pTimer = &pQueue->paTimers[idxTimer]; + if ( pTimer->enmType == TMTIMERTYPE_DEV + && pTimer->u.Dev.pDevIns == pDevIns + && pTimer->enmState < TMTIMERSTATE_DESTROY) + { + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + + int rc = tmR3TimerDestroy(pVM, pTimer); + AssertRC(rc); + + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + } } - } - TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + } LogFlow(("TMR3TimerDestroyDevice: returns VINF_SUCCESS\n")); @@ -1984,18 +2115,26 @@ return VERR_INVALID_PARAMETER; - TM_LOCK_TIMERS(pVM); - PTMTIMER pCur = pVM->tm.s.pCreated; - while (pCur) - { - PTMTIMER pDestroy = pCur; - pCur = pDestroy->pBigNext; - if ( pDestroy->enmType == TMTIMERTYPE_USB - && pDestroy->u.Usb.pUsbIns == pUsbIns) + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + uint32_t idxTimer = pQueue->cTimersAlloc; + while (idxTimer-- > 0) { - int rc = tmR3TimerDestroy(pVM, pDestroy); - AssertRC(rc); + PTMTIMER pTimer = &pQueue->paTimers[idxTimer]; + if ( pTimer->enmType == TMTIMERTYPE_USB + && pTimer->u.Usb.pUsbIns == pUsbIns + && pTimer->enmState < TMTIMERSTATE_DESTROY) + { + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + + int rc = tmR3TimerDestroy(pVM, pTimer); + AssertRC(rc); + + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + } } - } - TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + } LogFlow(("TMR3TimerDestroyUsb: returns VINF_SUCCESS\n")); @@ -2017,18 +2156,26 @@ return VERR_INVALID_PARAMETER; - TM_LOCK_TIMERS(pVM); - PTMTIMER pCur = pVM->tm.s.pCreated; - while (pCur) - { - PTMTIMER pDestroy = pCur; - pCur = pDestroy->pBigNext; - if ( pDestroy->enmType == TMTIMERTYPE_DRV - && pDestroy->u.Drv.pDrvIns == pDrvIns) + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + uint32_t idxTimer = pQueue->cTimersAlloc; + while (idxTimer-- > 0) { - int rc = tmR3TimerDestroy(pVM, pDestroy); - AssertRC(rc); + PTMTIMER pTimer = &pQueue->paTimers[idxTimer]; + if ( pTimer->enmType == TMTIMERTYPE_DRV + && pTimer->u.Drv.pDrvIns == pDrvIns + && pTimer->enmState < TMTIMERSTATE_DESTROY) + { + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + + int rc = tmR3TimerDestroy(pVM, pTimer); + AssertRC(rc); + + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + } } - } - TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + } LogFlow(("TMR3TimerDestroyDriver: returns VINF_SUCCESS\n")); @@ -2069,5 +2216,5 @@ DECLINLINE(bool) tmR3HasExpiredTimer(PVM pVM, TMCLOCK enmClock) { - const uint64_t u64Expire = pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock].u64Expire; + const uint64_t u64Expire = pVM->tm.s.aTimerQueues[enmClock].u64Expire; return u64Expire != INT64_MAX && u64Expire <= tmClock(pVM, enmClock); } @@ -2087,10 +2234,10 @@ */ uint64_t u64Now = TMVirtualGetNoCheck(pVM); - if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64Now) + if (pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64Now) return true; u64Now = pVM->tm.s.fVirtualSyncTicking ? u64Now - pVM->tm.s.offVirtualSync : pVM->tm.s.u64VirtualSync; - if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64Now) + if (pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64Now) return true; @@ -2132,8 +2279,8 @@ #endif if ( !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER) - && ( pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule /** @todo FIXME - reconsider offSchedule as a reason for running the timer queues. */ - || pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule - || pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule - || pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offSchedule + && ( pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].idxSchedule != UINT32_MAX /** @todo FIXME - reconsider offSchedule as a reason for running the timer queues. */ + || pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].idxSchedule != UINT32_MAX + || pVM->tm.s.aTimerQueues[TMCLOCK_REAL].idxSchedule != UINT32_MAX + || pVM->tm.s.aTimerQueues[TMCLOCK_TSC].idxSchedule != UINT32_MAX || tmR3AnyExpiredTimers(pVM) ) @@ -2189,5 +2336,5 @@ VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER); /* Clear the FF once we started working for real. */ - Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule); + Assert(pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].idxSchedule == UINT32_MAX); tmR3TimerQueueRunVirtualSync(pVM); if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */ @@ -2200,22 +2347,24 @@ /* TMCLOCK_VIRTUAL */ STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2); - if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule) - tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]); - tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]); + if (pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].idxSchedule != UINT32_MAX) + tmTimerQueueSchedule(pVM, &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL], &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL]); + tmR3TimerQueueRun(pVM, &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL]); STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2); /* TMCLOCK_TSC */ - Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offActive); /* not used */ + Assert(pVM->tm.s.aTimerQueues[TMCLOCK_TSC].idxActive == UINT32_MAX); /* not used */ /* TMCLOCK_REAL */ STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3); - if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule) - tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]); - tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]); + if (pVM->tm.s.aTimerQueues[TMCLOCK_REAL].idxSchedule != UINT32_MAX) + tmTimerQueueSchedule(pVM, &pVM->tm.s.aTimerQueues[TMCLOCK_REAL], &pVM->tm.s.aTimerQueues[TMCLOCK_REAL]); + tmR3TimerQueueRun(pVM, &pVM->tm.s.aTimerQueues[TMCLOCK_REAL]); STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3); #ifdef VBOX_STRICT /* check that we didn't screw up. */ + TM_LOCK_TIMERS(pVM); tmTimerQueuesSanityChecks(pVM, "TMR3TimerQueuesDo"); + TM_UNLOCK_TIMERS(pVM); #endif @@ -2257,5 +2406,5 @@ * arm the timer again. */ - PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue); + PTMTIMER pNext = tmTimerQueueGetHead(pQueue, pQueue); if (!pNext) return; @@ -2264,5 +2413,5 @@ { PTMTIMER pTimer = pNext; - pNext = TMTIMER_GET_NEXT(pTimer); + pNext = tmTimerGetNext(pQueue, pTimer); PPDMCRITSECT pCritSect = pTimer->pCritSect; if (pCritSect) @@ -2278,19 +2427,19 @@ if (fRc) { - Assert(!pTimer->offScheduleNext); /* this can trigger falsely */ + Assert(pTimer->idxScheduleNext == UINT32_MAX); /* this can trigger falsely */ /* unlink */ - const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer); + const PTMTIMER pPrev = tmTimerGetPrev(pQueue, pTimer); if (pPrev) - TMTIMER_SET_NEXT(pPrev, pNext); + tmTimerSetNext(pQueue, pPrev, pNext); else { - TMTIMER_SET_HEAD(pQueue, pNext); + tmTimerQueueSetHead(pQueue, pQueue, pNext); pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX; } if (pNext) - TMTIMER_SET_PREV(pNext, pPrev); - pTimer->offNext = 0; - pTimer->offPrev = 0; + tmTimerSetPrev(pQueue, pNext, pPrev); + pTimer->idxNext = UINT32_MAX; + pTimer->idxPrev = UINT32_MAX; /* fire */ @@ -2334,5 +2483,5 @@ static void tmR3TimerQueueRunVirtualSync(PVM pVM) { - PTMTIMERQUEUE const pQueue = &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC]; + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; VM_ASSERT_EMT(pVM); Assert(PDMCritSectIsOwner(&pVM->tm.s.VirtualSyncLock)); @@ -2341,5 +2490,5 @@ * Any timers? */ - PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue); + PTMTIMER pNext = tmTimerQueueGetHead(pQueue, pQueue); if (RT_UNLIKELY(!pNext)) { @@ -2454,5 +2603,5 @@ /* Advance */ PTMTIMER pTimer = pNext; - pNext = TMTIMER_GET_NEXT(pTimer); + pNext = tmTimerGetNext(pQueue, pTimer); /* Take the associated lock. */ @@ -2478,5 +2627,5 @@ /* Unlink it, change the state and do the callout. */ - tmTimerQueueUnlinkActive(pVM, pQueue, pTimer); + tmTimerQueueUnlinkActive(pVM, pQueue, pQueue, pTimer); TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER); STAM_PROFILE_START(&pTimer->StatTimer, PrfTimer); @@ -2684,5 +2833,5 @@ Log2(("TMR3VirtualSyncFF: running queue\n")); - Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule); + Assert(pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].idxSchedule == UINT32_MAX); tmR3TimerQueueRunVirtualSync(pVM); if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */ @@ -2744,4 +2893,5 @@ case TMTIMERSTATE_DESTROY: case TMTIMERSTATE_FREE: + case TMTIMERSTATE_INVALID: AssertMsgFailed(("Invalid timer state %d %s (%s)\n", pTimer->enmState, tmTimerState(pTimer->enmState), pTimer->szName)); return SSMR3HandleSetStatus(pSSM, VERR_TM_INVALID_STATE); @@ -3692,21 +3842,28 @@ "HzHint", "State"); - TM_LOCK_TIMERS(pVM); - for (PTMTIMERR3 pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext) - { - pHlp->pfnPrintf(pHlp, - "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n", - pTimer, - pTimer->offNext, - pTimer->offPrev, - pTimer->offScheduleNext, - tmR3Get5CharClockName(pTimer->enmClock), - TMTimerGet(pVM, pTimer->hSelf), - pTimer->u64Expire, - pTimer->uHzHint, - tmTimerState(pTimer->enmState), - pTimer->szName); - } - TM_UNLOCK_TIMERS(pVM); + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); + for (uint32_t idxTimer = 0; idxTimer < pQueue->cTimersAlloc; idxTimer++) + { + PTMTIMER pTimer = &pQueue->paTimers[idxTimer]; + TMTIMERSTATE enmState = pTimer->enmState; + if (enmState < TMTIMERSTATE_DESTROY && enmState > TMTIMERSTATE_INVALID) + pHlp->pfnPrintf(pHlp, + "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n", + pTimer, + pTimer->idxNext, + pTimer->idxPrev, + pTimer->idxScheduleNext, + tmR3Get5CharClockName(pTimer->enmClock), + TMTimerGet(pVM, pTimer->hSelf), + pTimer->u64Expire, + pTimer->uHzHint, + tmTimerState(enmState), + pTimer->szName); + } + PDMCritSectRwLeaveShared(&pQueue->AllocLock); + } } @@ -3734,17 +3891,19 @@ "HzHint", "State"); - for (unsigned iQueue = 0; iQueue < TMCLOCK_MAX; iQueue++) - { + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PDMCritSectRwEnterShared(&pQueue->AllocLock, VERR_IGNORED); TM_LOCK_TIMERS(pVM); - for (PTMTIMERR3 pTimer = TMTIMER_GET_HEAD(&pVM->tm.s.paTimerQueuesR3[iQueue]); + for (PTMTIMERR3 pTimer = tmTimerQueueGetHead(pQueue, pQueue); pTimer; - pTimer = TMTIMER_GET_NEXT(pTimer)) + pTimer = tmTimerGetNext(pQueue, pTimer)) { pHlp->pfnPrintf(pHlp, "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n", pTimer, - pTimer->offNext, - pTimer->offPrev, - pTimer->offScheduleNext, + pTimer->idxNext, + pTimer->idxPrev, + pTimer->idxScheduleNext, tmR3Get5CharClockName(pTimer->enmClock), TMTimerGet(pVM, pTimer->hSelf), @@ -3755,4 +3914,5 @@ } TM_UNLOCK_TIMERS(pVM); + PDMCritSectRwLeaveShared(&pQueue->AllocLock); } } Index: /trunk/src/VBox/VMM/include/TMInline.h =================================================================== --- /trunk/src/VBox/VMM/include/TMInline.h (revision 87791) +++ /trunk/src/VBox/VMM/include/TMInline.h (revision 87792) @@ -23,14 +23,119 @@ +DECLINLINE(PTMTIMER) tmTimerQueueGetHead(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueueShared) +{ +#ifdef IN_RING3 + RT_NOREF(pQueueShared); + uint32_t const idx = pQueueCC->idxActive; +#else + uint32_t const idx = pQueueShared->idxActive; +#endif + if (idx < pQueueCC->cTimersAlloc) + return &pQueueCC->paTimers[idx]; + return NULL; +} + + +DECLINLINE(void) tmTimerQueueSetHead(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueueShared, PTMTIMER pHead) +{ + uint32_t idx; + if (pHead) + { + idx = (uint32_t)(pHead - &pQueueCC->paTimers[0]); + AssertMsgStmt(idx < pQueueCC->cTimersAlloc, + ("idx=%u (%s) cTimersAlloc=%u\n", idx, pHead->szName, pQueueCC->cTimersAlloc), + idx = UINT32_MAX); + } + else + idx = UINT32_MAX; +#ifndef IN_RING3 + pQueueShared->idxActive = idx; +#else + pQueueCC->idxActive = idx; + RT_NOREF(pQueueShared); +#endif +} + + +/** + * Get the previous timer - translates TMTIMER::idxPrev. + */ +DECLINLINE(PTMTIMER) tmTimerGetPrev(PTMTIMERQUEUECC pQueueCC, PTMTIMER pTimer) +{ + uint32_t const idxPrev = pTimer->idxPrev; + Assert(idxPrev); + if (idxPrev < pQueueCC->cTimersAlloc) + return &pQueueCC->paTimers[idxPrev]; + Assert(idxPrev == UINT32_MAX); + return NULL; +} + + +/** + * Get the next timer - translates TMTIMER::idxNext. + */ +DECLINLINE(PTMTIMER) tmTimerGetNext(PTMTIMERQUEUECC pQueueCC, PTMTIMER pTimer) +{ + uint32_t const idxNext = pTimer->idxNext; + Assert(idxNext); + if (idxNext < pQueueCC->cTimersAlloc) + return &pQueueCC->paTimers[idxNext]; + Assert(idxNext == UINT32_MAX); + return NULL; +} + + +/** + * Set the previous timer link (TMTIMER::idxPrev). + */ +DECLINLINE(void) tmTimerSetPrev(PTMTIMERQUEUECC pQueueCC, PTMTIMER pTimer, PTMTIMER pPrev) +{ + uint32_t idxPrev; + if (pPrev) + { + idxPrev = (uint32_t)(pPrev - &pQueueCC->paTimers[0]); + Assert(idxPrev); + AssertMsgStmt(idxPrev < pQueueCC->cTimersAlloc, + ("idxPrev=%u (%s) cTimersAlloc=%u\n", idxPrev, pPrev->szName, pQueueCC->cTimersAlloc), + idxPrev = UINT32_MAX); + } + else + idxPrev = UINT32_MAX; + pTimer->idxPrev = idxPrev; +} + + +/** + * Set the next timer link (TMTIMER::idxNext). + */ +DECLINLINE(void) tmTimerSetNext(PTMTIMERQUEUECC pQueueCC, PTMTIMER pTimer, PTMTIMER pNext) +{ + uint32_t idxNext; + if (pNext) + { + idxNext = (uint32_t)(pNext - &pQueueCC->paTimers[0]); + Assert(idxNext); + AssertMsgStmt(idxNext < pQueueCC->cTimersAlloc, + ("idxNext=%u (%s) cTimersAlloc=%u\n", idxNext, pNext->szName, pQueueCC->cTimersAlloc), + idxNext = UINT32_MAX); + } + else + idxNext = UINT32_MAX; + pTimer->idxNext = idxNext; +} + + /** * Used to unlink a timer from the active list. * * @param pVM The cross context VM structure. - * @param pQueue The timer queue. + * @param pQueueCC The context specific queue data (same as @a pQueue for + * ring-3). + * @param pQueue The shared timer queue data. * @param pTimer The timer that needs linking. * * @remarks Called while owning the relevant queue lock. */ -DECL_FORCE_INLINE(void) tmTimerQueueUnlinkActive(PVMCC pVM, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) +DECL_FORCE_INLINE(void) tmTimerQueueUnlinkActive(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) { #ifdef VBOX_STRICT @@ -42,18 +147,18 @@ RT_NOREF(pVM); - const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer); - const PTMTIMER pNext = TMTIMER_GET_NEXT(pTimer); + const PTMTIMER pPrev = tmTimerGetPrev(pQueueCC, pTimer); + const PTMTIMER pNext = tmTimerGetNext(pQueueCC, pTimer); if (pPrev) - TMTIMER_SET_NEXT(pPrev, pNext); - else - { - TMTIMER_SET_HEAD(pQueue, pNext); + tmTimerSetNext(pQueueCC, pPrev, pNext); + else + { + tmTimerQueueSetHead(pQueueCC, pQueue, pNext); pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX; DBGFTRACE_U64_TAG(pVM, pQueue->u64Expire, "tmTimerQueueUnlinkActive"); } if (pNext) - TMTIMER_SET_PREV(pNext, pPrev); - pTimer->offNext = 0; - pTimer->offPrev = 0; + tmTimerSetPrev(pQueueCC, pNext, pPrev); + pTimer->idxNext = UINT32_MAX; + pTimer->idxPrev = UINT32_MAX; } @@ -70,15 +175,28 @@ #ifdef IN_RING3 # define TMTIMER_HANDLE_TO_PTR_RETURN_EX(a_pVM, a_hTimer, a_rcRet, a_pTimerVar) do { \ - RT_NOREF(a_pVM); \ - (a_pTimerVar) = (PTMTIMER)hTimer; \ - AssertPtrReturn((a_pTimerVar), a_rcRet); \ + uintptr_t const idxQueue = (uintptr_t)((a_hTimer) >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) \ + & (uintptr_t)TMTIMERHANDLE_QUEUE_IDX_SMASK; \ + AssertReturn(idxQueue < RT_ELEMENTS((a_pVM)->tm.s.aTimerQueues), a_rcRet); \ + \ + uintptr_t const idxTimer = (uintptr_t)((a_hTimer) & TMTIMERHANDLE_TIMER_IDX_MASK); \ + AssertReturn(idxQueue < (a_pVM)->tm.s.aTimerQueues[idxQueue].cTimersAlloc, a_rcRet); \ + \ + (a_pTimerVar) = &(a_pVM)->tm.s.aTimerQueues[idxQueue].paTimers[idxTimer]; \ AssertReturn((a_pTimerVar)->hSelf == a_hTimer, a_rcRet); \ } while (0) #else # define TMTIMER_HANDLE_TO_PTR_RETURN_EX(a_pVM, a_hTimer, a_rcRet, a_pTimerVar) do { \ - (a_pTimerVar) = (PTMTIMER)MMHyperR3ToCC(pVM, (RTR3PTR)hTimer); \ - AssertPtrReturn((a_pTimerVar), a_rcRet); \ + uintptr_t const idxQueue = (uintptr_t)((a_hTimer) >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) \ + & (uintptr_t)TMTIMERHANDLE_QUEUE_IDX_SMASK; \ + AssertReturn(idxQueue < RT_ELEMENTS((a_pVM)->tm.s.aTimerQueues), a_rcRet); \ + AssertCompile(RT_ELEMENTS((a_pVM)->tm.s.aTimerQueues) == RT_ELEMENTS((a_pVM)->tmr0.s.aTimerQueues)); \ + \ + uintptr_t const idxTimer = (uintptr_t)((a_hTimer) & TMTIMERHANDLE_TIMER_IDX_MASK); \ + AssertReturn(idxQueue < (a_pVM)->tmr0.s.aTimerQueues[idxQueue].cTimersAlloc, a_rcRet); \ + \ + (a_pTimerVar) = &(a_pVM)->tmr0.s.aTimerQueues[idxQueue].paTimers[idxTimer]; \ AssertReturn((a_pTimerVar)->hSelf == a_hTimer, a_rcRet); \ Assert((a_pTimerVar)->fFlags & TMTIMER_FLAGS_RING0); \ + Assert(VM_IS_EMT(pVM)); \ } while (0) #endif @@ -105,15 +223,28 @@ #ifdef IN_RING3 # define TMTIMER_HANDLE_TO_PTR_RETURN_VOID(a_pVM, a_hTimer, a_pTimerVar) do { \ - RT_NOREF(a_pVM); \ - (a_pTimerVar) = (PTMTIMER)hTimer; \ - AssertPtrReturnVoid((a_pTimerVar)); \ + uintptr_t const idxQueue = (uintptr_t)((a_hTimer) >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) \ + & (uintptr_t)TMTIMERHANDLE_QUEUE_IDX_SMASK; \ + AssertReturnVoid(idxQueue < RT_ELEMENTS((a_pVM)->tm.s.aTimerQueues)); \ + \ + uintptr_t const idxTimer = (uintptr_t)((a_hTimer) & TMTIMERHANDLE_TIMER_IDX_MASK); \ + AssertReturnVoid(idxQueue < (a_pVM)->tm.s.aTimerQueues[idxQueue].cTimersAlloc); \ + \ + (a_pTimerVar) = &(a_pVM)->tm.s.aTimerQueues[idxQueue].paTimers[idxTimer]; \ AssertReturnVoid((a_pTimerVar)->hSelf == a_hTimer); \ } while (0) #else # define TMTIMER_HANDLE_TO_PTR_RETURN_VOID(a_pVM, a_hTimer, a_pTimerVar) do { \ - (a_pTimerVar) = (PTMTIMER)MMHyperR3ToCC(pVM, (RTR3PTR)hTimer); \ - AssertPtrReturnVoid((a_pTimerVar)); \ + uintptr_t const idxQueue = (uintptr_t)((a_hTimer) >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) \ + & (uintptr_t)TMTIMERHANDLE_QUEUE_IDX_SMASK; \ + AssertReturnVoid(idxQueue < RT_ELEMENTS((a_pVM)->tm.s.aTimerQueues)); \ + AssertCompile(RT_ELEMENTS((a_pVM)->tm.s.aTimerQueues) == RT_ELEMENTS((a_pVM)->tmr0.s.aTimerQueues)); \ + \ + uintptr_t const idxTimer = (uintptr_t)((a_hTimer) & TMTIMERHANDLE_TIMER_IDX_MASK); \ + AssertReturnVoid(idxQueue < (a_pVM)->tmr0.s.aTimerQueues[idxQueue].cTimersAlloc); \ + \ + (a_pTimerVar) = &(a_pVM)->tmr0.s.aTimerQueues[idxQueue].paTimers[idxTimer]; \ AssertReturnVoid((a_pTimerVar)->hSelf == a_hTimer); \ Assert((a_pTimerVar)->fFlags & TMTIMER_FLAGS_RING0); \ + Assert(VM_IS_EMT(pVM)); \ } while (0) #endif Index: /trunk/src/VBox/VMM/include/TMInternal.h =================================================================== --- /trunk/src/VBox/VMM/include/TMInternal.h (revision 87791) +++ /trunk/src/VBox/VMM/include/TMInternal.h (revision 87792) @@ -29,4 +29,5 @@ #include #include +#include RT_C_DECLS_BEGIN @@ -50,6 +51,8 @@ typedef enum TMTIMERTYPE { + /** Invalid zero value. */ + TMTIMERTYPE_INVALID = 0, /** Device timer. */ - TMTIMERTYPE_DEV = 1, + TMTIMERTYPE_DEV, /** USB device timer. */ TMTIMERTYPE_USB, @@ -65,6 +68,8 @@ typedef enum TMTIMERSTATE { + /** Invalid zero entry (used for table entry zero). */ + TMTIMERSTATE_INVALID = 0, /** Timer is stopped. */ - TMTIMERSTATE_STOPPED = 1, + TMTIMERSTATE_STOPPED, /** Timer is active. */ TMTIMERSTATE_ACTIVE, @@ -104,4 +109,13 @@ && (enmState) >= TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE) +/** @name Timer handle value elements + * @{ */ +#define TMTIMERHANDLE_RANDOM_MASK UINT64_C(0xffffffffff000000) +#define TMTIMERHANDLE_QUEUE_IDX_SHIFT 16 +#define TMTIMERHANDLE_QUEUE_IDX_MASK UINT64_C(0x0000000000ff0000) +#define TMTIMERHANDLE_QUEUE_IDX_SMASK UINT64_C(0x00000000000000ff) +#define TMTIMERHANDLE_TIMER_IDX_MASK UINT64_C(0x000000000000ffff) +/** @} */ + /** @@ -166,11 +180,11 @@ /** Timer state. */ volatile TMTIMERSTATE enmState; - /** Timer relative offset to the next timer in the schedule list. */ - int32_t volatile offScheduleNext; - - /** Timer relative offset to the next timer in the chain. */ - int32_t offNext; - /** Timer relative offset to the previous timer in the chain. */ - int32_t offPrev; + /** The index of the next next timer in the schedule list. */ + int32_t volatile idxScheduleNext; + + /** The index of the next timer in the chain. */ + uint32_t idxNext; + /** The index of the previous timer in the chain. */ + uint32_t idxPrev; /** It's own handle value. */ @@ -186,8 +200,4 @@ R3PTRTYPE(PPDMCRITSECT) pCritSect; - /** Pointer to the next timer in the list of created or free timers. (TM::pTimers or TM::pFree) */ - PTMTIMERR3 pBigNext; - /** Pointer to the previous timer in the list of all created timers. (TM::pTimers) */ - PTMTIMERR3 pBigPrev; /** The timer name. */ char szName[32]; @@ -234,21 +244,13 @@ #endif -/** Get the previous timer. */ -#define TMTIMER_GET_PREV(pTimer) ((PTMTIMER)((pTimer)->offPrev ? (intptr_t)(pTimer) + (pTimer)->offPrev : 0)) -/** Get the next timer. */ -#define TMTIMER_GET_NEXT(pTimer) ((PTMTIMER)((pTimer)->offNext ? (intptr_t)(pTimer) + (pTimer)->offNext : 0)) -/** Set the previous timer link. */ -#define TMTIMER_SET_PREV(pTimer, pPrev) ((pTimer)->offPrev = (pPrev) ? (intptr_t)(pPrev) - (intptr_t)(pTimer) : 0) -/** Set the next timer link. */ -#define TMTIMER_SET_NEXT(pTimer, pNext) ((pTimer)->offNext = (pNext) ? (intptr_t)(pNext) - (intptr_t)(pTimer) : 0) - - -/** - * A timer queue. - * - * This is allocated on the hyper heap. + +/** + * A timer queue, shared. */ typedef struct TMTIMERQUEUE { + /** The ring-0 mapping of the timer table. */ + R3PTRTYPE(PTMTIMER) paTimers; + /** The cached expire time for this queue. * Updated by EMT when scheduling the queue or modifying the head timer. @@ -259,8 +261,6 @@ * When no scheduling is pending, this list is will be ordered by expire time (ascending). * Access is serialized by only letting the emulation thread (EMT) do changes. - * - * The offset is relative to the queue structure. */ - int32_t offActive; + uint32_t idxActive; /** List of timers pending scheduling of some kind. * @@ -268,21 +268,59 @@ * TMTIMERSTATE_PENDING_DESTRUCTION, TMTIMERSTATE_PENDING_STOPPING_DESTRUCTION, * TMTIMERSTATE_PENDING_RESCHEDULING and TMTIMERSTATE_PENDING_SCHEDULE. - * - * The offset is relative to the queue structure. */ - int32_t volatile offSchedule; + uint32_t volatile idxSchedule; /** The clock for this queue. */ TMCLOCK enmClock; - /** Pad the structure up to 32 bytes. */ - uint32_t au32Padding[3]; + + /** The size of the paTimers allocation (in entries). */ + uint32_t cTimersAlloc; + /** Number of free timer entries. */ + uint32_t cTimersFree; + /** Where to start looking for free timers. */ + uint32_t idxFreeHint; + /** The queue name. */ + char szName[16]; + /** Set if we've disabled growing. */ + bool fCannotGrow; + /** Align on 64-byte boundrary. */ + bool afAlignment[7]; + /** Lock serializing timer allocation and deallocation. */ + PDMCRITSECTRW AllocLock; } TMTIMERQUEUE; - +AssertCompileSizeAlignment(TMTIMERQUEUE, 64); /** Pointer to a timer queue. */ typedef TMTIMERQUEUE *PTMTIMERQUEUE; -/** Get the head of the active timer list. */ -#define TMTIMER_GET_HEAD(pQueue) ((PTMTIMER)((pQueue)->offActive ? (intptr_t)(pQueue) + (pQueue)->offActive : 0)) -/** Set the head of the active timer list. */ -#define TMTIMER_SET_HEAD(pQueue, pHead) ((pQueue)->offActive = pHead ? (intptr_t)pHead - (intptr_t)(pQueue) : 0) +/** + * A timer queue, ring-0 only bits. + */ +typedef struct TMTIMERQUEUER0 +{ + /** The size of the paTimers allocation (in entries). */ + uint32_t cTimersAlloc; + uint32_t uAlignment; + /** The ring-0 mapping of the timer table. */ + R0PTRTYPE(PTMTIMER) paTimers; + /** Handle to the timer table allocation. */ + RTR0MEMOBJ hMemObj; + /** Handle to the ring-3 mapping of the timer table. */ + RTR0MEMOBJ hMapObj; +} TMTIMERQUEUER0; +/** Pointer to the ring-0 timer queue data. */ +typedef TMTIMERQUEUER0 *PTMTIMERQUEUER0; + +/** Pointer to the current context data for a timer queue. + * @note In ring-3 this is the same as the shared data. */ +#ifdef IN_RING3 +typedef TMTIMERQUEUE *PTMTIMERQUEUECC; +#else +typedef TMTIMERQUEUER0 *PTMTIMERQUEUECC; +#endif +/** Helper macro for getting the current context queue point. */ +#ifdef IN_RING3 +# define TM_GET_TIMER_QUEUE_CC(a_pVM, a_idxQueue, a_pQueueShared) (a_pQueueShared) +#else +# define TM_GET_TIMER_QUEUE_CC(a_pVM, a_idxQueue, a_pQueueShared) (&(a_pVM)->tmr0.s.aTimerQueues[a_idxQueue]) +#endif @@ -348,12 +386,4 @@ /** - * Converts a TM pointer into a VM pointer. - * @returns Pointer to the VM structure the TM is part of. - * @param pTM Pointer to TM instance data. - */ -#define TM2VM(pTM) ( (PVM)((char*)pTM - pTM->offVM) ) - - -/** * TM VM Instance data. * Changes to this must checked against the padding of the cfgm union in VM! @@ -361,8 +391,4 @@ typedef struct TM { - /** Offset to the VM structure. - * See TM2VM(). */ - RTUINT offVM; - /** The current TSC mode of the VM. * Config variable: Mode (string). */ @@ -370,6 +396,4 @@ /** The original TSC mode of the VM. */ TMTSCMODE enmOriginalTSCMode; - /** Alignment padding. */ - uint32_t u32Alignment0; /** Whether the TSC is tied to the execution of code. * Config variable: TSCTiedToExecution (bool) */ @@ -513,10 +537,6 @@ R3PTRTYPE(char *) pszAlignment2b; - /** Timer queues for the different clock types - R3 Ptr */ - R3PTRTYPE(PTMTIMERQUEUE) paTimerQueuesR3; - /** Timer queues for the different clock types - R0 Ptr */ - R0PTRTYPE(PTMTIMERQUEUE) paTimerQueuesR0; - /** Timer queues for the different clock types - RC Ptr */ - RCPTRTYPE(PTMTIMERQUEUE) paTimerQueuesRC; + /** Timer queues for the different clock types. */ + TMTIMERQUEUE aTimerQueues[TMCLOCK_MAX]; /** Pointer to our RC mapping of the GIP. */ @@ -524,14 +544,4 @@ /** Pointer to our R3 mapping of the GIP. */ R3PTRTYPE(void *) pvGIPR3; - - /** Pointer to a singly linked list of free timers. - * This chain is using the TMTIMER::pBigNext members. - * Only accessible from the emulation thread. */ - PTMTIMERR3 pFree; - - /** Pointer to a doubly linked list of created timers. - * This chain is using the TMTIMER::pBigNext and TMTIMER::pBigPrev members. - * Only accessible from the emulation thread. */ - PTMTIMERR3 pCreated; /** The schedule timer timer handle (runtime timer). @@ -696,4 +706,5 @@ /** Pointer to TM VM instance data. */ typedef TM *PTM; + /** @@ -796,8 +807,19 @@ typedef TMCPU *PTMCPU; + +/** + * TM data kept in the ring-0 GVM. + */ +typedef struct TMR0PERVM +{ + /** Timer queues for the different clock types. */ + TMTIMERQUEUER0 aTimerQueues[TMCLOCK_MAX]; +} TMR0PERVM; + + const char *tmTimerState(TMTIMERSTATE enmState); -void tmTimerQueueSchedule(PVMCC pVM, PTMTIMERQUEUE pQueue); +void tmTimerQueueSchedule(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue); #ifdef VBOX_STRICT -void tmTimerQueuesSanityChecks(PVM pVM, const char *pszWhere); +void tmTimerQueuesSanityChecks(PVMCC pVM, const char *pszWhere); #endif