/* $Id: PDMAllCritSectRw.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */ /** @file * IPRT - Read/Write Critical Section, Generic. */ /* * Copyright (C) 2009-2023 Oracle and/or its affiliates. * * This file is part of VirtualBox base platform packages, as * available from https://www.virtualbox.org. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, in version 3 of the * License. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, see . * * SPDX-License-Identifier: GPL-3.0-only */ /********************************************************************************************************************************* * Header Files * *********************************************************************************************************************************/ #define LOG_GROUP LOG_GROUP_PDM_CRITSECTRW #include "PDMInternal.h" #include #include #include #include #include #include #include #include #include #ifdef IN_RING3 # include #endif #if defined(IN_RING3) || defined(IN_RING0) # include # include #endif #ifdef IN_RING0 # include #endif #ifdef RT_ARCH_AMD64 # include #endif /********************************************************************************************************************************* * Defined Constants And Macros * *********************************************************************************************************************************/ #if 0 /* unused */ /** The number loops to spin for shared access in ring-3. */ #define PDMCRITSECTRW_SHRD_SPIN_COUNT_R3 20 /** The number loops to spin for shared access in ring-0. */ #define PDMCRITSECTRW_SHRD_SPIN_COUNT_R0 128 /** The number loops to spin for shared access in the raw-mode context. */ #define PDMCRITSECTRW_SHRD_SPIN_COUNT_RC 128 /** The number loops to spin for exclusive access in ring-3. */ #define PDMCRITSECTRW_EXCL_SPIN_COUNT_R3 20 /** The number loops to spin for exclusive access in ring-0. */ #define PDMCRITSECTRW_EXCL_SPIN_COUNT_R0 256 /** The number loops to spin for exclusive access in the raw-mode context. */ #define PDMCRITSECTRW_EXCL_SPIN_COUNT_RC 256 #endif /** Max number of write or write/read recursions. */ #define PDM_CRITSECTRW_MAX_RECURSIONS _1M /** Skips some of the overly paranoid atomic reads and updates. * Makes some assumptions about cache coherence, though not brave enough not to * always end with an atomic update. */ #define PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF /** For reading RTCRITSECTRWSTATE::s::u64State. */ #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF # define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicUoReadU64(a_pu64State) #else # define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicReadU64(a_pu64State) #endif /* Undefine the automatic VBOX_STRICT API mappings. */ #undef PDMCritSectRwEnterExcl #undef PDMCritSectRwTryEnterExcl #undef PDMCritSectRwEnterShared #undef PDMCritSectRwTryEnterShared /********************************************************************************************************************************* * Defined Constants And Macros * *********************************************************************************************************************************/ #if defined(RTASM_HAVE_CMP_WRITE_U128) && defined(RT_ARCH_AMD64) static int32_t g_fCmpWriteSupported = -1; #endif /********************************************************************************************************************************* * Internal Functions * *********************************************************************************************************************************/ static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal); #ifdef RTASM_HAVE_CMP_WRITE_U128 # ifdef RT_ARCH_AMD64 /** * Called once to initialize g_fCmpWriteSupported. */ DECL_NO_INLINE(static, bool) pdmCritSectRwIsCmpWriteU128SupportedSlow(void) { bool const fCmpWriteSupported = RT_BOOL(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16); ASMAtomicWriteS32(&g_fCmpWriteSupported, fCmpWriteSupported); return fCmpWriteSupported; } # endif /** * Indicates whether hardware actually supports 128-bit compare & write. */ DECL_FORCE_INLINE(bool) pdmCritSectRwIsCmpWriteU128Supported(void) { # ifdef RT_ARCH_AMD64 int32_t const fCmpWriteSupported = g_fCmpWriteSupported; if (RT_LIKELY(fCmpWriteSupported >= 0)) return fCmpWriteSupported != 0; return pdmCritSectRwIsCmpWriteU128SupportedSlow(); # else return true; # endif } #endif /* RTASM_HAVE_CMP_WRITE_U128 */ /** * Gets the ring-3 native thread handle of the calling thread. * * @returns native thread handle (ring-3). * @param pVM The cross context VM structure. * @param pThis The read/write critical section. This is only used in * R0 and RC. */ DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectRwGetNativeSelf(PVMCC pVM, PCPDMCRITSECTRW pThis) { #ifdef IN_RING3 RT_NOREF(pVM, pThis); RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); #elif defined(IN_RING0) AssertMsgReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, ("%RX32\n", pThis->s.Core.u32Magic), NIL_RTNATIVETHREAD); RTNATIVETHREAD hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM); Assert(hNativeSelf != NIL_RTNATIVETHREAD); #else # error "invalid context" #endif return hNativeSelf; } DECL_NO_INLINE(static, int) pdmCritSectRwCorrupted(PPDMCRITSECTRW pThis, const char *pszMsg) { ASMAtomicWriteU32(&pThis->s.Core.u32Magic, PDMCRITSECTRW_MAGIC_CORRUPT); LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pThis)); return VERR_PDM_CRITSECTRW_IPE; } #ifdef IN_RING3 /** * Changes the lock validator sub-class of the read/write critical section. * * It is recommended to try make sure that nobody is using this critical section * while changing the value. * * @returns The old sub-class. RTLOCKVAL_SUB_CLASS_INVALID is returns if the * lock validator isn't compiled in or either of the parameters are * invalid. * @param pThis Pointer to the read/write critical section. * @param uSubClass The new sub-class value. */ VMMDECL(uint32_t) PDMR3CritSectRwSetSubClass(PPDMCRITSECTRW pThis, uint32_t uSubClass) { AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID); # if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) AssertReturn(!(pThis->s.Core.fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID); RTLockValidatorRecSharedSetSubClass(pThis->s.Core.pValidatorRead, uSubClass); return RTLockValidatorRecExclSetSubClass(pThis->s.Core.pValidatorWrite, uSubClass); # else NOREF(uSubClass); return RTLOCKVAL_SUB_CLASS_INVALID; # endif } #endif /* IN_RING3 */ /** * Worker for pdmCritSectRwEnterShared returning with read-ownership of the CS. */ DECL_FORCE_INLINE(int) pdmCritSectRwEnterSharedGotIt(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) { #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (!fNoVal) RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos); #else RT_NOREF(pSrcPos, fNoVal, hThreadSelf); #endif /* got it! */ STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared)); Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)); return VINF_SUCCESS; } /** * Worker for pdmCritSectRwEnterShared and pdmCritSectRwEnterSharedBailOut * that decrement the wait count and maybe resets the semaphore. */ DECLINLINE(int) pdmCritSectRwEnterSharedGotItAfterWaiting(PVMCC pVM, PPDMCRITSECTRW pThis, uint64_t u64State, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) { for (;;) { uint64_t const u64OldState = u64State; uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count")); AssertReturn((u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT > 0, pdmCritSectRwCorrupted(pThis, "Invalid read count")); cWait--; u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { if (cWait == 0) { if (ASMAtomicXchgBool(&pThis->s.Core.fNeedReset, false)) { int rc = SUPSemEventMultiReset(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead); AssertRCReturn(rc, rc); } } return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); } ASMNopPause(); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); ASMNopPause(); u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); } /* not reached */ } #if defined(IN_RING0) || (defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)) /** * Worker for pdmCritSectRwEnterSharedContended that decrements both read counts * and returns @a rc. * * @note May return VINF_SUCCESS if we race the exclusive leave function and * come out on the bottom. * * Ring-3 only calls in a case where it is _not_ acceptable to take the * lock, so even if we get the lock we'll have to leave. In the ring-0 * contexts, we can safely return VINF_SUCCESS in case of a race. */ DECL_NO_INLINE(static, int) pdmCritSectRwEnterSharedBailOut(PVMCC pVM, PPDMCRITSECTRW pThis, int rc, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) { #ifdef IN_RING0 uint64_t const tsStart = RTTimeNanoTS(); uint64_t cNsElapsed = 0; #endif for (;;) { uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); uint64_t u64OldState = u64State; uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count on bailout")); cWait--; uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid read count on bailout")); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) { c--; u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK); u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) return rc; } else { /* * The direction changed, so we can actually get the lock now. * * This means that we _have_ to wait on the semaphore to be signalled * so we can properly reset it. Otherwise the stuff gets out of wack, * because signalling and resetting will race one another. An * exception would be if we're not the last reader waiting and don't * need to worry about the resetting. * * An option would be to do the resetting in PDMCritSectRwEnterExcl, * but that would still leave a racing PDMCritSectRwEnterShared * spinning hard for a little bit, which isn't great... */ if (cWait == 0) { # ifdef IN_RING0 /* Do timeout processing first to avoid redoing the above. */ uint32_t cMsWait; if (cNsElapsed <= RT_NS_10SEC) cMsWait = 32; else { u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { LogFunc(("%p: giving up\n", pThis)); return rc; } cMsWait = 2; } int rcWait = SUPSemEventMultiWait(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, cMsWait); Log11Func(("%p: rc=%Rrc %'RU64 ns (hNativeWriter=%p u64State=%#RX64)\n", pThis, rcWait, RTTimeNanoTS() - tsStart, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State)); # else RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false); int rcWait = SUPSemEventMultiWaitNoResume(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, RT_MS_5SEC); RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); # endif if (rcWait == VINF_SUCCESS) { # ifdef IN_RING0 return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf); # else /* ring-3: Cannot return VINF_SUCCESS. */ Assert(RT_FAILURE_NP(rc)); int rc2 = pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf); if (RT_SUCCESS(rc2)) rc2 = pdmCritSectRwLeaveSharedWorker(pVM, pThis, fNoVal); return rc; # endif } AssertMsgReturn(rcWait == VERR_TIMEOUT || rcWait == VERR_INTERRUPTED, ("%p: rcWait=%Rrc rc=%Rrc", pThis, rcWait, rc), RT_FAILURE_NP(rcWait) ? rcWait : -rcWait); } else { u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); } # ifdef IN_RING0 /* Calculate the elapsed time here to avoid redoing state work. */ cNsElapsed = RTTimeNanoTS() - tsStart; # endif } ASMNopPause(); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); ASMNopPause(); } } #endif /* IN_RING0 || (IN_RING3 && PDMCRITSECTRW_STRICT) */ /** * Worker for pdmCritSectRwEnterShared that handles waiting for a contended CS. * Caller has already added us to the read and read-wait counters. */ static int pdmCritSectRwEnterSharedContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) { PSUPDRVSESSION const pSession = pVM->pSession; SUPSEMEVENTMULTI const hEventMulti = (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead; # ifdef IN_RING0 uint64_t const tsStart = RTTimeNanoTS(); uint64_t const cNsMaxTotalDef = RT_NS_5MIN; uint64_t cNsMaxTotal = cNsMaxTotalDef; uint32_t cMsMaxOne = RT_MS_5SEC; bool fNonInterruptible = false; # endif for (uint32_t iLoop = 0; ; iLoop++) { /* * Wait for the direction to switch. */ int rc; # ifdef IN_RING3 # if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) rc = RTLockValidatorRecSharedCheckBlocking(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, true, RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false); if (RT_FAILURE(rc)) return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf); # else RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false); # endif # endif for (;;) { /* * We always wait with a timeout so we can re-check the structure sanity * and not get stuck waiting on a corrupt or deleted section. */ # ifdef IN_RING3 rc = SUPSemEventMultiWaitNoResume(pSession, hEventMulti, RT_MS_5SEC); # else rc = !fNonInterruptible ? SUPSemEventMultiWaitNoResume(pSession, hEventMulti, cMsMaxOne) : SUPSemEventMultiWait(pSession, hEventMulti, cMsMaxOne); Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p u64State=%#RX64)\n", pThis, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State)); # endif if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) { /* likely */ } else { # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); # endif return VERR_SEM_DESTROYED; } if (RT_LIKELY(rc == VINF_SUCCESS)) break; /* * Timeout and interrupted waits needs careful handling in ring-0 * because we're cooperating with ring-3 on this critical section * and thus need to make absolutely sure we won't get stuck here. * * The r0 interrupted case means something is pending (termination, * signal, APC, debugger, whatever), so we must try our best to * return to the caller and to ring-3 so it can be dealt with. */ if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED) { # ifdef IN_RING0 uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart; int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD); AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING, ("rcTerm=%Rrc\n", rcTerm)); if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef) cNsMaxTotal = RT_NS_1MIN; if (rc == VERR_TIMEOUT) { /* Try return get out of here with a non-VINF_SUCCESS status if the thread is terminating or if the timeout has been exceeded. */ STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrTimeout); if ( rcTerm == VINF_THREAD_IS_TERMINATING || cNsElapsed > cNsMaxTotal) return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc, pSrcPos, fNoVal, hThreadSelf); } else { /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS, we will try non-interruptible sleep for a while to help resolve the issue w/o guru'ing. */ STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrInterrupted); if ( rcTerm != VINF_THREAD_IS_TERMINATING && rcBusy == VINF_SUCCESS && pVCpu != NULL && cNsElapsed <= cNsMaxTotal) { if (!fNonInterruptible) { STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedNonInterruptibleWaits); fNonInterruptible = true; cMsMaxOne = 32; uint64_t cNsLeft = cNsMaxTotal - cNsElapsed; if (cNsLeft > RT_NS_10SEC) cNsMaxTotal = cNsElapsed + RT_NS_10SEC; } } else return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc, pSrcPos, fNoVal, hThreadSelf); } # else /* IN_RING3 */ RT_NOREF(pVM, pVCpu, rcBusy); # endif /* IN_RING3 */ } /* * Any other return code is fatal. */ else { # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); # endif AssertMsgFailed(("rc=%Rrc\n", rc)); return RT_FAILURE_NP(rc) ? rc : -rc; } } # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ); # endif /* * Check the direction. */ Assert(pThis->s.Core.fNeedReset); uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) { /* * Decrement the wait count and maybe reset the semaphore (if we're last). */ return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf); } AssertMsg(iLoop < 1, ("%p: %u u64State=%#RX64 hNativeWriter=%p\n", pThis, iLoop, u64State, pThis->s.Core.u.s.hNativeWriter)); RTThreadYield(); } /* not reached */ } /** * Worker that enters a read/write critical section with shard access. * * @returns VBox status code. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param rcBusy The busy return code for ring-0 and ring-3. * @param fTryOnly Only try enter it, don't wait. * @param pSrcPos The source position. (Can be NULL.) * @param fNoVal No validation records. */ static int pdmCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal) { /* * Validate input. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt(); if (!fTryOnly) { int rc9; RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis)) rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); else rc9 = RTLockValidatorRecSharedCheckOrder(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc9)) return rc9; } #else RTTHREAD hThreadSelf = NIL_RTTHREAD; #endif /* * Work the state. */ uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); uint64_t u64OldState = u64State; for (;;) { if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) { /* It flows in the right direction, try follow it before it changes. */ uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; c++; Assert(c < RTCSRW_CNT_MASK / 4); AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS); u64State &= ~RTCSRW_CNT_RD_MASK; u64State |= c << RTCSRW_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); } else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0) { /* Wrong direction, but we're alone here and can simply try switch the direction. */ u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { Assert(!pThis->s.Core.fNeedReset); return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); } } else { /* Is the writer perhaps doing a read recursion? */ RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); if (hNativeWriter != NIL_RTNATIVETHREAD) { RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); if (hNativeSelf == hNativeWriter) { #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (!fNoVal) { int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core, pSrcPos); if (RT_FAILURE(rc9)) return rc9; } #endif uint32_t const cReads = ASMAtomicIncU32(&pThis->s.Core.cWriterReads); Assert(cReads < _16K); AssertReturnStmt(cReads < PDM_CRITSECTRW_MAX_RECURSIONS, ASMAtomicDecU32(&pThis->s.Core.cWriterReads), VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS); STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared)); return VINF_SUCCESS; /* don't break! */ } } /* * If we're only trying, return already. */ if (fTryOnly) { STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared)); return VERR_SEM_BUSY; } #if defined(IN_RING3) || defined(IN_RING0) /* * Add ourselves to the queue and wait for the direction to change. */ uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; c++; Assert(c < RTCSRW_CNT_MASK / 2); AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS); uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; cWait++; Assert(cWait <= c); Assert(cWait < RTCSRW_CNT_MASK / 2); AssertReturn(cWait < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS); u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK); u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { /* * In ring-3 it's straight forward, just optimize the RTThreadSelf() call. */ # if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT) return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf); # elif defined(IN_RING3) return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, RTThreadSelf()); # else /* IN_RING0 */ /* * In ring-0 context we have to take the special VT-x/AMD-V HM context into * account when waiting on contended locks. */ PVMCPUCC pVCpu = VMMGetCpu(pVM); if (pVCpu) { VMMR0EMTBLOCKCTX Ctx; int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx); if (rc == VINF_SUCCESS) { Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); rc = pdmCritSectRwEnterSharedContended(pVM, pVCpu, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf); VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); } else { //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy); rc = pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf); } return rc; } /* Non-EMT. */ Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf); # endif /* IN_RING0 */ } #else /* !IN_RING3 && !IN_RING0 */ /* * We cannot call SUPSemEventMultiWaitNoResume in this context. Go * back to ring-3 and do it there or return rcBusy. */ # error "Unused code." STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared)); if (rcBusy == VINF_SUCCESS) { PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); /** @todo Should actually do this in via VMMR0.cpp instead of going all the way * back to ring-3. Goes for both kind of crit sects. */ return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED, MMHyperCCToR3(pVM, pThis)); } return rcBusy; #endif /* !IN_RING3 && !IN_RING0 */ } ASMNopPause(); if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) { /* likely */ } else return VERR_SEM_DESTROYED; ASMNopPause(); u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); u64OldState = u64State; } /* not reached */ } /** * Enter a critical section with shared (read) access. * * @returns VBox status code. * @retval VINF_SUCCESS on success. * @retval rcBusy if in ring-0 or raw-mode context and it is busy. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param rcBusy The status code to return when we're in RC or R0 and the * section is busy. Pass VINF_SUCCESS to acquired the * critical section thru a ring-3 call if necessary. * @sa PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterShared, * PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared, * RTCritSectRwEnterShared. */ VMMDECL(int) PDMCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy) { #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/); #endif } /** * Enter a critical section with shared (read) access. * * @returns VBox status code. * @retval VINF_SUCCESS on success. * @retval rcBusy if in ring-0 or raw-mode context and it is busy. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param rcBusy The status code to return when we're in RC or R0 and the * section is busy. Pass VINF_SUCCESS to acquired the * critical section thru a ring-3 call if necessary. * @param uId Where we're entering the section. * @param SRC_POS The source position. * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared, * PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared, * RTCritSectRwEnterSharedDebug. */ VMMDECL(int) PDMCritSectRwEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL) { NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/); #endif } /** * Try enter a critical section with shared (read) access. * * @returns VBox status code. * @retval VINF_SUCCESS on success. * @retval VERR_SEM_BUSY if the critsect was owned. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwEnterShared, * PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared, * RTCritSectRwTryEnterShared. */ VMMDECL(int) PDMCritSectRwTryEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis) { #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/); #endif } /** * Try enter a critical section with shared (read) access. * * @returns VBox status code. * @retval VINF_SUCCESS on success. * @retval VERR_SEM_BUSY if the critsect was owned. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param uId Where we're entering the section. * @param SRC_POS The source position. * @sa PDMCritSectRwTryEnterShared, PDMCritSectRwEnterShared, * PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared, * RTCritSectRwTryEnterSharedDebug. */ VMMDECL(int) PDMCritSectRwTryEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) { NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/); #endif } #ifdef IN_RING3 /** * Enters a PDM read/write critical section with shared (read) access. * * @returns VINF_SUCCESS if entered successfully. * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param fCallRing3 Whether this is a VMMRZCallRing3()request. */ VMMR3DECL(int) PDMR3CritSectRwEnterSharedEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3) { return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3); } #endif /** * Leave a critical section held with shared access. * * @returns VBox status code. * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param fNoVal No validation records (i.e. queued release). * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared, * PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug, * PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared. */ static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) NOREF(fNoVal); #endif /* * Check the direction and take action accordingly. */ #ifdef IN_RING0 PVMCPUCC pVCpu = NULL; #endif uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); uint64_t u64OldState = u64State; if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) { #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (fNoVal) Assert(!RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD)); else { int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->s.Core.pValidatorRead, NIL_RTTHREAD); if (RT_FAILURE(rc9)) return rc9; } #endif for (;;) { uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; AssertReturn(c > 0, VERR_NOT_OWNER); c--; if ( c > 0 || (u64State & RTCSRW_CNT_WR_MASK) == 0) { /* Don't change the direction. */ u64State &= ~RTCSRW_CNT_RD_MASK; u64State |= c << RTCSRW_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) break; } else { #if defined(IN_RING3) || defined(IN_RING0) # ifdef IN_RING0 Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe()); if (!pVCpu) pVCpu = VMMGetCpu(pVM); if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */ || VMMRZCallRing3IsEnabled(pVCpu) || RTSemEventIsSignalSafe() || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */ && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */ && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */ ) # endif { /* Reverse the direction and signal the writer threads. */ u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK); u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { int rc; # ifdef IN_RING0 STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared); if (!RTSemEventIsSignalSafe() && pVCpu != NULL) { VMMR0EMTBLOCKCTX Ctx; rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx); VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); } else # endif rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); AssertRC(rc); return rc; } } #endif /* IN_RING3 || IN_RING0 */ #ifndef IN_RING3 # ifdef IN_RING0 else # endif { /* Queue the exit request (ring-3). */ # ifndef IN_RING0 PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); # endif uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves++; LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3 c=%d (%#llx)\n", i, pThis, c, u64State)); VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves), ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE); pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] = pThis->s.pSelfR3; VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i]) && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] & HOST_PAGE_OFFSET_MASK) == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK), ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i], pThis), pdmCritSectRwCorrupted(pThis, "Invalid self pointer")); VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT); VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves); STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared); break; } #endif } ASMNopPause(); if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) { } else return VERR_SEM_DESTROYED; ASMNopPause(); u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); u64OldState = u64State; } } else { /* * Write direction. Check that it's the owner calling and that it has reads to undo. */ RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER); AssertReturn(pThis->s.Core.cWriterReads > 0, VERR_NOT_OWNER); #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (!fNoVal) { int rc = RTLockValidatorRecExclUnwindMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core); if (RT_FAILURE(rc)) return rc; } #endif uint32_t cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriterReads); AssertReturn(cDepth < PDM_CRITSECTRW_MAX_RECURSIONS, pdmCritSectRwCorrupted(pThis, "too many writer-read recursions")); } return VINF_SUCCESS; } /** * Leave a critical section held with shared access. * * @returns VBox status code. * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared, * PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug, * PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared. */ VMMDECL(int) PDMCritSectRwLeaveShared(PVMCC pVM, PPDMCRITSECTRW pThis) { return pdmCritSectRwLeaveSharedWorker(pVM, pThis, false /*fNoVal*/); } #if defined(IN_RING3) || defined(IN_RING0) /** * PDMCritSectBothFF interface. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. */ void pdmCritSectRwLeaveSharedQueued(PVMCC pVM, PPDMCRITSECTRW pThis) { pdmCritSectRwLeaveSharedWorker(pVM, pThis, true /*fNoVal*/); } #endif /** * Worker for pdmCritSectRwEnterExcl that bails out on wait failure. * * @returns @a rc unless corrupted. * @param pThis Pointer to the read/write critical section. * @param rc The status to return. */ DECL_NO_INLINE(static, int) pdmCritSectRwEnterExclBailOut(PPDMCRITSECTRW pThis, int rc) { /* * Decrement the counts and return the error. */ for (;;) { uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); uint64_t const u64OldState = u64State; uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on bailout")); c--; u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) return rc; ASMNopPause(); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); ASMNopPause(); } } /** * Worker for pdmCritSectRwEnterExcl that handles the red tape after we've * gotten exclusive ownership of the critical section. */ DECL_FORCE_INLINE(int) pdmCritSectRwEnterExclFirst(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) { RT_NOREF(hThreadSelf, fNoVal, pSrcPos); Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)); #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF pThis->s.Core.cWriteRecursions = 1; #else ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 1); #endif Assert(pThis->s.Core.cWriterReads == 0); #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (!fNoVal) { if (hThreadSelf == NIL_RTTHREAD) hThreadSelf = RTThreadSelfAutoAdopt(); RTLockValidatorRecExclSetOwner(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true); } #endif STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl)); STAM_PROFILE_ADV_START(&pThis->s.StatWriteLocked, swl); return VINF_SUCCESS; } #if defined(IN_RING3) || defined(IN_RING0) /** * Worker for pdmCritSectRwEnterExcl that handles waiting when the section is * contended. */ static int pdmR3R0CritSectRwEnterExclContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, int rcBusy, RTTHREAD hThreadSelf) { RT_NOREF(hThreadSelf, rcBusy, pSrcPos, fNoVal, pVCpu); PSUPDRVSESSION const pSession = pVM->pSession; SUPSEMEVENT const hEvent = (SUPSEMEVENT)pThis->s.Core.hEvtWrite; # ifdef IN_RING0 uint64_t const tsStart = RTTimeNanoTS(); uint64_t const cNsMaxTotalDef = RT_NS_5MIN; uint64_t cNsMaxTotal = cNsMaxTotalDef; uint32_t cMsMaxOne = RT_MS_5SEC; bool fNonInterruptible = false; # endif for (uint32_t iLoop = 0; ; iLoop++) { /* * Wait for our turn. */ int rc; # ifdef IN_RING3 # ifdef PDMCRITSECTRW_STRICT rc = RTLockValidatorRecExclCheckBlocking(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true, RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false); if (RT_SUCCESS(rc)) { /* likely */ } else return pdmCritSectRwEnterExclBailOut(pThis, rc); # else RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false); # endif # endif for (;;) { /* * We always wait with a timeout so we can re-check the structure sanity * and not get stuck waiting on a corrupt or deleted section. */ # ifdef IN_RING3 rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC); # else rc = !fNonInterruptible ? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne) : SUPSemEventWait(pSession, hEvent, cMsMaxOne); Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p)\n", pThis, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter)); # endif if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) { /* likely */ } else { # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); # endif return VERR_SEM_DESTROYED; } if (RT_LIKELY(rc == VINF_SUCCESS)) break; /* * Timeout and interrupted waits needs careful handling in ring-0 * because we're cooperating with ring-3 on this critical section * and thus need to make absolutely sure we won't get stuck here. * * The r0 interrupted case means something is pending (termination, * signal, APC, debugger, whatever), so we must try our best to * return to the caller and to ring-3 so it can be dealt with. */ if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED) { # ifdef IN_RING0 uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart; int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD); AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING, ("rcTerm=%Rrc\n", rcTerm)); if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef) cNsMaxTotal = RT_NS_1MIN; if (rc == VERR_TIMEOUT) { /* Try return get out of here with a non-VINF_SUCCESS status if the thread is terminating or if the timeout has been exceeded. */ STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrTimeout); if ( rcTerm == VINF_THREAD_IS_TERMINATING || cNsElapsed > cNsMaxTotal) return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc); } else { /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS, we will try non-interruptible sleep for a while to help resolve the issue w/o guru'ing. */ STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrInterrupted); if ( rcTerm != VINF_THREAD_IS_TERMINATING && rcBusy == VINF_SUCCESS && pVCpu != NULL && cNsElapsed <= cNsMaxTotal) { if (!fNonInterruptible) { STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclNonInterruptibleWaits); fNonInterruptible = true; cMsMaxOne = 32; uint64_t cNsLeft = cNsMaxTotal - cNsElapsed; if (cNsLeft > RT_NS_10SEC) cNsMaxTotal = cNsElapsed + RT_NS_10SEC; } } else return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc); } # else /* IN_RING3 */ RT_NOREF(pVM, pVCpu, rcBusy); # endif /* IN_RING3 */ } /* * Any other return code is fatal. */ else { # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); # endif AssertMsgFailed(("rc=%Rrc\n", rc)); return RT_FAILURE_NP(rc) ? rc : -rc; } } # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); # endif /* * Try take exclusive write ownership. */ uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) { bool fDone; ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone); if (fDone) return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf); } AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */ } } #endif /* IN_RING3 || IN_RING0 */ /** * Worker that enters a read/write critical section with exclusive access. * * @returns VBox status code. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param rcBusy The busy return code for ring-0 and ring-3. * @param fTryOnly Only try enter it, don't wait. * @param pSrcPos The source position. (Can be NULL.) * @param fNoVal No validation records. */ static int pdmCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal) { /* * Validate input. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); RTTHREAD hThreadSelf = NIL_RTTHREAD; #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (!fTryOnly) { hThreadSelf = RTThreadSelfAutoAdopt(); int rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc9)) return rc9; } #endif /* * Check if we're already the owner and just recursing. */ RTNATIVETHREAD const hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); if (hNativeSelf == hNativeWriter) { Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)); #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (!fNoVal) { int rc9 = RTLockValidatorRecExclRecursion(pThis->s.Core.pValidatorWrite, pSrcPos); if (RT_FAILURE(rc9)) return rc9; } #endif STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl)); #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF uint32_t const cDepth = ++pThis->s.Core.cWriteRecursions; #else uint32_t const cDepth = ASMAtomicIncU32(&pThis->s.Core.cWriteRecursions); #endif AssertReturnStmt(cDepth > 1 && cDepth <= PDM_CRITSECTRW_MAX_RECURSIONS, ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions), VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS); return VINF_SUCCESS; } /* * First we try grab an idle critical section using 128-bit atomics. */ /** @todo This could be moved up before the recursion check. */ uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); #ifdef RTASM_HAVE_CMP_WRITE_U128 if ( (u64State & ~RTCSRW_DIR_MASK) == 0 && pdmCritSectRwIsCmpWriteU128Supported()) { RTCRITSECTRWSTATE OldState; OldState.s.u64State = u64State; OldState.s.hNativeWriter = NIL_RTNATIVETHREAD; AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo)); RTCRITSECTRWSTATE NewState; NewState.s.u64State = (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT); NewState.s.hNativeWriter = hNativeSelf; if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128)) return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf); u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); } #endif /* * Do it step by step. Update the state to reflect our desire. */ uint64_t u64OldState = u64State; for (;;) { if ( (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT) || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0) { /* It flows in the right direction, try follow it before it changes. */ uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS); c++; Assert(c < RTCSRW_CNT_WR_MASK / 4); u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) break; } else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0) { /* Wrong direction, but we're alone here and can simply try switch the direction. */ u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) break; } else if (fTryOnly) { /* Wrong direction and we're not supposed to wait, just return. */ STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); return VERR_SEM_BUSY; } else { /* Add ourselves to the write count and break out to do the wait. */ uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS); c++; Assert(c < RTCSRW_CNT_WR_MASK / 4); u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) break; } ASMNopPause(); if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC) { /* likely */ } else return VERR_SEM_DESTROYED; ASMNopPause(); u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); u64OldState = u64State; } /* * If we're in write mode now try grab the ownership. Play fair if there * are threads already waiting. */ bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT) && ( ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1 || fTryOnly); if (fDone) { ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone); if (fDone) return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf); } /* * Okay, we have contention and will have to wait unless we're just trying. */ if (fTryOnly) { STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); /** @todo different statistics for this */ return pdmCritSectRwEnterExclBailOut(pThis, VERR_SEM_BUSY); } STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); /* * Ring-3 is pretty straight forward. */ #if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT) return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, hThreadSelf); #elif defined(IN_RING3) return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, RTThreadSelf()); #elif defined(IN_RING0) /* * In ring-0 context we have to take the special VT-x/AMD-V HM context into * account when waiting on contended locks. */ PVMCPUCC pVCpu = VMMGetCpu(pVM); if (pVCpu) { VMMR0EMTBLOCKCTX Ctx; int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx); if (rc == VINF_SUCCESS) { Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); rc = pdmR3R0CritSectRwEnterExclContended(pVM, pVCpu, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD); VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); } else { //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy); rc = pdmCritSectRwEnterExclBailOut(pThis, rc); } return rc; } /* Non-EMT. */ Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD); #else # error "Unused." /* * Raw-mode: Call host and take it there if rcBusy is VINF_SUCCESS. */ rcBusy = pdmCritSectRwEnterExclBailOut(pThis, rcBusy); if (rcBusy == VINF_SUCCESS) { Assert(!fTryOnly); PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); /** @todo Should actually do this in via VMMR0.cpp instead of going all the way * back to ring-3. Goes for both kind of crit sects. */ return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL, MMHyperCCToR3(pVM, pThis)); } return rcBusy; #endif } /** * Try enter a critical section with exclusive (write) access. * * @returns VBox status code. * @retval VINF_SUCCESS on success. * @retval rcBusy if in ring-0 or raw-mode context and it is busy. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param rcBusy The status code to return when we're in RC or R0 and the * section is busy. Pass VINF_SUCCESS to acquired the * critical section thru a ring-3 call if necessary. * @sa PDMCritSectRwEnterExclDebug, PDMCritSectRwTryEnterExcl, * PDMCritSectRwTryEnterExclDebug, * PDMCritSectEnterDebug, PDMCritSectEnter, * RTCritSectRwEnterExcl. */ VMMDECL(int) PDMCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy) { #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/); #endif } /** * Try enter a critical section with exclusive (write) access. * * @returns VBox status code. * @retval VINF_SUCCESS on success. * @retval rcBusy if in ring-0 or raw-mode context and it is busy. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param rcBusy The status code to return when we're in RC or R0 and the * section is busy. Pass VINF_SUCCESS to acquired the * critical section thru a ring-3 call if necessary. * @param uId Where we're entering the section. * @param SRC_POS The source position. * @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExcl, * PDMCritSectRwTryEnterExclDebug, * PDMCritSectEnterDebug, PDMCritSectEnter, * RTCritSectRwEnterExclDebug. */ VMMDECL(int) PDMCritSectRwEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL) { NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/); #endif } /** * Try enter a critical section with exclusive (write) access. * * @retval VINF_SUCCESS on success. * @retval VERR_SEM_BUSY if the critsect was owned. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExclDebug, * PDMCritSectRwEnterExclDebug, * PDMCritSectTryEnter, PDMCritSectTryEnterDebug, * RTCritSectRwTryEnterExcl. */ VMMDECL(int) PDMCritSectRwTryEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis) { #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/); #endif } /** * Try enter a critical section with exclusive (write) access. * * @retval VINF_SUCCESS on success. * @retval VERR_SEM_BUSY if the critsect was owned. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param uId Where we're entering the section. * @param SRC_POS The source position. * @sa PDMCritSectRwTryEnterExcl, PDMCritSectRwEnterExcl, * PDMCritSectRwEnterExclDebug, * PDMCritSectTryEnterDebug, PDMCritSectTryEnter, * RTCritSectRwTryEnterExclDebug. */ VMMDECL(int) PDMCritSectRwTryEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) { NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL, false /*fNoVal*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/); #endif } #ifdef IN_RING3 /** * Enters a PDM read/write critical section with exclusive (write) access. * * @returns VINF_SUCCESS if entered successfully. * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param fCallRing3 Whether this is a VMMRZCallRing3()request. */ VMMR3DECL(int) PDMR3CritSectRwEnterExclEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3) { return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3 /*fNoVal*/); } #endif /* IN_RING3 */ /** * Leave a critical section held exclusively. * * @returns VBox status code. * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param fNoVal No validation records (i.e. queued release). * @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl. */ static int pdmCritSectRwLeaveExclWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) NOREF(fNoVal); #endif /* * Check ownership. */ RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER); /* * Unwind one recursion. Not the last? */ if (pThis->s.Core.cWriteRecursions != 1) { #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (fNoVal) Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD); else { int rc9 = RTLockValidatorRecExclUnwind(pThis->s.Core.pValidatorWrite); if (RT_FAILURE(rc9)) return rc9; } #endif #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF uint32_t const cDepth = --pThis->s.Core.cWriteRecursions; #else uint32_t const cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions); #endif AssertReturn(cDepth != 0 && cDepth < UINT32_MAX, pdmCritSectRwCorrupted(pThis, "Invalid write recursion value on leave")); return VINF_SUCCESS; } /* * Final recursion. */ AssertReturn(pThis->s.Core.cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */ #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) if (fNoVal) Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD); else { int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->s.Core.pValidatorWrite, true); if (RT_FAILURE(rc9)) return rc9; } #endif #ifdef RTASM_HAVE_CMP_WRITE_U128 /* * See if we can get out w/o any signalling as this is a common case. */ if (pdmCritSectRwIsCmpWriteU128Supported()) { RTCRITSECTRWSTATE OldState; OldState.s.u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); if (OldState.s.u64State == ((UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))) { OldState.s.hNativeWriter = hNativeSelf; AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo)); RTCRITSECTRWSTATE NewState; NewState.s.u64State = RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT; NewState.s.hNativeWriter = NIL_RTNATIVETHREAD; # ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF pThis->s.Core.cWriteRecursions = 0; # else ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0); # endif STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl); if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128)) return VINF_SUCCESS; /* bail out. */ pThis->s.Core.cWriteRecursions = 1; } } #endif /* RTASM_HAVE_CMP_WRITE_U128 */ #if defined(IN_RING3) || defined(IN_RING0) /* * Ring-3: Straight forward, just update the state and if necessary signal waiters. * Ring-0: Try leave for real, depends on host and context. */ # ifdef IN_RING0 Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe()); PVMCPUCC pVCpu = VMMGetCpu(pVM); if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */ || VMMRZCallRing3IsEnabled(pVCpu) || RTSemEventIsSignalSafe() || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */ && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */ && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */ ) # endif { # ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF pThis->s.Core.cWriteRecursions = 0; # else ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0); # endif STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl); ASMAtomicWriteHandle(&pThis->s.Core.u.s.hNativeWriter, NIL_RTNATIVETHREAD); for (;;) { uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); uint64_t u64OldState = u64State; uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on leave")); c--; if ( c > 0 || (u64State & RTCSRW_CNT_RD_MASK) == 0) { /* * Don't change the direction, wake up the next writer if any. */ u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl)); int rc; if (c == 0) rc = VINF_SUCCESS; # ifdef IN_RING0 else if (!RTSemEventIsSignalSafe() && pVCpu != NULL) { VMMR0EMTBLOCKCTX Ctx; rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx); VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); } # endif else rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); AssertRC(rc); return rc; } } else { /* * Reverse the direction and signal the reader threads. */ u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) { Assert(!pThis->s.Core.fNeedReset); ASMAtomicWriteBool(&pThis->s.Core.fNeedReset, true); STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl)); int rc; # ifdef IN_RING0 if (!RTSemEventMultiIsSignalSafe() && pVCpu != NULL) { VMMR0EMTBLOCKCTX Ctx; rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx); VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead); VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); } else # endif rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead); AssertRC(rc); return rc; } } ASMNopPause(); if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC) { /*likely*/ } else return VERR_SEM_DESTROYED; ASMNopPause(); } /* not reached! */ } #endif /* IN_RING3 || IN_RING0 */ #ifndef IN_RING3 /* * Queue the requested exit for ring-3 execution. */ # ifndef IN_RING0 PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); # endif uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves++; LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3\n", i, pThis)); VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves), ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE); pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] = pThis->s.pSelfR3; VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i]) && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] & HOST_PAGE_OFFSET_MASK) == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK), ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i], pThis), pdmCritSectRwCorrupted(pThis, "Invalid self pointer on queue (excl)")); VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT); VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves); STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveExcl); return VINF_SUCCESS; #endif } /** * Leave a critical section held exclusively. * * @returns VBox status code. * @retval VERR_SEM_DESTROYED if the critical section is delete before or * during the operation. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl. */ VMMDECL(int) PDMCritSectRwLeaveExcl(PVMCC pVM, PPDMCRITSECTRW pThis) { return pdmCritSectRwLeaveExclWorker(pVM, pThis, false /*fNoVal*/); } #if defined(IN_RING3) || defined(IN_RING0) /** * PDMCritSectBothFF interface. * * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. */ void pdmCritSectRwLeaveExclQueued(PVMCC pVM, PPDMCRITSECTRW pThis) { pdmCritSectRwLeaveExclWorker(pVM, pThis, true /*fNoVal*/); } #endif /** * Checks the caller is the exclusive (write) owner of the critical section. * * @retval true if owner. * @retval false if not owner. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwIsReadOwner, PDMCritSectIsOwner, * RTCritSectRwIsWriteOwner. */ VMMDECL(bool) PDMCritSectRwIsWriteOwner(PVMCC pVM, PPDMCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false); /* * Check ownership. */ RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); if (hNativeWriter == NIL_RTNATIVETHREAD) return false; return hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis); } /** * Checks if the caller is one of the read owners of the critical section. * * @note !CAUTION! This API doesn't work reliably if lock validation isn't * enabled. Meaning, the answer is not trustworhty unless * RT_LOCK_STRICT or PDMCRITSECTRW_STRICT was defined at build time. * Also, make sure you do not use RTCRITSECTRW_FLAGS_NO_LOCK_VAL when * creating the semaphore. And finally, if you used a locking class, * don't disable deadlock detection by setting cMsMinDeadlock to * RT_INDEFINITE_WAIT. * * In short, only use this for assertions. * * @returns @c true if reader, @c false if not. * @param pVM The cross context VM structure. * @param pThis Pointer to the read/write critical section. * @param fWannaHear What you'd like to hear when lock validation is not * available. (For avoiding asserting all over the place.) * @sa PDMCritSectRwIsWriteOwner, RTCritSectRwIsReadOwner. */ VMMDECL(bool) PDMCritSectRwIsReadOwner(PVMCC pVM, PPDMCRITSECTRW pThis, bool fWannaHear) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false); /* * Inspect the state. */ uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) { /* * It's in write mode, so we can only be a reader if we're also the * current writer. */ RTNATIVETHREAD hWriter; ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hWriter); if (hWriter == NIL_RTNATIVETHREAD) return false; return hWriter == pdmCritSectRwGetNativeSelf(pVM, pThis); } /* * Read mode. If there are no current readers, then we cannot be a reader. */ if (!(u64State & RTCSRW_CNT_RD_MASK)) return false; #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) /* * Ask the lock validator. * Note! It doesn't know everything, let's deal with that if it becomes an issue... */ NOREF(fWannaHear); return RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD); #else /* * Ok, we don't know, just tell the caller what he want to hear. */ return fWannaHear; #endif } /** * Gets the write recursion count. * * @returns The write recursion count (0 if bad critsect). * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwGetWriterReadRecursion, PDMCritSectRwGetReadCount, * RTCritSectRwGetWriteRecursion. */ VMMDECL(uint32_t) PDMCritSectRwGetWriteRecursion(PPDMCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0); /* * Return the requested data. */ return pThis->s.Core.cWriteRecursions; } /** * Gets the read recursion count of the current writer. * * @returns The read recursion count (0 if bad critsect). * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetReadCount, * RTCritSectRwGetWriterReadRecursion. */ VMMDECL(uint32_t) PDMCritSectRwGetWriterReadRecursion(PPDMCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0); /* * Return the requested data. */ return pThis->s.Core.cWriterReads; } /** * Gets the current number of reads. * * This includes all read recursions, so it might be higher than the number of * read owners. It does not include reads done by the current writer. * * @returns The read count (0 if bad critsect). * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetWriterReadRecursion, * RTCritSectRwGetReadCount. */ VMMDECL(uint32_t) PDMCritSectRwGetReadCount(PPDMCRITSECTRW pThis) { /* * Validate input. */ AssertPtr(pThis); AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0); /* * Return the requested data. */ uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) return 0; return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; } /** * Checks if the read/write critical section is initialized or not. * * @retval true if initialized. * @retval false if not initialized. * @param pThis Pointer to the read/write critical section. * @sa PDMCritSectIsInitialized, RTCritSectRwIsInitialized. */ VMMDECL(bool) PDMCritSectRwIsInitialized(PCPDMCRITSECTRW pThis) { AssertPtr(pThis); return pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC; }