[25426] | 1 | /* $Id: PDMAllCritSectRw.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
|
---|
| 2 | /** @file
|
---|
[45110] | 3 | * IPRT - Read/Write Critical Section, Generic.
|
---|
[25426] | 4 | */
|
---|
| 5 |
|
---|
| 6 | /*
|
---|
[98103] | 7 | * Copyright (C) 2009-2023 Oracle and/or its affiliates.
|
---|
[25426] | 8 | *
|
---|
[96407] | 9 | * This file is part of VirtualBox base platform packages, as
|
---|
| 10 | * available from https://www.virtualbox.org.
|
---|
| 11 | *
|
---|
| 12 | * This program is free software; you can redistribute it and/or
|
---|
| 13 | * modify it under the terms of the GNU General Public License
|
---|
| 14 | * as published by the Free Software Foundation, in version 3 of the
|
---|
| 15 | * License.
|
---|
| 16 | *
|
---|
| 17 | * This program is distributed in the hope that it will be useful, but
|
---|
| 18 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
---|
| 20 | * General Public License for more details.
|
---|
| 21 | *
|
---|
| 22 | * You should have received a copy of the GNU General Public License
|
---|
| 23 | * along with this program; if not, see <https://www.gnu.org/licenses>.
|
---|
| 24 | *
|
---|
| 25 | * SPDX-License-Identifier: GPL-3.0-only
|
---|
[25426] | 26 | */
|
---|
| 27 |
|
---|
| 28 |
|
---|
[57358] | 29 | /*********************************************************************************************************************************
|
---|
| 30 | * Header Files *
|
---|
| 31 | *********************************************************************************************************************************/
|
---|
[90677] | 32 | #define LOG_GROUP LOG_GROUP_PDM_CRITSECTRW
|
---|
[45152] | 33 | #include "PDMInternal.h"
|
---|
| 34 | #include <VBox/vmm/pdmcritsectrw.h>
|
---|
| 35 | #include <VBox/vmm/mm.h>
|
---|
| 36 | #include <VBox/vmm/vmm.h>
|
---|
[80268] | 37 | #include <VBox/vmm/vmcc.h>
|
---|
[45152] | 38 | #include <VBox/err.h>
|
---|
| 39 | #include <VBox/vmm/hm.h>
|
---|
[25426] | 40 |
|
---|
[45152] | 41 | #include <VBox/log.h>
|
---|
[25426] | 42 | #include <iprt/asm.h>
|
---|
| 43 | #include <iprt/assert.h>
|
---|
[45152] | 44 | #ifdef IN_RING3
|
---|
| 45 | # include <iprt/lockvalidator.h>
|
---|
| 46 | #endif
|
---|
| 47 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
[90659] | 48 | # include <iprt/semaphore.h>
|
---|
[45152] | 49 | # include <iprt/thread.h>
|
---|
| 50 | #endif
|
---|
[90634] | 51 | #ifdef IN_RING0
|
---|
| 52 | # include <iprt/time.h>
|
---|
| 53 | #endif
|
---|
[90650] | 54 | #ifdef RT_ARCH_AMD64
|
---|
| 55 | # include <iprt/x86.h>
|
---|
| 56 | #endif
|
---|
[25426] | 57 |
|
---|
| 58 |
|
---|
[57358] | 59 | /*********************************************************************************************************************************
|
---|
| 60 | * Defined Constants And Macros *
|
---|
| 61 | *********************************************************************************************************************************/
|
---|
[90673] | 62 | #if 0 /* unused */
|
---|
[45152] | 63 | /** The number loops to spin for shared access in ring-3. */
|
---|
[90486] | 64 | #define PDMCRITSECTRW_SHRD_SPIN_COUNT_R3 20
|
---|
[45152] | 65 | /** The number loops to spin for shared access in ring-0. */
|
---|
[90486] | 66 | #define PDMCRITSECTRW_SHRD_SPIN_COUNT_R0 128
|
---|
[45152] | 67 | /** The number loops to spin for shared access in the raw-mode context. */
|
---|
[90486] | 68 | #define PDMCRITSECTRW_SHRD_SPIN_COUNT_RC 128
|
---|
[25431] | 69 |
|
---|
[45152] | 70 | /** The number loops to spin for exclusive access in ring-3. */
|
---|
[90486] | 71 | #define PDMCRITSECTRW_EXCL_SPIN_COUNT_R3 20
|
---|
[45152] | 72 | /** The number loops to spin for exclusive access in ring-0. */
|
---|
[90486] | 73 | #define PDMCRITSECTRW_EXCL_SPIN_COUNT_R0 256
|
---|
[45152] | 74 | /** The number loops to spin for exclusive access in the raw-mode context. */
|
---|
[90486] | 75 | #define PDMCRITSECTRW_EXCL_SPIN_COUNT_RC 256
|
---|
[90673] | 76 | #endif
|
---|
[25431] | 77 |
|
---|
[90486] | 78 | /** Max number of write or write/read recursions. */
|
---|
| 79 | #define PDM_CRITSECTRW_MAX_RECURSIONS _1M
|
---|
[25549] | 80 |
|
---|
[90654] | 81 | /** Skips some of the overly paranoid atomic reads and updates.
|
---|
| 82 | * Makes some assumptions about cache coherence, though not brave enough not to
|
---|
| 83 | * always end with an atomic update. */
|
---|
| 84 | #define PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
| 85 |
|
---|
| 86 | /** For reading RTCRITSECTRWSTATE::s::u64State. */
|
---|
| 87 | #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
| 88 | # define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicUoReadU64(a_pu64State)
|
---|
| 89 | #else
|
---|
| 90 | # define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicReadU64(a_pu64State)
|
---|
| 91 | #endif
|
---|
| 92 |
|
---|
| 93 |
|
---|
[45152] | 94 | /* Undefine the automatic VBOX_STRICT API mappings. */
|
---|
| 95 | #undef PDMCritSectRwEnterExcl
|
---|
| 96 | #undef PDMCritSectRwTryEnterExcl
|
---|
| 97 | #undef PDMCritSectRwEnterShared
|
---|
| 98 | #undef PDMCritSectRwTryEnterShared
|
---|
[25549] | 99 |
|
---|
[25707] | 100 |
|
---|
[90650] | 101 | /*********************************************************************************************************************************
|
---|
| 102 | * Defined Constants And Macros *
|
---|
| 103 | *********************************************************************************************************************************/
|
---|
| 104 | #if defined(RTASM_HAVE_CMP_WRITE_U128) && defined(RT_ARCH_AMD64)
|
---|
| 105 | static int32_t g_fCmpWriteSupported = -1;
|
---|
| 106 | #endif
|
---|
| 107 |
|
---|
| 108 |
|
---|
[90677] | 109 | /*********************************************************************************************************************************
|
---|
| 110 | * Internal Functions *
|
---|
| 111 | *********************************************************************************************************************************/
|
---|
| 112 | static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal);
|
---|
| 113 |
|
---|
| 114 |
|
---|
[90650] | 115 | #ifdef RTASM_HAVE_CMP_WRITE_U128
|
---|
| 116 |
|
---|
| 117 | # ifdef RT_ARCH_AMD64
|
---|
[45152] | 118 | /**
|
---|
[90650] | 119 | * Called once to initialize g_fCmpWriteSupported.
|
---|
| 120 | */
|
---|
| 121 | DECL_NO_INLINE(static, bool) pdmCritSectRwIsCmpWriteU128SupportedSlow(void)
|
---|
| 122 | {
|
---|
| 123 | bool const fCmpWriteSupported = RT_BOOL(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16);
|
---|
| 124 | ASMAtomicWriteS32(&g_fCmpWriteSupported, fCmpWriteSupported);
|
---|
| 125 | return fCmpWriteSupported;
|
---|
| 126 | }
|
---|
| 127 | # endif
|
---|
| 128 |
|
---|
| 129 |
|
---|
| 130 | /**
|
---|
| 131 | * Indicates whether hardware actually supports 128-bit compare & write.
|
---|
| 132 | */
|
---|
| 133 | DECL_FORCE_INLINE(bool) pdmCritSectRwIsCmpWriteU128Supported(void)
|
---|
| 134 | {
|
---|
| 135 | # ifdef RT_ARCH_AMD64
|
---|
| 136 | int32_t const fCmpWriteSupported = g_fCmpWriteSupported;
|
---|
| 137 | if (RT_LIKELY(fCmpWriteSupported >= 0))
|
---|
| 138 | return fCmpWriteSupported != 0;
|
---|
| 139 | return pdmCritSectRwIsCmpWriteU128SupportedSlow();
|
---|
| 140 | # else
|
---|
| 141 | return true;
|
---|
| 142 | # endif
|
---|
| 143 | }
|
---|
| 144 |
|
---|
| 145 | #endif /* RTASM_HAVE_CMP_WRITE_U128 */
|
---|
| 146 |
|
---|
| 147 | /**
|
---|
[45152] | 148 | * Gets the ring-3 native thread handle of the calling thread.
|
---|
| 149 | *
|
---|
| 150 | * @returns native thread handle (ring-3).
|
---|
[90385] | 151 | * @param pVM The cross context VM structure.
|
---|
[45152] | 152 | * @param pThis The read/write critical section. This is only used in
|
---|
| 153 | * R0 and RC.
|
---|
| 154 | */
|
---|
[90347] | 155 | DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectRwGetNativeSelf(PVMCC pVM, PCPDMCRITSECTRW pThis)
|
---|
[25707] | 156 | {
|
---|
[45152] | 157 | #ifdef IN_RING3
|
---|
[90347] | 158 | RT_NOREF(pVM, pThis);
|
---|
[45152] | 159 | RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
|
---|
[92204] | 160 |
|
---|
| 161 | #elif defined(IN_RING0)
|
---|
[45299] | 162 | AssertMsgReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, ("%RX32\n", pThis->s.Core.u32Magic),
|
---|
[45152] | 163 | NIL_RTNATIVETHREAD);
|
---|
[92204] | 164 | RTNATIVETHREAD hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM);
|
---|
[90347] | 165 | Assert(hNativeSelf != NIL_RTNATIVETHREAD);
|
---|
[92204] | 166 |
|
---|
| 167 | #else
|
---|
| 168 | # error "invalid context"
|
---|
[45110] | 169 | #endif
|
---|
[45152] | 170 | return hNativeSelf;
|
---|
| 171 | }
|
---|
[25431] | 172 |
|
---|
[45110] | 173 |
|
---|
[90573] | 174 | DECL_NO_INLINE(static, int) pdmCritSectRwCorrupted(PPDMCRITSECTRW pThis, const char *pszMsg)
|
---|
[90486] | 175 | {
|
---|
| 176 | ASMAtomicWriteU32(&pThis->s.Core.u32Magic, PDMCRITSECTRW_MAGIC_CORRUPT);
|
---|
[90573] | 177 | LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pThis));
|
---|
[90486] | 178 | return VERR_PDM_CRITSECTRW_IPE;
|
---|
| 179 | }
|
---|
[25426] | 180 |
|
---|
| 181 |
|
---|
| 182 |
|
---|
[45152] | 183 | #ifdef IN_RING3
|
---|
| 184 | /**
|
---|
| 185 | * Changes the lock validator sub-class of the read/write critical section.
|
---|
| 186 | *
|
---|
| 187 | * It is recommended to try make sure that nobody is using this critical section
|
---|
| 188 | * while changing the value.
|
---|
| 189 | *
|
---|
| 190 | * @returns The old sub-class. RTLOCKVAL_SUB_CLASS_INVALID is returns if the
|
---|
| 191 | * lock validator isn't compiled in or either of the parameters are
|
---|
| 192 | * invalid.
|
---|
| 193 | * @param pThis Pointer to the read/write critical section.
|
---|
| 194 | * @param uSubClass The new sub-class value.
|
---|
| 195 | */
|
---|
| 196 | VMMDECL(uint32_t) PDMR3CritSectRwSetSubClass(PPDMCRITSECTRW pThis, uint32_t uSubClass)
|
---|
[25707] | 197 | {
|
---|
| 198 | AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
|
---|
[45152] | 199 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
|
---|
[48199] | 200 | # if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[45152] | 201 | AssertReturn(!(pThis->s.Core.fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID);
|
---|
[25707] | 202 |
|
---|
[45152] | 203 | RTLockValidatorRecSharedSetSubClass(pThis->s.Core.pValidatorRead, uSubClass);
|
---|
| 204 | return RTLockValidatorRecExclSetSubClass(pThis->s.Core.pValidatorWrite, uSubClass);
|
---|
| 205 | # else
|
---|
[45110] | 206 | NOREF(uSubClass);
|
---|
[25707] | 207 | return RTLOCKVAL_SUB_CLASS_INVALID;
|
---|
[45152] | 208 | # endif
|
---|
[25707] | 209 | }
|
---|
[45152] | 210 | #endif /* IN_RING3 */
|
---|
[25707] | 211 |
|
---|
| 212 |
|
---|
[45299] | 213 | /**
|
---|
[90677] | 214 | * Worker for pdmCritSectRwEnterShared returning with read-ownership of the CS.
|
---|
[50001] | 215 | */
|
---|
[90677] | 216 | DECL_FORCE_INLINE(int) pdmCritSectRwEnterSharedGotIt(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos,
|
---|
| 217 | bool fNoVal, RTTHREAD hThreadSelf)
|
---|
[50001] | 218 | {
|
---|
[90677] | 219 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
| 220 | if (!fNoVal)
|
---|
| 221 | RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos);
|
---|
| 222 | #else
|
---|
| 223 | RT_NOREF(pSrcPos, fNoVal, hThreadSelf);
|
---|
| 224 | #endif
|
---|
| 225 |
|
---|
| 226 | /* got it! */
|
---|
| 227 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
|
---|
| 228 | Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT));
|
---|
| 229 | return VINF_SUCCESS;
|
---|
[50001] | 230 | }
|
---|
| 231 |
|
---|
| 232 | /**
|
---|
[90677] | 233 | * Worker for pdmCritSectRwEnterShared and pdmCritSectRwEnterSharedBailOut
|
---|
| 234 | * that decrement the wait count and maybe resets the semaphore.
|
---|
[90672] | 235 | */
|
---|
[90677] | 236 | DECLINLINE(int) pdmCritSectRwEnterSharedGotItAfterWaiting(PVMCC pVM, PPDMCRITSECTRW pThis, uint64_t u64State,
|
---|
| 237 | PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
|
---|
[90672] | 238 | {
|
---|
| 239 | for (;;)
|
---|
| 240 | {
|
---|
| 241 | uint64_t const u64OldState = u64State;
|
---|
| 242 | uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
|
---|
[90677] | 243 | AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count"));
|
---|
| 244 | AssertReturn((u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT > 0,
|
---|
| 245 | pdmCritSectRwCorrupted(pThis, "Invalid read count"));
|
---|
[90672] | 246 | cWait--;
|
---|
[90677] | 247 | u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
|
---|
| 248 | u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
|
---|
| 249 |
|
---|
[90672] | 250 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[90677] | 251 | {
|
---|
| 252 | if (cWait == 0)
|
---|
| 253 | {
|
---|
| 254 | if (ASMAtomicXchgBool(&pThis->s.Core.fNeedReset, false))
|
---|
| 255 | {
|
---|
| 256 | int rc = SUPSemEventMultiReset(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
|
---|
| 257 | AssertRCReturn(rc, rc);
|
---|
| 258 | }
|
---|
| 259 | }
|
---|
| 260 | return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 261 | }
|
---|
[90672] | 262 |
|
---|
| 263 | ASMNopPause();
|
---|
| 264 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
[90677] | 265 | ASMNopPause();
|
---|
| 266 |
|
---|
| 267 | u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[90672] | 268 | }
|
---|
[90677] | 269 | /* not reached */
|
---|
[90672] | 270 | }
|
---|
| 271 |
|
---|
| 272 |
|
---|
[90677] | 273 | #if defined(IN_RING0) || (defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT))
|
---|
[90672] | 274 | /**
|
---|
[90677] | 275 | * Worker for pdmCritSectRwEnterSharedContended that decrements both read counts
|
---|
| 276 | * and returns @a rc.
|
---|
| 277 | *
|
---|
| 278 | * @note May return VINF_SUCCESS if we race the exclusive leave function and
|
---|
| 279 | * come out on the bottom.
|
---|
| 280 | *
|
---|
| 281 | * Ring-3 only calls in a case where it is _not_ acceptable to take the
|
---|
| 282 | * lock, so even if we get the lock we'll have to leave. In the ring-0
|
---|
| 283 | * contexts, we can safely return VINF_SUCCESS in case of a race.
|
---|
[90670] | 284 | */
|
---|
[90677] | 285 | DECL_NO_INLINE(static, int) pdmCritSectRwEnterSharedBailOut(PVMCC pVM, PPDMCRITSECTRW pThis, int rc,
|
---|
| 286 | PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
|
---|
[90670] | 287 | {
|
---|
[90677] | 288 | #ifdef IN_RING0
|
---|
| 289 | uint64_t const tsStart = RTTimeNanoTS();
|
---|
| 290 | uint64_t cNsElapsed = 0;
|
---|
[90670] | 291 | #endif
|
---|
[90677] | 292 | for (;;)
|
---|
| 293 | {
|
---|
| 294 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
| 295 | uint64_t u64OldState = u64State;
|
---|
[90670] | 296 |
|
---|
[90677] | 297 | uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
|
---|
| 298 | AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count on bailout"));
|
---|
| 299 | cWait--;
|
---|
| 300 |
|
---|
| 301 | uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
|
---|
| 302 | AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid read count on bailout"));
|
---|
| 303 |
|
---|
| 304 | if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
|
---|
| 305 | {
|
---|
| 306 | c--;
|
---|
| 307 | u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
|
---|
| 308 | u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
|
---|
| 309 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
| 310 | return rc;
|
---|
| 311 | }
|
---|
| 312 | else
|
---|
| 313 | {
|
---|
| 314 | /*
|
---|
| 315 | * The direction changed, so we can actually get the lock now.
|
---|
| 316 | *
|
---|
| 317 | * This means that we _have_ to wait on the semaphore to be signalled
|
---|
| 318 | * so we can properly reset it. Otherwise the stuff gets out of wack,
|
---|
| 319 | * because signalling and resetting will race one another. An
|
---|
| 320 | * exception would be if we're not the last reader waiting and don't
|
---|
| 321 | * need to worry about the resetting.
|
---|
| 322 | *
|
---|
| 323 | * An option would be to do the resetting in PDMCritSectRwEnterExcl,
|
---|
| 324 | * but that would still leave a racing PDMCritSectRwEnterShared
|
---|
| 325 | * spinning hard for a little bit, which isn't great...
|
---|
| 326 | */
|
---|
| 327 | if (cWait == 0)
|
---|
| 328 | {
|
---|
| 329 | # ifdef IN_RING0
|
---|
| 330 | /* Do timeout processing first to avoid redoing the above. */
|
---|
| 331 | uint32_t cMsWait;
|
---|
| 332 | if (cNsElapsed <= RT_NS_10SEC)
|
---|
| 333 | cMsWait = 32;
|
---|
| 334 | else
|
---|
| 335 | {
|
---|
| 336 | u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
|
---|
| 337 | u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
|
---|
| 338 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
| 339 | {
|
---|
| 340 | LogFunc(("%p: giving up\n", pThis));
|
---|
| 341 | return rc;
|
---|
| 342 | }
|
---|
| 343 | cMsWait = 2;
|
---|
| 344 | }
|
---|
| 345 |
|
---|
| 346 | int rcWait = SUPSemEventMultiWait(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, cMsWait);
|
---|
| 347 | Log11Func(("%p: rc=%Rrc %'RU64 ns (hNativeWriter=%p u64State=%#RX64)\n", pThis, rcWait,
|
---|
| 348 | RTTimeNanoTS() - tsStart, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State));
|
---|
| 349 | # else
|
---|
| 350 | RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
|
---|
| 351 | int rcWait = SUPSemEventMultiWaitNoResume(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, RT_MS_5SEC);
|
---|
| 352 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
|
---|
| 353 | # endif
|
---|
| 354 | if (rcWait == VINF_SUCCESS)
|
---|
| 355 | {
|
---|
| 356 | # ifdef IN_RING0
|
---|
| 357 | return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 358 | # else
|
---|
| 359 | /* ring-3: Cannot return VINF_SUCCESS. */
|
---|
| 360 | Assert(RT_FAILURE_NP(rc));
|
---|
| 361 | int rc2 = pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 362 | if (RT_SUCCESS(rc2))
|
---|
| 363 | rc2 = pdmCritSectRwLeaveSharedWorker(pVM, pThis, fNoVal);
|
---|
| 364 | return rc;
|
---|
| 365 | # endif
|
---|
| 366 | }
|
---|
| 367 | AssertMsgReturn(rcWait == VERR_TIMEOUT || rcWait == VERR_INTERRUPTED,
|
---|
| 368 | ("%p: rcWait=%Rrc rc=%Rrc", pThis, rcWait, rc),
|
---|
| 369 | RT_FAILURE_NP(rcWait) ? rcWait : -rcWait);
|
---|
| 370 | }
|
---|
| 371 | else
|
---|
| 372 | {
|
---|
| 373 | u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
|
---|
| 374 | u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
|
---|
| 375 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
| 376 | return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 377 | }
|
---|
| 378 |
|
---|
| 379 | # ifdef IN_RING0
|
---|
| 380 | /* Calculate the elapsed time here to avoid redoing state work. */
|
---|
| 381 | cNsElapsed = RTTimeNanoTS() - tsStart;
|
---|
| 382 | # endif
|
---|
| 383 | }
|
---|
| 384 |
|
---|
| 385 | ASMNopPause();
|
---|
| 386 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
| 387 | ASMNopPause();
|
---|
| 388 | }
|
---|
[90670] | 389 | }
|
---|
[90677] | 390 | #endif /* IN_RING0 || (IN_RING3 && PDMCRITSECTRW_STRICT) */
|
---|
[90670] | 391 |
|
---|
| 392 |
|
---|
| 393 | /**
|
---|
[90671] | 394 | * Worker for pdmCritSectRwEnterShared that handles waiting for a contended CS.
|
---|
| 395 | * Caller has already added us to the read and read-wait counters.
|
---|
| 396 | */
|
---|
| 397 | static int pdmCritSectRwEnterSharedContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis,
|
---|
| 398 | int rcBusy, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
|
---|
| 399 | {
|
---|
[90677] | 400 | PSUPDRVSESSION const pSession = pVM->pSession;
|
---|
| 401 | SUPSEMEVENTMULTI const hEventMulti = (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead;
|
---|
| 402 | # ifdef IN_RING0
|
---|
| 403 | uint64_t const tsStart = RTTimeNanoTS();
|
---|
[90910] | 404 | uint64_t const cNsMaxTotalDef = RT_NS_5MIN;
|
---|
| 405 | uint64_t cNsMaxTotal = cNsMaxTotalDef;
|
---|
[90677] | 406 | uint32_t cMsMaxOne = RT_MS_5SEC;
|
---|
| 407 | bool fNonInterruptible = false;
|
---|
[90671] | 408 | # endif
|
---|
| 409 |
|
---|
| 410 | for (uint32_t iLoop = 0; ; iLoop++)
|
---|
| 411 | {
|
---|
[90673] | 412 | /*
|
---|
| 413 | * Wait for the direction to switch.
|
---|
| 414 | */
|
---|
[90671] | 415 | int rc;
|
---|
| 416 | # ifdef IN_RING3
|
---|
| 417 | # if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
| 418 | rc = RTLockValidatorRecSharedCheckBlocking(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, true,
|
---|
| 419 | RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false);
|
---|
[90672] | 420 | if (RT_FAILURE(rc))
|
---|
[90677] | 421 | return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf);
|
---|
[90671] | 422 | # else
|
---|
| 423 | RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
|
---|
| 424 | # endif
|
---|
| 425 | # endif
|
---|
[90677] | 426 |
|
---|
[90672] | 427 | for (;;)
|
---|
[90671] | 428 | {
|
---|
[90677] | 429 | /*
|
---|
| 430 | * We always wait with a timeout so we can re-check the structure sanity
|
---|
| 431 | * and not get stuck waiting on a corrupt or deleted section.
|
---|
| 432 | */
|
---|
| 433 | # ifdef IN_RING3
|
---|
| 434 | rc = SUPSemEventMultiWaitNoResume(pSession, hEventMulti, RT_MS_5SEC);
|
---|
| 435 | # else
|
---|
| 436 | rc = !fNonInterruptible
|
---|
| 437 | ? SUPSemEventMultiWaitNoResume(pSession, hEventMulti, cMsMaxOne)
|
---|
| 438 | : SUPSemEventMultiWait(pSession, hEventMulti, cMsMaxOne);
|
---|
| 439 | Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p u64State=%#RX64)\n", pThis, rc,
|
---|
| 440 | RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State));
|
---|
| 441 | # endif
|
---|
| 442 | if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
|
---|
| 443 | { /* likely */ }
|
---|
| 444 | else
|
---|
| 445 | {
|
---|
| 446 | # ifdef IN_RING3
|
---|
| 447 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
|
---|
| 448 | # endif
|
---|
| 449 | return VERR_SEM_DESTROYED;
|
---|
| 450 | }
|
---|
| 451 | if (RT_LIKELY(rc == VINF_SUCCESS))
|
---|
[90672] | 452 | break;
|
---|
[90677] | 453 |
|
---|
| 454 | /*
|
---|
| 455 | * Timeout and interrupted waits needs careful handling in ring-0
|
---|
| 456 | * because we're cooperating with ring-3 on this critical section
|
---|
| 457 | * and thus need to make absolutely sure we won't get stuck here.
|
---|
| 458 | *
|
---|
| 459 | * The r0 interrupted case means something is pending (termination,
|
---|
| 460 | * signal, APC, debugger, whatever), so we must try our best to
|
---|
| 461 | * return to the caller and to ring-3 so it can be dealt with.
|
---|
| 462 | */
|
---|
| 463 | if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED)
|
---|
| 464 | {
|
---|
[90671] | 465 | # ifdef IN_RING0
|
---|
[90677] | 466 | uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
|
---|
| 467 | int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
|
---|
| 468 | AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING,
|
---|
| 469 | ("rcTerm=%Rrc\n", rcTerm));
|
---|
[90910] | 470 | if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
|
---|
[90677] | 471 | cNsMaxTotal = RT_NS_1MIN;
|
---|
| 472 |
|
---|
| 473 | if (rc == VERR_TIMEOUT)
|
---|
| 474 | {
|
---|
| 475 | /* Try return get out of here with a non-VINF_SUCCESS status if
|
---|
| 476 | the thread is terminating or if the timeout has been exceeded. */
|
---|
| 477 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrTimeout);
|
---|
| 478 | if ( rcTerm == VINF_THREAD_IS_TERMINATING
|
---|
| 479 | || cNsElapsed > cNsMaxTotal)
|
---|
| 480 | return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc,
|
---|
| 481 | pSrcPos, fNoVal, hThreadSelf);
|
---|
| 482 | }
|
---|
| 483 | else
|
---|
| 484 | {
|
---|
| 485 | /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS,
|
---|
| 486 | we will try non-interruptible sleep for a while to help resolve the issue
|
---|
| 487 | w/o guru'ing. */
|
---|
| 488 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrInterrupted);
|
---|
| 489 | if ( rcTerm != VINF_THREAD_IS_TERMINATING
|
---|
| 490 | && rcBusy == VINF_SUCCESS
|
---|
| 491 | && pVCpu != NULL
|
---|
| 492 | && cNsElapsed <= cNsMaxTotal)
|
---|
| 493 | {
|
---|
| 494 | if (!fNonInterruptible)
|
---|
| 495 | {
|
---|
| 496 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedNonInterruptibleWaits);
|
---|
| 497 | fNonInterruptible = true;
|
---|
| 498 | cMsMaxOne = 32;
|
---|
| 499 | uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
|
---|
| 500 | if (cNsLeft > RT_NS_10SEC)
|
---|
| 501 | cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
|
---|
| 502 | }
|
---|
| 503 | }
|
---|
| 504 | else
|
---|
| 505 | return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc,
|
---|
| 506 | pSrcPos, fNoVal, hThreadSelf);
|
---|
| 507 | }
|
---|
| 508 | # else /* IN_RING3 */
|
---|
| 509 | RT_NOREF(pVM, pVCpu, rcBusy);
|
---|
| 510 | # endif /* IN_RING3 */
|
---|
| 511 | }
|
---|
| 512 | /*
|
---|
| 513 | * Any other return code is fatal.
|
---|
| 514 | */
|
---|
| 515 | else
|
---|
| 516 | {
|
---|
| 517 | # ifdef IN_RING3
|
---|
| 518 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
|
---|
[90671] | 519 | # endif
|
---|
[90677] | 520 | AssertMsgFailed(("rc=%Rrc\n", rc));
|
---|
| 521 | return RT_FAILURE_NP(rc) ? rc : -rc;
|
---|
| 522 | }
|
---|
[90672] | 523 | }
|
---|
[90677] | 524 |
|
---|
[90671] | 525 | # ifdef IN_RING3
|
---|
[90672] | 526 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
|
---|
[90671] | 527 | # endif
|
---|
[90672] | 528 |
|
---|
| 529 | /*
|
---|
| 530 | * Check the direction.
|
---|
| 531 | */
|
---|
| 532 | Assert(pThis->s.Core.fNeedReset);
|
---|
| 533 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
| 534 | if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
|
---|
[90671] | 535 | {
|
---|
[90672] | 536 | /*
|
---|
| 537 | * Decrement the wait count and maybe reset the semaphore (if we're last).
|
---|
| 538 | */
|
---|
[90677] | 539 | return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
|
---|
[90671] | 540 | }
|
---|
| 541 |
|
---|
[90677] | 542 | AssertMsg(iLoop < 1,
|
---|
| 543 | ("%p: %u u64State=%#RX64 hNativeWriter=%p\n", pThis, iLoop, u64State, pThis->s.Core.u.s.hNativeWriter));
|
---|
[90672] | 544 | RTThreadYield();
|
---|
[90671] | 545 | }
|
---|
| 546 |
|
---|
| 547 | /* not reached */
|
---|
| 548 | }
|
---|
| 549 |
|
---|
| 550 |
|
---|
| 551 | /**
|
---|
[45299] | 552 | * Worker that enters a read/write critical section with shard access.
|
---|
| 553 | *
|
---|
| 554 | * @returns VBox status code.
|
---|
[90347] | 555 | * @param pVM The cross context VM structure.
|
---|
[45299] | 556 | * @param pThis Pointer to the read/write critical section.
|
---|
| 557 | * @param rcBusy The busy return code for ring-0 and ring-3.
|
---|
| 558 | * @param fTryOnly Only try enter it, don't wait.
|
---|
| 559 | * @param pSrcPos The source position. (Can be NULL.)
|
---|
| 560 | * @param fNoVal No validation records.
|
---|
| 561 | */
|
---|
[90347] | 562 | static int pdmCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly,
|
---|
| 563 | PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
|
---|
[25431] | 564 | {
|
---|
[25663] | 565 | /*
|
---|
| 566 | * Validate input.
|
---|
| 567 | */
|
---|
[45110] | 568 | AssertPtr(pThis);
|
---|
[45152] | 569 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
[57851] | 570 |
|
---|
[45189] | 571 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[25663] | 572 | RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
|
---|
[45110] | 573 | if (!fTryOnly)
|
---|
[25663] | 574 | {
|
---|
[25710] | 575 | int rc9;
|
---|
| 576 | RTNATIVETHREAD hNativeWriter;
|
---|
[90637] | 577 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
|
---|
[90347] | 578 | if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis))
|
---|
[45152] | 579 | rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
|
---|
[25710] | 580 | else
|
---|
[45152] | 581 | rc9 = RTLockValidatorRecSharedCheckOrder(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
|
---|
[25663] | 582 | if (RT_FAILURE(rc9))
|
---|
| 583 | return rc9;
|
---|
| 584 | }
|
---|
[90670] | 585 | #else
|
---|
| 586 | RTTHREAD hThreadSelf = NIL_RTTHREAD;
|
---|
[25663] | 587 | #endif
|
---|
[25549] | 588 |
|
---|
[25663] | 589 | /*
|
---|
[90673] | 590 | * Work the state.
|
---|
[25663] | 591 | */
|
---|
[90654] | 592 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[25663] | 593 | uint64_t u64OldState = u64State;
|
---|
[25431] | 594 | for (;;)
|
---|
| 595 | {
|
---|
[45110] | 596 | if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
|
---|
[25431] | 597 | {
|
---|
[25549] | 598 | /* It flows in the right direction, try follow it before it changes. */
|
---|
[45110] | 599 | uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
|
---|
[25431] | 600 | c++;
|
---|
[90486] | 601 | Assert(c < RTCSRW_CNT_MASK / 4);
|
---|
| 602 | AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
|
---|
[45110] | 603 | u64State &= ~RTCSRW_CNT_RD_MASK;
|
---|
| 604 | u64State |= c << RTCSRW_CNT_RD_SHIFT;
|
---|
[90637] | 605 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[90670] | 606 | return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
[25431] | 607 | }
|
---|
[45110] | 608 | else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
|
---|
[25431] | 609 | {
|
---|
| 610 | /* Wrong direction, but we're alone here and can simply try switch the direction. */
|
---|
[45110] | 611 | u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
|
---|
| 612 | u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT);
|
---|
[90637] | 613 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[25549] | 614 | {
|
---|
[45152] | 615 | Assert(!pThis->s.Core.fNeedReset);
|
---|
[90670] | 616 | return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
[25549] | 617 | }
|
---|
[25431] | 618 | }
|
---|
| 619 | else
|
---|
| 620 | {
|
---|
[25663] | 621 | /* Is the writer perhaps doing a read recursion? */
|
---|
| 622 | RTNATIVETHREAD hNativeWriter;
|
---|
[90637] | 623 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
|
---|
[90486] | 624 | if (hNativeWriter != NIL_RTNATIVETHREAD)
|
---|
[25663] | 625 | {
|
---|
[90486] | 626 | RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
|
---|
| 627 | if (hNativeSelf == hNativeWriter)
|
---|
| 628 | {
|
---|
[45189] | 629 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[90486] | 630 | if (!fNoVal)
|
---|
| 631 | {
|
---|
| 632 | int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core, pSrcPos);
|
---|
| 633 | if (RT_FAILURE(rc9))
|
---|
| 634 | return rc9;
|
---|
| 635 | }
|
---|
| 636 | #endif
|
---|
| 637 | uint32_t const cReads = ASMAtomicIncU32(&pThis->s.Core.cWriterReads);
|
---|
| 638 | Assert(cReads < _16K);
|
---|
| 639 | AssertReturnStmt(cReads < PDM_CRITSECTRW_MAX_RECURSIONS, ASMAtomicDecU32(&pThis->s.Core.cWriterReads),
|
---|
| 640 | VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS);
|
---|
| 641 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
|
---|
| 642 | return VINF_SUCCESS; /* don't break! */
|
---|
[45299] | 643 | }
|
---|
[25663] | 644 | }
|
---|
| 645 |
|
---|
[45299] | 646 | /*
|
---|
| 647 | * If we're only trying, return already.
|
---|
| 648 | */
|
---|
[45110] | 649 | if (fTryOnly)
|
---|
[45299] | 650 | {
|
---|
| 651 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
|
---|
[45110] | 652 | return VERR_SEM_BUSY;
|
---|
[45299] | 653 | }
|
---|
[25663] | 654 |
|
---|
[48199] | 655 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
[90677] | 656 | /*
|
---|
| 657 | * Add ourselves to the queue and wait for the direction to change.
|
---|
| 658 | */
|
---|
| 659 | uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
|
---|
| 660 | c++;
|
---|
| 661 | Assert(c < RTCSRW_CNT_MASK / 2);
|
---|
| 662 | AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
|
---|
[25549] | 663 |
|
---|
[90677] | 664 | uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
|
---|
| 665 | cWait++;
|
---|
| 666 | Assert(cWait <= c);
|
---|
| 667 | Assert(cWait < RTCSRW_CNT_MASK / 2);
|
---|
| 668 | AssertReturn(cWait < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
|
---|
[25549] | 669 |
|
---|
[90677] | 670 | u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
|
---|
| 671 | u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
|
---|
[25549] | 672 |
|
---|
[90677] | 673 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[45293] | 674 | {
|
---|
[48199] | 675 | /*
|
---|
[90677] | 676 | * In ring-3 it's straight forward, just optimize the RTThreadSelf() call.
|
---|
[48199] | 677 | */
|
---|
[90677] | 678 | # if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)
|
---|
| 679 | return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 680 | # elif defined(IN_RING3)
|
---|
| 681 | return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, RTThreadSelf());
|
---|
| 682 | # else /* IN_RING0 */
|
---|
| 683 | /*
|
---|
| 684 | * In ring-0 context we have to take the special VT-x/AMD-V HM context into
|
---|
| 685 | * account when waiting on contended locks.
|
---|
| 686 | */
|
---|
| 687 | PVMCPUCC pVCpu = VMMGetCpu(pVM);
|
---|
| 688 | if (pVCpu)
|
---|
[48199] | 689 | {
|
---|
[90677] | 690 | VMMR0EMTBLOCKCTX Ctx;
|
---|
| 691 | int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx);
|
---|
| 692 | if (rc == VINF_SUCCESS)
|
---|
| 693 | {
|
---|
| 694 | Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
|
---|
| 695 |
|
---|
| 696 | rc = pdmCritSectRwEnterSharedContended(pVM, pVCpu, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 697 |
|
---|
| 698 | VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
|
---|
| 699 | }
|
---|
| 700 | else
|
---|
| 701 | {
|
---|
| 702 | //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy);
|
---|
| 703 | rc = pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 704 | }
|
---|
| 705 | return rc;
|
---|
[48199] | 706 | }
|
---|
[90677] | 707 |
|
---|
| 708 | /* Non-EMT. */
|
---|
| 709 | Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
|
---|
| 710 | return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 711 | # endif /* IN_RING0 */
|
---|
[45293] | 712 | }
|
---|
[90677] | 713 |
|
---|
| 714 | #else /* !IN_RING3 && !IN_RING0 */
|
---|
| 715 | /*
|
---|
| 716 | * We cannot call SUPSemEventMultiWaitNoResume in this context. Go
|
---|
| 717 | * back to ring-3 and do it there or return rcBusy.
|
---|
| 718 | */
|
---|
| 719 | # error "Unused code."
|
---|
| 720 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
|
---|
| 721 | if (rcBusy == VINF_SUCCESS)
|
---|
| 722 | {
|
---|
| 723 | PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
|
---|
| 724 | /** @todo Should actually do this in via VMMR0.cpp instead of going all the way
|
---|
| 725 | * back to ring-3. Goes for both kind of crit sects. */
|
---|
| 726 | return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED, MMHyperCCToR3(pVM, pThis));
|
---|
| 727 | }
|
---|
| 728 | return rcBusy;
|
---|
| 729 | #endif /* !IN_RING3 && !IN_RING0 */
|
---|
[25431] | 730 | }
|
---|
| 731 |
|
---|
[90670] | 732 | ASMNopPause();
|
---|
| 733 | if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
|
---|
| 734 | { /* likely */ }
|
---|
| 735 | else
|
---|
[25431] | 736 | return VERR_SEM_DESTROYED;
|
---|
[90670] | 737 | ASMNopPause();
|
---|
[25549] | 738 |
|
---|
[90654] | 739 | u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[25549] | 740 | u64OldState = u64State;
|
---|
[25431] | 741 | }
|
---|
[90670] | 742 | /* not reached */
|
---|
[25431] | 743 | }
|
---|
| 744 |
|
---|
| 745 |
|
---|
[45152] | 746 | /**
|
---|
| 747 | * Enter a critical section with shared (read) access.
|
---|
| 748 | *
|
---|
| 749 | * @returns VBox status code.
|
---|
| 750 | * @retval VINF_SUCCESS on success.
|
---|
[58116] | 751 | * @retval rcBusy if in ring-0 or raw-mode context and it is busy.
|
---|
[45152] | 752 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 753 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 754 | * during the operation.
|
---|
| 755 | *
|
---|
[90347] | 756 | * @param pVM The cross context VM structure.
|
---|
[45152] | 757 | * @param pThis Pointer to the read/write critical section.
|
---|
| 758 | * @param rcBusy The status code to return when we're in RC or R0 and the
|
---|
| 759 | * section is busy. Pass VINF_SUCCESS to acquired the
|
---|
| 760 | * critical section thru a ring-3 call if necessary.
|
---|
| 761 | * @sa PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterShared,
|
---|
| 762 | * PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
|
---|
| 763 | * RTCritSectRwEnterShared.
|
---|
| 764 | */
|
---|
[90347] | 765 | VMMDECL(int) PDMCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy)
|
---|
[25431] | 766 | {
|
---|
[45293] | 767 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 768 | return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL, false /*fNoVal*/);
|
---|
[25663] | 769 | #else
|
---|
| 770 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
|
---|
[90347] | 771 | return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
|
---|
[25663] | 772 | #endif
|
---|
| 773 | }
|
---|
[25431] | 774 |
|
---|
| 775 |
|
---|
[45152] | 776 | /**
|
---|
| 777 | * Enter a critical section with shared (read) access.
|
---|
| 778 | *
|
---|
| 779 | * @returns VBox status code.
|
---|
| 780 | * @retval VINF_SUCCESS on success.
|
---|
[58126] | 781 | * @retval rcBusy if in ring-0 or raw-mode context and it is busy.
|
---|
[45152] | 782 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 783 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 784 | * during the operation.
|
---|
| 785 | *
|
---|
[90347] | 786 | * @param pVM The cross context VM structure.
|
---|
[45152] | 787 | * @param pThis Pointer to the read/write critical section.
|
---|
| 788 | * @param rcBusy The status code to return when we're in RC or R0 and the
|
---|
| 789 | * section is busy. Pass VINF_SUCCESS to acquired the
|
---|
| 790 | * critical section thru a ring-3 call if necessary.
|
---|
| 791 | * @param uId Where we're entering the section.
|
---|
[58116] | 792 | * @param SRC_POS The source position.
|
---|
[45152] | 793 | * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
|
---|
| 794 | * PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
|
---|
| 795 | * RTCritSectRwEnterSharedDebug.
|
---|
| 796 | */
|
---|
[90347] | 797 | VMMDECL(int) PDMCritSectRwEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
|
---|
[25663] | 798 | {
|
---|
[49486] | 799 | NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
|
---|
[45293] | 800 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 801 | return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL, false /*fNoVal*/);
|
---|
[45293] | 802 | #else
|
---|
[25663] | 803 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
|
---|
[90347] | 804 | return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
|
---|
[45293] | 805 | #endif
|
---|
[25663] | 806 | }
|
---|
[25431] | 807 |
|
---|
| 808 |
|
---|
[45152] | 809 | /**
|
---|
| 810 | * Try enter a critical section with shared (read) access.
|
---|
| 811 | *
|
---|
| 812 | * @returns VBox status code.
|
---|
| 813 | * @retval VINF_SUCCESS on success.
|
---|
| 814 | * @retval VERR_SEM_BUSY if the critsect was owned.
|
---|
| 815 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 816 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 817 | * during the operation.
|
---|
| 818 | *
|
---|
[90347] | 819 | * @param pVM The cross context VM structure.
|
---|
[45152] | 820 | * @param pThis Pointer to the read/write critical section.
|
---|
| 821 | * @sa PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwEnterShared,
|
---|
| 822 | * PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
|
---|
| 823 | * RTCritSectRwTryEnterShared.
|
---|
| 824 | */
|
---|
[90347] | 825 | VMMDECL(int) PDMCritSectRwTryEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[25663] | 826 | {
|
---|
[45293] | 827 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 828 | return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL, false /*fNoVal*/);
|
---|
[25663] | 829 | #else
|
---|
| 830 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
|
---|
[90347] | 831 | return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
|
---|
[25663] | 832 | #endif
|
---|
| 833 | }
|
---|
| 834 |
|
---|
| 835 |
|
---|
[45152] | 836 | /**
|
---|
| 837 | * Try enter a critical section with shared (read) access.
|
---|
| 838 | *
|
---|
| 839 | * @returns VBox status code.
|
---|
| 840 | * @retval VINF_SUCCESS on success.
|
---|
| 841 | * @retval VERR_SEM_BUSY if the critsect was owned.
|
---|
| 842 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 843 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 844 | * during the operation.
|
---|
| 845 | *
|
---|
[90347] | 846 | * @param pVM The cross context VM structure.
|
---|
[45152] | 847 | * @param pThis Pointer to the read/write critical section.
|
---|
| 848 | * @param uId Where we're entering the section.
|
---|
[58116] | 849 | * @param SRC_POS The source position.
|
---|
[45152] | 850 | * @sa PDMCritSectRwTryEnterShared, PDMCritSectRwEnterShared,
|
---|
| 851 | * PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
|
---|
| 852 | * RTCritSectRwTryEnterSharedDebug.
|
---|
| 853 | */
|
---|
[90347] | 854 | VMMDECL(int) PDMCritSectRwTryEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
|
---|
[25663] | 855 | {
|
---|
[49486] | 856 | NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
|
---|
[45293] | 857 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 858 | return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL, false /*fNoVal*/);
|
---|
[45293] | 859 | #else
|
---|
[25663] | 860 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
|
---|
[90347] | 861 | return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
|
---|
[45293] | 862 | #endif
|
---|
[25663] | 863 | }
|
---|
| 864 |
|
---|
| 865 |
|
---|
[45293] | 866 | #ifdef IN_RING3
|
---|
[45152] | 867 | /**
|
---|
[45293] | 868 | * Enters a PDM read/write critical section with shared (read) access.
|
---|
| 869 | *
|
---|
| 870 | * @returns VINF_SUCCESS if entered successfully.
|
---|
| 871 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 872 | * during the operation.
|
---|
| 873 | *
|
---|
[90347] | 874 | * @param pVM The cross context VM structure.
|
---|
[45293] | 875 | * @param pThis Pointer to the read/write critical section.
|
---|
| 876 | * @param fCallRing3 Whether this is a VMMRZCallRing3()request.
|
---|
| 877 | */
|
---|
[90347] | 878 | VMMR3DECL(int) PDMR3CritSectRwEnterSharedEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3)
|
---|
[45293] | 879 | {
|
---|
[90347] | 880 | return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3);
|
---|
[45293] | 881 | }
|
---|
[45299] | 882 | #endif
|
---|
[45293] | 883 |
|
---|
| 884 |
|
---|
| 885 | /**
|
---|
[45152] | 886 | * Leave a critical section held with shared access.
|
---|
| 887 | *
|
---|
| 888 | * @returns VBox status code.
|
---|
| 889 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 890 | * during the operation.
|
---|
[90347] | 891 | * @param pVM The cross context VM structure.
|
---|
[48936] | 892 | * @param pThis Pointer to the read/write critical section.
|
---|
[45299] | 893 | * @param fNoVal No validation records (i.e. queued release).
|
---|
[45152] | 894 | * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
|
---|
| 895 | * PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
|
---|
| 896 | * PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
|
---|
| 897 | */
|
---|
[90347] | 898 | static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal)
|
---|
[25663] | 899 | {
|
---|
| 900 | /*
|
---|
| 901 | * Validate handle.
|
---|
| 902 | */
|
---|
[45110] | 903 | AssertPtr(pThis);
|
---|
[45152] | 904 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
[25663] | 905 |
|
---|
[57851] | 906 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
| 907 | NOREF(fNoVal);
|
---|
| 908 | #endif
|
---|
| 909 |
|
---|
[25663] | 910 | /*
|
---|
| 911 | * Check the direction and take action accordingly.
|
---|
| 912 | */
|
---|
[90667] | 913 | #ifdef IN_RING0
|
---|
| 914 | PVMCPUCC pVCpu = NULL;
|
---|
| 915 | #endif
|
---|
[90654] | 916 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[25663] | 917 | uint64_t u64OldState = u64State;
|
---|
[45110] | 918 | if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
|
---|
[25663] | 919 | {
|
---|
[45189] | 920 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[45299] | 921 | if (fNoVal)
|
---|
| 922 | Assert(!RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD));
|
---|
| 923 | else
|
---|
| 924 | {
|
---|
| 925 | int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
|
---|
| 926 | if (RT_FAILURE(rc9))
|
---|
| 927 | return rc9;
|
---|
| 928 | }
|
---|
[25663] | 929 | #endif
|
---|
| 930 | for (;;)
|
---|
[25431] | 931 | {
|
---|
[45110] | 932 | uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
|
---|
[25663] | 933 | AssertReturn(c > 0, VERR_NOT_OWNER);
|
---|
| 934 | c--;
|
---|
| 935 |
|
---|
| 936 | if ( c > 0
|
---|
[45310] | 937 | || (u64State & RTCSRW_CNT_WR_MASK) == 0)
|
---|
[25549] | 938 | {
|
---|
[25663] | 939 | /* Don't change the direction. */
|
---|
[45110] | 940 | u64State &= ~RTCSRW_CNT_RD_MASK;
|
---|
| 941 | u64State |= c << RTCSRW_CNT_RD_SHIFT;
|
---|
[90637] | 942 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[25663] | 943 | break;
|
---|
[25549] | 944 | }
|
---|
[25663] | 945 | else
|
---|
| 946 | {
|
---|
[48199] | 947 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
| 948 | # ifdef IN_RING0
|
---|
[90667] | 949 | Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe());
|
---|
| 950 | if (!pVCpu)
|
---|
| 951 | pVCpu = VMMGetCpu(pVM);
|
---|
| 952 | if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
|
---|
| 953 | || VMMRZCallRing3IsEnabled(pVCpu)
|
---|
| 954 | || RTSemEventIsSignalSafe()
|
---|
| 955 | || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */
|
---|
| 956 | && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */
|
---|
| 957 | && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
|
---|
| 958 | )
|
---|
[48199] | 959 | # endif
|
---|
[25663] | 960 | {
|
---|
[48199] | 961 | /* Reverse the direction and signal the writer threads. */
|
---|
| 962 | u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK);
|
---|
| 963 | u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
|
---|
[90637] | 964 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[48199] | 965 | {
|
---|
[90667] | 966 | int rc;
|
---|
| 967 | # ifdef IN_RING0
|
---|
| 968 | STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
|
---|
| 969 | if (!RTSemEventIsSignalSafe() && pVCpu != NULL)
|
---|
| 970 | {
|
---|
| 971 | VMMR0EMTBLOCKCTX Ctx;
|
---|
| 972 | rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
|
---|
| 973 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
|
---|
| 974 |
|
---|
| 975 | rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
|
---|
| 976 |
|
---|
| 977 | VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
|
---|
| 978 | }
|
---|
| 979 | else
|
---|
| 980 | # endif
|
---|
| 981 | rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
|
---|
[48199] | 982 | AssertRC(rc);
|
---|
[90667] | 983 | return rc;
|
---|
[48199] | 984 | }
|
---|
| 985 | }
|
---|
| 986 | #endif /* IN_RING3 || IN_RING0 */
|
---|
| 987 | #ifndef IN_RING3
|
---|
| 988 | # ifdef IN_RING0
|
---|
| 989 | else
|
---|
| 990 | # endif
|
---|
| 991 | {
|
---|
| 992 | /* Queue the exit request (ring-3). */
|
---|
[90667] | 993 | # ifndef IN_RING0
|
---|
| 994 | PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
|
---|
| 995 | # endif
|
---|
[48199] | 996 | uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves++;
|
---|
| 997 | LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3 c=%d (%#llx)\n", i, pThis, c, u64State));
|
---|
[90571] | 998 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves),
|
---|
| 999 | ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE);
|
---|
[90570] | 1000 | pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] = pThis->s.pSelfR3;
|
---|
[90571] | 1001 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
|
---|
| 1002 | RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i])
|
---|
[93554] | 1003 | && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] & HOST_PAGE_OFFSET_MASK)
|
---|
| 1004 | == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK),
|
---|
[90571] | 1005 | ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i], pThis),
|
---|
[90573] | 1006 | pdmCritSectRwCorrupted(pThis, "Invalid self pointer"));
|
---|
[48199] | 1007 | VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
|
---|
| 1008 | VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
|
---|
| 1009 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
|
---|
| 1010 | STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
|
---|
[25663] | 1011 | break;
|
---|
| 1012 | }
|
---|
[45293] | 1013 | #endif
|
---|
[25663] | 1014 | }
|
---|
| 1015 |
|
---|
| 1016 | ASMNopPause();
|
---|
[90667] | 1017 | if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
|
---|
| 1018 | { }
|
---|
| 1019 | else
|
---|
| 1020 | return VERR_SEM_DESTROYED;
|
---|
| 1021 | ASMNopPause();
|
---|
| 1022 |
|
---|
[90654] | 1023 | u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[25663] | 1024 | u64OldState = u64State;
|
---|
[25431] | 1025 | }
|
---|
[25663] | 1026 | }
|
---|
| 1027 | else
|
---|
| 1028 | {
|
---|
[90486] | 1029 | /*
|
---|
| 1030 | * Write direction. Check that it's the owner calling and that it has reads to undo.
|
---|
| 1031 | */
|
---|
[90347] | 1032 | RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
|
---|
[90486] | 1033 | AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
|
---|
| 1034 |
|
---|
[25663] | 1035 | RTNATIVETHREAD hNativeWriter;
|
---|
[90637] | 1036 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
|
---|
[25663] | 1037 | AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
|
---|
[45152] | 1038 | AssertReturn(pThis->s.Core.cWriterReads > 0, VERR_NOT_OWNER);
|
---|
[45189] | 1039 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[45299] | 1040 | if (!fNoVal)
|
---|
| 1041 | {
|
---|
| 1042 | int rc = RTLockValidatorRecExclUnwindMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core);
|
---|
| 1043 | if (RT_FAILURE(rc))
|
---|
| 1044 | return rc;
|
---|
| 1045 | }
|
---|
[25663] | 1046 | #endif
|
---|
[90486] | 1047 | uint32_t cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriterReads);
|
---|
[90573] | 1048 | AssertReturn(cDepth < PDM_CRITSECTRW_MAX_RECURSIONS, pdmCritSectRwCorrupted(pThis, "too many writer-read recursions"));
|
---|
[25663] | 1049 | }
|
---|
| 1050 |
|
---|
| 1051 | return VINF_SUCCESS;
|
---|
| 1052 | }
|
---|
| 1053 |
|
---|
[90486] | 1054 |
|
---|
[45299] | 1055 | /**
|
---|
| 1056 | * Leave a critical section held with shared access.
|
---|
| 1057 | *
|
---|
| 1058 | * @returns VBox status code.
|
---|
| 1059 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1060 | * during the operation.
|
---|
[90347] | 1061 | * @param pVM The cross context VM structure.
|
---|
[45299] | 1062 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1063 | * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
|
---|
| 1064 | * PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
|
---|
| 1065 | * PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
|
---|
| 1066 | */
|
---|
[90347] | 1067 | VMMDECL(int) PDMCritSectRwLeaveShared(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[45299] | 1068 | {
|
---|
[90347] | 1069 | return pdmCritSectRwLeaveSharedWorker(pVM, pThis, false /*fNoVal*/);
|
---|
[45299] | 1070 | }
|
---|
[25663] | 1071 |
|
---|
[45299] | 1072 |
|
---|
| 1073 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
| 1074 | /**
|
---|
| 1075 | * PDMCritSectBothFF interface.
|
---|
| 1076 | *
|
---|
[90347] | 1077 | * @param pVM The cross context VM structure.
|
---|
[45299] | 1078 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1079 | */
|
---|
[90347] | 1080 | void pdmCritSectRwLeaveSharedQueued(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[25663] | 1081 | {
|
---|
[90347] | 1082 | pdmCritSectRwLeaveSharedWorker(pVM, pThis, true /*fNoVal*/);
|
---|
[45299] | 1083 | }
|
---|
| 1084 | #endif
|
---|
| 1085 |
|
---|
| 1086 |
|
---|
| 1087 | /**
|
---|
[90608] | 1088 | * Worker for pdmCritSectRwEnterExcl that bails out on wait failure.
|
---|
| 1089 | *
|
---|
| 1090 | * @returns @a rc unless corrupted.
|
---|
[90610] | 1091 | * @param pThis Pointer to the read/write critical section.
|
---|
[90608] | 1092 | * @param rc The status to return.
|
---|
| 1093 | */
|
---|
| 1094 | DECL_NO_INLINE(static, int) pdmCritSectRwEnterExclBailOut(PPDMCRITSECTRW pThis, int rc)
|
---|
| 1095 | {
|
---|
| 1096 | /*
|
---|
| 1097 | * Decrement the counts and return the error.
|
---|
| 1098 | */
|
---|
| 1099 | for (;;)
|
---|
| 1100 | {
|
---|
[90654] | 1101 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[90608] | 1102 | uint64_t const u64OldState = u64State;
|
---|
| 1103 | uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
|
---|
| 1104 | AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on bailout"));
|
---|
| 1105 | c--;
|
---|
| 1106 | u64State &= ~RTCSRW_CNT_WR_MASK;
|
---|
| 1107 | u64State |= c << RTCSRW_CNT_WR_SHIFT;
|
---|
[90637] | 1108 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[90608] | 1109 | return rc;
|
---|
| 1110 |
|
---|
| 1111 | ASMNopPause();
|
---|
| 1112 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
| 1113 | ASMNopPause();
|
---|
| 1114 | }
|
---|
| 1115 | }
|
---|
| 1116 |
|
---|
[90611] | 1117 |
|
---|
| 1118 | /**
|
---|
| 1119 | * Worker for pdmCritSectRwEnterExcl that handles the red tape after we've
|
---|
| 1120 | * gotten exclusive ownership of the critical section.
|
---|
| 1121 | */
|
---|
[90665] | 1122 | DECL_FORCE_INLINE(int) pdmCritSectRwEnterExclFirst(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos,
|
---|
| 1123 | bool fNoVal, RTTHREAD hThreadSelf)
|
---|
[90611] | 1124 | {
|
---|
| 1125 | RT_NOREF(hThreadSelf, fNoVal, pSrcPos);
|
---|
[90654] | 1126 | Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
|
---|
[90611] | 1127 |
|
---|
[90654] | 1128 | #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
| 1129 | pThis->s.Core.cWriteRecursions = 1;
|
---|
| 1130 | #else
|
---|
[90611] | 1131 | ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 1);
|
---|
| 1132 | #endif
|
---|
| 1133 | Assert(pThis->s.Core.cWriterReads == 0);
|
---|
| 1134 |
|
---|
| 1135 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
| 1136 | if (!fNoVal)
|
---|
| 1137 | {
|
---|
| 1138 | if (hThreadSelf == NIL_RTTHREAD)
|
---|
| 1139 | hThreadSelf = RTThreadSelfAutoAdopt();
|
---|
| 1140 | RTLockValidatorRecExclSetOwner(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true);
|
---|
| 1141 | }
|
---|
| 1142 | #endif
|
---|
| 1143 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
|
---|
| 1144 | STAM_PROFILE_ADV_START(&pThis->s.StatWriteLocked, swl);
|
---|
| 1145 | return VINF_SUCCESS;
|
---|
| 1146 | }
|
---|
| 1147 |
|
---|
| 1148 |
|
---|
[90610] | 1149 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
| 1150 | /**
|
---|
| 1151 | * Worker for pdmCritSectRwEnterExcl that handles waiting when the section is
|
---|
| 1152 | * contended.
|
---|
| 1153 | */
|
---|
| 1154 | static int pdmR3R0CritSectRwEnterExclContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, RTNATIVETHREAD hNativeSelf,
|
---|
[90611] | 1155 | PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, int rcBusy, RTTHREAD hThreadSelf)
|
---|
[90610] | 1156 | {
|
---|
[90611] | 1157 | RT_NOREF(hThreadSelf, rcBusy, pSrcPos, fNoVal, pVCpu);
|
---|
[90608] | 1158 |
|
---|
[90634] | 1159 | PSUPDRVSESSION const pSession = pVM->pSession;
|
---|
| 1160 | SUPSEMEVENT const hEvent = (SUPSEMEVENT)pThis->s.Core.hEvtWrite;
|
---|
| 1161 | # ifdef IN_RING0
|
---|
| 1162 | uint64_t const tsStart = RTTimeNanoTS();
|
---|
[90910] | 1163 | uint64_t const cNsMaxTotalDef = RT_NS_5MIN;
|
---|
| 1164 | uint64_t cNsMaxTotal = cNsMaxTotalDef;
|
---|
[90634] | 1165 | uint32_t cMsMaxOne = RT_MS_5SEC;
|
---|
| 1166 | bool fNonInterruptible = false;
|
---|
| 1167 | # endif
|
---|
| 1168 |
|
---|
[90610] | 1169 | for (uint32_t iLoop = 0; ; iLoop++)
|
---|
| 1170 | {
|
---|
| 1171 | /*
|
---|
| 1172 | * Wait for our turn.
|
---|
| 1173 | */
|
---|
| 1174 | int rc;
|
---|
| 1175 | # ifdef IN_RING3
|
---|
| 1176 | # ifdef PDMCRITSECTRW_STRICT
|
---|
| 1177 | rc = RTLockValidatorRecExclCheckBlocking(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true,
|
---|
| 1178 | RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false);
|
---|
| 1179 | if (RT_SUCCESS(rc))
|
---|
[90634] | 1180 | { /* likely */ }
|
---|
| 1181 | else
|
---|
| 1182 | return pdmCritSectRwEnterExclBailOut(pThis, rc);
|
---|
[90610] | 1183 | # else
|
---|
| 1184 | RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
|
---|
| 1185 | # endif
|
---|
| 1186 | # endif
|
---|
[90677] | 1187 |
|
---|
[90634] | 1188 | for (;;)
|
---|
[90610] | 1189 | {
|
---|
[90634] | 1190 | /*
|
---|
| 1191 | * We always wait with a timeout so we can re-check the structure sanity
|
---|
| 1192 | * and not get stuck waiting on a corrupt or deleted section.
|
---|
| 1193 | */
|
---|
| 1194 | # ifdef IN_RING3
|
---|
| 1195 | rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC);
|
---|
| 1196 | # else
|
---|
| 1197 | rc = !fNonInterruptible
|
---|
| 1198 | ? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne)
|
---|
| 1199 | : SUPSemEventWait(pSession, hEvent, cMsMaxOne);
|
---|
| 1200 | Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p)\n",
|
---|
[90637] | 1201 | pThis, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter));
|
---|
[90634] | 1202 | # endif
|
---|
| 1203 | if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
|
---|
| 1204 | { /* likely */ }
|
---|
| 1205 | else
|
---|
[90610] | 1206 | {
|
---|
[90634] | 1207 | # ifdef IN_RING3
|
---|
| 1208 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
|
---|
[90610] | 1209 | # endif
|
---|
[90634] | 1210 | return VERR_SEM_DESTROYED;
|
---|
[90610] | 1211 | }
|
---|
[90677] | 1212 | if (RT_LIKELY(rc == VINF_SUCCESS))
|
---|
[90634] | 1213 | break;
|
---|
[90610] | 1214 |
|
---|
[90634] | 1215 | /*
|
---|
| 1216 | * Timeout and interrupted waits needs careful handling in ring-0
|
---|
| 1217 | * because we're cooperating with ring-3 on this critical section
|
---|
| 1218 | * and thus need to make absolutely sure we won't get stuck here.
|
---|
| 1219 | *
|
---|
| 1220 | * The r0 interrupted case means something is pending (termination,
|
---|
| 1221 | * signal, APC, debugger, whatever), so we must try our best to
|
---|
| 1222 | * return to the caller and to ring-3 so it can be dealt with.
|
---|
| 1223 | */
|
---|
[90677] | 1224 | if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED)
|
---|
[90634] | 1225 | {
|
---|
| 1226 | # ifdef IN_RING0
|
---|
| 1227 | uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
|
---|
| 1228 | int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
|
---|
| 1229 | AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING,
|
---|
| 1230 | ("rcTerm=%Rrc\n", rcTerm));
|
---|
[90910] | 1231 | if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
|
---|
[90634] | 1232 | cNsMaxTotal = RT_NS_1MIN;
|
---|
| 1233 |
|
---|
| 1234 | if (rc == VERR_TIMEOUT)
|
---|
| 1235 | {
|
---|
| 1236 | /* Try return get out of here with a non-VINF_SUCCESS status if
|
---|
| 1237 | the thread is terminating or if the timeout has been exceeded. */
|
---|
| 1238 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrTimeout);
|
---|
| 1239 | if ( rcTerm == VINF_THREAD_IS_TERMINATING
|
---|
| 1240 | || cNsElapsed > cNsMaxTotal)
|
---|
| 1241 | return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc);
|
---|
| 1242 | }
|
---|
| 1243 | else
|
---|
| 1244 | {
|
---|
| 1245 | /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS,
|
---|
| 1246 | we will try non-interruptible sleep for a while to help resolve the issue
|
---|
| 1247 | w/o guru'ing. */
|
---|
| 1248 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrInterrupted);
|
---|
| 1249 | if ( rcTerm != VINF_THREAD_IS_TERMINATING
|
---|
| 1250 | && rcBusy == VINF_SUCCESS
|
---|
| 1251 | && pVCpu != NULL
|
---|
| 1252 | && cNsElapsed <= cNsMaxTotal)
|
---|
| 1253 | {
|
---|
| 1254 | if (!fNonInterruptible)
|
---|
| 1255 | {
|
---|
| 1256 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclNonInterruptibleWaits);
|
---|
| 1257 | fNonInterruptible = true;
|
---|
| 1258 | cMsMaxOne = 32;
|
---|
| 1259 | uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
|
---|
| 1260 | if (cNsLeft > RT_NS_10SEC)
|
---|
| 1261 | cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
|
---|
| 1262 | }
|
---|
| 1263 | }
|
---|
| 1264 | else
|
---|
| 1265 | return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc);
|
---|
| 1266 | }
|
---|
| 1267 | # else /* IN_RING3 */
|
---|
| 1268 | RT_NOREF(pVM, pVCpu, rcBusy);
|
---|
| 1269 | # endif /* IN_RING3 */
|
---|
| 1270 | }
|
---|
| 1271 | /*
|
---|
| 1272 | * Any other return code is fatal.
|
---|
| 1273 | */
|
---|
| 1274 | else
|
---|
| 1275 | {
|
---|
[90610] | 1276 | # ifdef IN_RING3
|
---|
[90634] | 1277 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
|
---|
[90610] | 1278 | # endif
|
---|
[90634] | 1279 | AssertMsgFailed(("rc=%Rrc\n", rc));
|
---|
| 1280 | return RT_FAILURE_NP(rc) ? rc : -rc;
|
---|
| 1281 | }
|
---|
[90610] | 1282 | }
|
---|
| 1283 |
|
---|
[90677] | 1284 | # ifdef IN_RING3
|
---|
| 1285 | RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
|
---|
| 1286 | # endif
|
---|
| 1287 |
|
---|
[90610] | 1288 | /*
|
---|
| 1289 | * Try take exclusive write ownership.
|
---|
| 1290 | */
|
---|
[90654] | 1291 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[90610] | 1292 | if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
|
---|
| 1293 | {
|
---|
| 1294 | bool fDone;
|
---|
[90637] | 1295 | ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
|
---|
[90610] | 1296 | if (fDone)
|
---|
[90611] | 1297 | return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
[90610] | 1298 | }
|
---|
| 1299 | AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
|
---|
| 1300 | }
|
---|
| 1301 | }
|
---|
| 1302 | #endif /* IN_RING3 || IN_RING0 */
|
---|
| 1303 |
|
---|
| 1304 |
|
---|
[90608] | 1305 | /**
|
---|
[45299] | 1306 | * Worker that enters a read/write critical section with exclusive access.
|
---|
| 1307 | *
|
---|
| 1308 | * @returns VBox status code.
|
---|
[90347] | 1309 | * @param pVM The cross context VM structure.
|
---|
[45299] | 1310 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1311 | * @param rcBusy The busy return code for ring-0 and ring-3.
|
---|
| 1312 | * @param fTryOnly Only try enter it, don't wait.
|
---|
| 1313 | * @param pSrcPos The source position. (Can be NULL.)
|
---|
| 1314 | * @param fNoVal No validation records.
|
---|
| 1315 | */
|
---|
[90347] | 1316 | static int pdmCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly,
|
---|
| 1317 | PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
|
---|
[45299] | 1318 | {
|
---|
[25663] | 1319 | /*
|
---|
| 1320 | * Validate input.
|
---|
| 1321 | */
|
---|
[45110] | 1322 | AssertPtr(pThis);
|
---|
[45152] | 1323 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
[25663] | 1324 |
|
---|
[90611] | 1325 | RTTHREAD hThreadSelf = NIL_RTTHREAD;
|
---|
[45189] | 1326 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[45110] | 1327 | if (!fTryOnly)
|
---|
[25663] | 1328 | {
|
---|
| 1329 | hThreadSelf = RTThreadSelfAutoAdopt();
|
---|
[45152] | 1330 | int rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
|
---|
[25663] | 1331 | if (RT_FAILURE(rc9))
|
---|
| 1332 | return rc9;
|
---|
| 1333 | }
|
---|
| 1334 | #endif
|
---|
| 1335 |
|
---|
| 1336 | /*
|
---|
| 1337 | * Check if we're already the owner and just recursing.
|
---|
| 1338 | */
|
---|
[90486] | 1339 | RTNATIVETHREAD const hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
|
---|
| 1340 | AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
|
---|
[25663] | 1341 | RTNATIVETHREAD hNativeWriter;
|
---|
[90637] | 1342 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
|
---|
[25663] | 1343 | if (hNativeSelf == hNativeWriter)
|
---|
| 1344 | {
|
---|
[90654] | 1345 | Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
|
---|
[45189] | 1346 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[45299] | 1347 | if (!fNoVal)
|
---|
| 1348 | {
|
---|
| 1349 | int rc9 = RTLockValidatorRecExclRecursion(pThis->s.Core.pValidatorWrite, pSrcPos);
|
---|
| 1350 | if (RT_FAILURE(rc9))
|
---|
| 1351 | return rc9;
|
---|
| 1352 | }
|
---|
[25663] | 1353 | #endif
|
---|
[45178] | 1354 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
|
---|
[90654] | 1355 | #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
| 1356 | uint32_t const cDepth = ++pThis->s.Core.cWriteRecursions;
|
---|
| 1357 | #else
|
---|
[90486] | 1358 | uint32_t const cDepth = ASMAtomicIncU32(&pThis->s.Core.cWriteRecursions);
|
---|
[90654] | 1359 | #endif
|
---|
[90486] | 1360 | AssertReturnStmt(cDepth > 1 && cDepth <= PDM_CRITSECTRW_MAX_RECURSIONS,
|
---|
| 1361 | ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions),
|
---|
| 1362 | VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS);
|
---|
[25663] | 1363 | return VINF_SUCCESS;
|
---|
| 1364 | }
|
---|
| 1365 |
|
---|
| 1366 | /*
|
---|
[90665] | 1367 | * First we try grab an idle critical section using 128-bit atomics.
|
---|
[25663] | 1368 | */
|
---|
[90665] | 1369 | /** @todo This could be moved up before the recursion check. */
|
---|
| 1370 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
| 1371 | #ifdef RTASM_HAVE_CMP_WRITE_U128
|
---|
| 1372 | if ( (u64State & ~RTCSRW_DIR_MASK) == 0
|
---|
| 1373 | && pdmCritSectRwIsCmpWriteU128Supported())
|
---|
| 1374 | {
|
---|
| 1375 | RTCRITSECTRWSTATE OldState;
|
---|
| 1376 | OldState.s.u64State = u64State;
|
---|
| 1377 | OldState.s.hNativeWriter = NIL_RTNATIVETHREAD;
|
---|
| 1378 | AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo));
|
---|
| 1379 |
|
---|
| 1380 | RTCRITSECTRWSTATE NewState;
|
---|
| 1381 | NewState.s.u64State = (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
|
---|
| 1382 | NewState.s.hNativeWriter = hNativeSelf;
|
---|
| 1383 |
|
---|
| 1384 | if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128))
|
---|
| 1385 | return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 1386 |
|
---|
| 1387 | u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
| 1388 | }
|
---|
| 1389 | #endif
|
---|
| 1390 |
|
---|
| 1391 | /*
|
---|
| 1392 | * Do it step by step. Update the state to reflect our desire.
|
---|
| 1393 | */
|
---|
[25663] | 1394 | uint64_t u64OldState = u64State;
|
---|
| 1395 |
|
---|
| 1396 | for (;;)
|
---|
| 1397 | {
|
---|
[45110] | 1398 | if ( (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
|
---|
| 1399 | || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0)
|
---|
[25663] | 1400 | {
|
---|
| 1401 | /* It flows in the right direction, try follow it before it changes. */
|
---|
[45110] | 1402 | uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
|
---|
[90486] | 1403 | AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS);
|
---|
[25663] | 1404 | c++;
|
---|
[90486] | 1405 | Assert(c < RTCSRW_CNT_WR_MASK / 4);
|
---|
[45110] | 1406 | u64State &= ~RTCSRW_CNT_WR_MASK;
|
---|
| 1407 | u64State |= c << RTCSRW_CNT_WR_SHIFT;
|
---|
[90637] | 1408 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[25663] | 1409 | break;
|
---|
| 1410 | }
|
---|
[45110] | 1411 | else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
|
---|
[25663] | 1412 | {
|
---|
| 1413 | /* Wrong direction, but we're alone here and can simply try switch the direction. */
|
---|
[45110] | 1414 | u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
|
---|
| 1415 | u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
|
---|
[90637] | 1416 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[25663] | 1417 | break;
|
---|
| 1418 | }
|
---|
[45110] | 1419 | else if (fTryOnly)
|
---|
[45299] | 1420 | {
|
---|
[25663] | 1421 | /* Wrong direction and we're not supposed to wait, just return. */
|
---|
[45299] | 1422 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
|
---|
[45110] | 1423 | return VERR_SEM_BUSY;
|
---|
[45299] | 1424 | }
|
---|
[25431] | 1425 | else
|
---|
| 1426 | {
|
---|
[25663] | 1427 | /* Add ourselves to the write count and break out to do the wait. */
|
---|
[45110] | 1428 | uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
|
---|
[90486] | 1429 | AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS);
|
---|
[25663] | 1430 | c++;
|
---|
[90486] | 1431 | Assert(c < RTCSRW_CNT_WR_MASK / 4);
|
---|
[45110] | 1432 | u64State &= ~RTCSRW_CNT_WR_MASK;
|
---|
| 1433 | u64State |= c << RTCSRW_CNT_WR_SHIFT;
|
---|
[90637] | 1434 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
[25663] | 1435 | break;
|
---|
| 1436 | }
|
---|
| 1437 |
|
---|
[90611] | 1438 | ASMNopPause();
|
---|
| 1439 |
|
---|
| 1440 | if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)
|
---|
| 1441 | { /* likely */ }
|
---|
| 1442 | else
|
---|
[25663] | 1443 | return VERR_SEM_DESTROYED;
|
---|
| 1444 |
|
---|
| 1445 | ASMNopPause();
|
---|
[90654] | 1446 | u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[25663] | 1447 | u64OldState = u64State;
|
---|
| 1448 | }
|
---|
| 1449 |
|
---|
| 1450 | /*
|
---|
| 1451 | * If we're in write mode now try grab the ownership. Play fair if there
|
---|
[90659] | 1452 | * are threads already waiting.
|
---|
[25663] | 1453 | */
|
---|
[45110] | 1454 | bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
|
---|
| 1455 | && ( ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1
|
---|
[90659] | 1456 | || fTryOnly);
|
---|
[25663] | 1457 | if (fDone)
|
---|
[90611] | 1458 | {
|
---|
[90637] | 1459 | ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
|
---|
[90611] | 1460 | if (fDone)
|
---|
| 1461 | return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
|
---|
| 1462 | }
|
---|
[45299] | 1463 |
|
---|
[90611] | 1464 | /*
|
---|
| 1465 | * Okay, we have contention and will have to wait unless we're just trying.
|
---|
| 1466 | */
|
---|
[90612] | 1467 | if (fTryOnly)
|
---|
| 1468 | {
|
---|
| 1469 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); /** @todo different statistics for this */
|
---|
| 1470 | return pdmCritSectRwEnterExclBailOut(pThis, VERR_SEM_BUSY);
|
---|
| 1471 | }
|
---|
| 1472 |
|
---|
[90611] | 1473 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
|
---|
[90634] | 1474 |
|
---|
| 1475 | /*
|
---|
| 1476 | * Ring-3 is pretty straight forward.
|
---|
| 1477 | */
|
---|
| 1478 | #if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)
|
---|
| 1479 | return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, hThreadSelf);
|
---|
| 1480 | #elif defined(IN_RING3)
|
---|
| 1481 | return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, RTThreadSelf());
|
---|
| 1482 |
|
---|
| 1483 | #elif defined(IN_RING0)
|
---|
| 1484 | /*
|
---|
| 1485 | * In ring-0 context we have to take the special VT-x/AMD-V HM context into
|
---|
| 1486 | * account when waiting on contended locks.
|
---|
| 1487 | */
|
---|
| 1488 | PVMCPUCC pVCpu = VMMGetCpu(pVM);
|
---|
| 1489 | if (pVCpu)
|
---|
[90611] | 1490 | {
|
---|
[90634] | 1491 | VMMR0EMTBLOCKCTX Ctx;
|
---|
| 1492 | int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx);
|
---|
| 1493 | if (rc == VINF_SUCCESS)
|
---|
| 1494 | {
|
---|
| 1495 | Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
|
---|
| 1496 |
|
---|
| 1497 | rc = pdmR3R0CritSectRwEnterExclContended(pVM, pVCpu, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD);
|
---|
| 1498 |
|
---|
| 1499 | VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
|
---|
| 1500 | }
|
---|
| 1501 | else
|
---|
| 1502 | {
|
---|
| 1503 | //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy);
|
---|
| 1504 | rc = pdmCritSectRwEnterExclBailOut(pThis, rc);
|
---|
| 1505 | }
|
---|
| 1506 | return rc;
|
---|
[90611] | 1507 | }
|
---|
| 1508 |
|
---|
[90634] | 1509 | /* Non-EMT. */
|
---|
| 1510 | Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
|
---|
| 1511 | return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD);
|
---|
| 1512 |
|
---|
| 1513 | #else
|
---|
| 1514 | # error "Unused."
|
---|
| 1515 | /*
|
---|
| 1516 | * Raw-mode: Call host and take it there if rcBusy is VINF_SUCCESS.
|
---|
| 1517 | */
|
---|
[90611] | 1518 | rcBusy = pdmCritSectRwEnterExclBailOut(pThis, rcBusy);
|
---|
| 1519 | if (rcBusy == VINF_SUCCESS)
|
---|
| 1520 | {
|
---|
| 1521 | Assert(!fTryOnly);
|
---|
| 1522 | PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
|
---|
| 1523 | /** @todo Should actually do this in via VMMR0.cpp instead of going all the way
|
---|
| 1524 | * back to ring-3. Goes for both kind of crit sects. */
|
---|
| 1525 | return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL, MMHyperCCToR3(pVM, pThis));
|
---|
[25663] | 1526 | }
|
---|
[90611] | 1527 | return rcBusy;
|
---|
[90608] | 1528 | #endif
|
---|
[25663] | 1529 | }
|
---|
| 1530 |
|
---|
| 1531 |
|
---|
[45152] | 1532 | /**
|
---|
| 1533 | * Try enter a critical section with exclusive (write) access.
|
---|
| 1534 | *
|
---|
| 1535 | * @returns VBox status code.
|
---|
| 1536 | * @retval VINF_SUCCESS on success.
|
---|
[58116] | 1537 | * @retval rcBusy if in ring-0 or raw-mode context and it is busy.
|
---|
[45152] | 1538 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 1539 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1540 | * during the operation.
|
---|
| 1541 | *
|
---|
[90347] | 1542 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1543 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1544 | * @param rcBusy The status code to return when we're in RC or R0 and the
|
---|
| 1545 | * section is busy. Pass VINF_SUCCESS to acquired the
|
---|
| 1546 | * critical section thru a ring-3 call if necessary.
|
---|
| 1547 | * @sa PDMCritSectRwEnterExclDebug, PDMCritSectRwTryEnterExcl,
|
---|
| 1548 | * PDMCritSectRwTryEnterExclDebug,
|
---|
| 1549 | * PDMCritSectEnterDebug, PDMCritSectEnter,
|
---|
| 1550 | * RTCritSectRwEnterExcl.
|
---|
| 1551 | */
|
---|
[90347] | 1552 | VMMDECL(int) PDMCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy)
|
---|
[25663] | 1553 | {
|
---|
[45293] | 1554 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 1555 | return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL, false /*fNoVal*/);
|
---|
[25663] | 1556 | #else
|
---|
| 1557 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
|
---|
[90347] | 1558 | return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
|
---|
[25663] | 1559 | #endif
|
---|
| 1560 | }
|
---|
| 1561 |
|
---|
| 1562 |
|
---|
[45152] | 1563 | /**
|
---|
| 1564 | * Try enter a critical section with exclusive (write) access.
|
---|
| 1565 | *
|
---|
| 1566 | * @returns VBox status code.
|
---|
| 1567 | * @retval VINF_SUCCESS on success.
|
---|
[58116] | 1568 | * @retval rcBusy if in ring-0 or raw-mode context and it is busy.
|
---|
[45152] | 1569 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 1570 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1571 | * during the operation.
|
---|
| 1572 | *
|
---|
[90347] | 1573 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1574 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1575 | * @param rcBusy The status code to return when we're in RC or R0 and the
|
---|
| 1576 | * section is busy. Pass VINF_SUCCESS to acquired the
|
---|
| 1577 | * critical section thru a ring-3 call if necessary.
|
---|
| 1578 | * @param uId Where we're entering the section.
|
---|
[58116] | 1579 | * @param SRC_POS The source position.
|
---|
[45152] | 1580 | * @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExcl,
|
---|
| 1581 | * PDMCritSectRwTryEnterExclDebug,
|
---|
| 1582 | * PDMCritSectEnterDebug, PDMCritSectEnter,
|
---|
| 1583 | * RTCritSectRwEnterExclDebug.
|
---|
| 1584 | */
|
---|
[90347] | 1585 | VMMDECL(int) PDMCritSectRwEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
|
---|
[25663] | 1586 | {
|
---|
[49486] | 1587 | NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
|
---|
[45293] | 1588 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 1589 | return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL, false /*fNoVal*/);
|
---|
[45293] | 1590 | #else
|
---|
[25663] | 1591 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
|
---|
[90347] | 1592 | return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
|
---|
[45293] | 1593 | #endif
|
---|
[25663] | 1594 | }
|
---|
| 1595 |
|
---|
| 1596 |
|
---|
[45152] | 1597 | /**
|
---|
| 1598 | * Try enter a critical section with exclusive (write) access.
|
---|
| 1599 | *
|
---|
| 1600 | * @retval VINF_SUCCESS on success.
|
---|
| 1601 | * @retval VERR_SEM_BUSY if the critsect was owned.
|
---|
| 1602 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 1603 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1604 | * during the operation.
|
---|
| 1605 | *
|
---|
[90347] | 1606 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1607 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1608 | * @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExclDebug,
|
---|
| 1609 | * PDMCritSectRwEnterExclDebug,
|
---|
| 1610 | * PDMCritSectTryEnter, PDMCritSectTryEnterDebug,
|
---|
| 1611 | * RTCritSectRwTryEnterExcl.
|
---|
| 1612 | */
|
---|
[90347] | 1613 | VMMDECL(int) PDMCritSectRwTryEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[25663] | 1614 | {
|
---|
[45293] | 1615 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 1616 | return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL, false /*fNoVal*/);
|
---|
[25663] | 1617 | #else
|
---|
| 1618 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
|
---|
[90347] | 1619 | return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
|
---|
[25663] | 1620 | #endif
|
---|
| 1621 | }
|
---|
| 1622 |
|
---|
| 1623 |
|
---|
[45152] | 1624 | /**
|
---|
| 1625 | * Try enter a critical section with exclusive (write) access.
|
---|
| 1626 | *
|
---|
| 1627 | * @retval VINF_SUCCESS on success.
|
---|
| 1628 | * @retval VERR_SEM_BUSY if the critsect was owned.
|
---|
| 1629 | * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
|
---|
| 1630 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1631 | * during the operation.
|
---|
| 1632 | *
|
---|
[90347] | 1633 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1634 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1635 | * @param uId Where we're entering the section.
|
---|
[58116] | 1636 | * @param SRC_POS The source position.
|
---|
[45152] | 1637 | * @sa PDMCritSectRwTryEnterExcl, PDMCritSectRwEnterExcl,
|
---|
| 1638 | * PDMCritSectRwEnterExclDebug,
|
---|
| 1639 | * PDMCritSectTryEnterDebug, PDMCritSectTryEnter,
|
---|
| 1640 | * RTCritSectRwTryEnterExclDebug.
|
---|
| 1641 | */
|
---|
[90347] | 1642 | VMMDECL(int) PDMCritSectRwTryEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
|
---|
[25663] | 1643 | {
|
---|
[49486] | 1644 | NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
|
---|
[45293] | 1645 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
[90347] | 1646 | return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL, false /*fNoVal*/);
|
---|
[45293] | 1647 | #else
|
---|
[25663] | 1648 | RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
|
---|
[90347] | 1649 | return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
|
---|
[45293] | 1650 | #endif
|
---|
[25663] | 1651 | }
|
---|
| 1652 |
|
---|
| 1653 |
|
---|
[45293] | 1654 | #ifdef IN_RING3
|
---|
[45152] | 1655 | /**
|
---|
[45293] | 1656 | * Enters a PDM read/write critical section with exclusive (write) access.
|
---|
| 1657 | *
|
---|
| 1658 | * @returns VINF_SUCCESS if entered successfully.
|
---|
| 1659 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1660 | * during the operation.
|
---|
| 1661 | *
|
---|
[90347] | 1662 | * @param pVM The cross context VM structure.
|
---|
[45293] | 1663 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1664 | * @param fCallRing3 Whether this is a VMMRZCallRing3()request.
|
---|
| 1665 | */
|
---|
[90347] | 1666 | VMMR3DECL(int) PDMR3CritSectRwEnterExclEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3)
|
---|
[45293] | 1667 | {
|
---|
[90347] | 1668 | return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3 /*fNoVal*/);
|
---|
[45293] | 1669 | }
|
---|
| 1670 | #endif /* IN_RING3 */
|
---|
| 1671 |
|
---|
| 1672 |
|
---|
| 1673 | /**
|
---|
[45152] | 1674 | * Leave a critical section held exclusively.
|
---|
| 1675 | *
|
---|
| 1676 | * @returns VBox status code.
|
---|
| 1677 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1678 | * during the operation.
|
---|
[90347] | 1679 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1680 | * @param pThis Pointer to the read/write critical section.
|
---|
[45299] | 1681 | * @param fNoVal No validation records (i.e. queued release).
|
---|
[45152] | 1682 | * @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
|
---|
| 1683 | */
|
---|
[90347] | 1684 | static int pdmCritSectRwLeaveExclWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal)
|
---|
[25663] | 1685 | {
|
---|
| 1686 | /*
|
---|
| 1687 | * Validate handle.
|
---|
| 1688 | */
|
---|
[45110] | 1689 | AssertPtr(pThis);
|
---|
[45152] | 1690 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
|
---|
[25663] | 1691 |
|
---|
[57851] | 1692 | #if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
|
---|
| 1693 | NOREF(fNoVal);
|
---|
| 1694 | #endif
|
---|
| 1695 |
|
---|
[90486] | 1696 | /*
|
---|
| 1697 | * Check ownership.
|
---|
| 1698 | */
|
---|
[90347] | 1699 | RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
|
---|
[90486] | 1700 | AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
|
---|
| 1701 |
|
---|
[25663] | 1702 | RTNATIVETHREAD hNativeWriter;
|
---|
[90637] | 1703 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
|
---|
[25663] | 1704 | AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
|
---|
| 1705 |
|
---|
[90665] | 1706 |
|
---|
[25663] | 1707 | /*
|
---|
[90658] | 1708 | * Unwind one recursion. Not the last?
|
---|
[25663] | 1709 | */
|
---|
[90658] | 1710 | if (pThis->s.Core.cWriteRecursions != 1)
|
---|
[25663] | 1711 | {
|
---|
[45189] | 1712 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[45299] | 1713 | if (fNoVal)
|
---|
| 1714 | Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
|
---|
| 1715 | else
|
---|
| 1716 | {
|
---|
[90658] | 1717 | int rc9 = RTLockValidatorRecExclUnwind(pThis->s.Core.pValidatorWrite);
|
---|
[45299] | 1718 | if (RT_FAILURE(rc9))
|
---|
| 1719 | return rc9;
|
---|
| 1720 | }
|
---|
[25663] | 1721 | #endif
|
---|
[90658] | 1722 | #ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
| 1723 | uint32_t const cDepth = --pThis->s.Core.cWriteRecursions;
|
---|
| 1724 | #else
|
---|
| 1725 | uint32_t const cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions);
|
---|
| 1726 | #endif
|
---|
| 1727 | AssertReturn(cDepth != 0 && cDepth < UINT32_MAX, pdmCritSectRwCorrupted(pThis, "Invalid write recursion value on leave"));
|
---|
| 1728 | return VINF_SUCCESS;
|
---|
| 1729 | }
|
---|
[90650] | 1730 |
|
---|
[90665] | 1731 |
|
---|
[90658] | 1732 | /*
|
---|
| 1733 | * Final recursion.
|
---|
| 1734 | */
|
---|
| 1735 | AssertReturn(pThis->s.Core.cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
|
---|
| 1736 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
| 1737 | if (fNoVal)
|
---|
| 1738 | Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
|
---|
| 1739 | else
|
---|
| 1740 | {
|
---|
| 1741 | int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->s.Core.pValidatorWrite, true);
|
---|
| 1742 | if (RT_FAILURE(rc9))
|
---|
| 1743 | return rc9;
|
---|
| 1744 | }
|
---|
| 1745 | #endif
|
---|
| 1746 |
|
---|
[90665] | 1747 |
|
---|
[90650] | 1748 | #ifdef RTASM_HAVE_CMP_WRITE_U128
|
---|
[90658] | 1749 | /*
|
---|
| 1750 | * See if we can get out w/o any signalling as this is a common case.
|
---|
| 1751 | */
|
---|
| 1752 | if (pdmCritSectRwIsCmpWriteU128Supported())
|
---|
| 1753 | {
|
---|
| 1754 | RTCRITSECTRWSTATE OldState;
|
---|
| 1755 | OldState.s.u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
| 1756 | if (OldState.s.u64State == ((UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)))
|
---|
[90650] | 1757 | {
|
---|
[90658] | 1758 | OldState.s.hNativeWriter = hNativeSelf;
|
---|
| 1759 | AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo));
|
---|
[90650] | 1760 |
|
---|
[90658] | 1761 | RTCRITSECTRWSTATE NewState;
|
---|
| 1762 | NewState.s.u64State = RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
|
---|
| 1763 | NewState.s.hNativeWriter = NIL_RTNATIVETHREAD;
|
---|
[90650] | 1764 |
|
---|
[90654] | 1765 | # ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
[90658] | 1766 | pThis->s.Core.cWriteRecursions = 0;
|
---|
[90654] | 1767 | # else
|
---|
[90658] | 1768 | ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
|
---|
[90654] | 1769 | # endif
|
---|
[90658] | 1770 | STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
|
---|
[90650] | 1771 |
|
---|
[90658] | 1772 | if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128))
|
---|
| 1773 | return VINF_SUCCESS;
|
---|
[90650] | 1774 |
|
---|
[90658] | 1775 | /* bail out. */
|
---|
| 1776 | pThis->s.Core.cWriteRecursions = 1;
|
---|
[90650] | 1777 | }
|
---|
[90658] | 1778 | }
|
---|
[90665] | 1779 | #endif /* RTASM_HAVE_CMP_WRITE_U128 */
|
---|
[90650] | 1780 |
|
---|
[90665] | 1781 |
|
---|
[90660] | 1782 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
[90658] | 1783 | /*
|
---|
| 1784 | * Ring-3: Straight forward, just update the state and if necessary signal waiters.
|
---|
[90659] | 1785 | * Ring-0: Try leave for real, depends on host and context.
|
---|
[90658] | 1786 | */
|
---|
[90660] | 1787 | # ifdef IN_RING0
|
---|
[90659] | 1788 | Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe());
|
---|
| 1789 | PVMCPUCC pVCpu = VMMGetCpu(pVM);
|
---|
| 1790 | if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
|
---|
| 1791 | || VMMRZCallRing3IsEnabled(pVCpu)
|
---|
| 1792 | || RTSemEventIsSignalSafe()
|
---|
| 1793 | || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */
|
---|
| 1794 | && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */
|
---|
| 1795 | && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
|
---|
| 1796 | )
|
---|
[90660] | 1797 | # endif
|
---|
[25663] | 1798 | {
|
---|
[90658] | 1799 | # ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
|
---|
| 1800 | pThis->s.Core.cWriteRecursions = 0;
|
---|
| 1801 | # else
|
---|
| 1802 | ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
|
---|
| 1803 | # endif
|
---|
| 1804 | STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
|
---|
| 1805 | ASMAtomicWriteHandle(&pThis->s.Core.u.s.hNativeWriter, NIL_RTNATIVETHREAD);
|
---|
| 1806 |
|
---|
| 1807 | for (;;)
|
---|
[45299] | 1808 | {
|
---|
[90658] | 1809 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
| 1810 | uint64_t u64OldState = u64State;
|
---|
| 1811 |
|
---|
| 1812 | uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
|
---|
| 1813 | AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on leave"));
|
---|
| 1814 | c--;
|
---|
| 1815 |
|
---|
| 1816 | if ( c > 0
|
---|
| 1817 | || (u64State & RTCSRW_CNT_RD_MASK) == 0)
|
---|
| 1818 | {
|
---|
[90659] | 1819 | /*
|
---|
| 1820 | * Don't change the direction, wake up the next writer if any.
|
---|
| 1821 | */
|
---|
[90658] | 1822 | u64State &= ~RTCSRW_CNT_WR_MASK;
|
---|
| 1823 | u64State |= c << RTCSRW_CNT_WR_SHIFT;
|
---|
| 1824 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
| 1825 | {
|
---|
[90672] | 1826 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl));
|
---|
[90659] | 1827 | int rc;
|
---|
| 1828 | if (c == 0)
|
---|
| 1829 | rc = VINF_SUCCESS;
|
---|
[90660] | 1830 | # ifdef IN_RING0
|
---|
| 1831 | else if (!RTSemEventIsSignalSafe() && pVCpu != NULL)
|
---|
[90658] | 1832 | {
|
---|
[90659] | 1833 | VMMR0EMTBLOCKCTX Ctx;
|
---|
| 1834 | rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
|
---|
| 1835 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
|
---|
| 1836 |
|
---|
| 1837 | rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
|
---|
| 1838 |
|
---|
| 1839 | VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
|
---|
[90658] | 1840 | }
|
---|
[90660] | 1841 | # endif
|
---|
| 1842 | else
|
---|
| 1843 | rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
|
---|
[90659] | 1844 | AssertRC(rc);
|
---|
| 1845 | return rc;
|
---|
[90658] | 1846 | }
|
---|
| 1847 | }
|
---|
| 1848 | else
|
---|
| 1849 | {
|
---|
[90659] | 1850 | /*
|
---|
| 1851 | * Reverse the direction and signal the reader threads.
|
---|
| 1852 | */
|
---|
[90658] | 1853 | u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
|
---|
| 1854 | u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT;
|
---|
| 1855 | if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
|
---|
| 1856 | {
|
---|
| 1857 | Assert(!pThis->s.Core.fNeedReset);
|
---|
| 1858 | ASMAtomicWriteBool(&pThis->s.Core.fNeedReset, true);
|
---|
[90672] | 1859 | STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl));
|
---|
[90659] | 1860 |
|
---|
| 1861 | int rc;
|
---|
[90660] | 1862 | # ifdef IN_RING0
|
---|
| 1863 | if (!RTSemEventMultiIsSignalSafe() && pVCpu != NULL)
|
---|
[90659] | 1864 | {
|
---|
| 1865 | VMMR0EMTBLOCKCTX Ctx;
|
---|
| 1866 | rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
|
---|
| 1867 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
|
---|
| 1868 |
|
---|
| 1869 | rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
|
---|
| 1870 |
|
---|
| 1871 | VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
|
---|
| 1872 | }
|
---|
[90660] | 1873 | else
|
---|
| 1874 | # endif
|
---|
| 1875 | rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
|
---|
[90658] | 1876 | AssertRC(rc);
|
---|
[90659] | 1877 | return rc;
|
---|
[90658] | 1878 | }
|
---|
| 1879 | }
|
---|
| 1880 |
|
---|
| 1881 | ASMNopPause();
|
---|
[90659] | 1882 | if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)
|
---|
| 1883 | { /*likely*/ }
|
---|
| 1884 | else
|
---|
[90658] | 1885 | return VERR_SEM_DESTROYED;
|
---|
[90659] | 1886 | ASMNopPause();
|
---|
[45299] | 1887 | }
|
---|
[90659] | 1888 | /* not reached! */
|
---|
[25663] | 1889 | }
|
---|
[90660] | 1890 | #endif /* IN_RING3 || IN_RING0 */
|
---|
[25663] | 1891 |
|
---|
[90665] | 1892 |
|
---|
[90658] | 1893 | #ifndef IN_RING3
|
---|
| 1894 | /*
|
---|
| 1895 | * Queue the requested exit for ring-3 execution.
|
---|
| 1896 | */
|
---|
[90659] | 1897 | # ifndef IN_RING0
|
---|
[90658] | 1898 | PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
|
---|
[90659] | 1899 | # endif
|
---|
[90658] | 1900 | uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves++;
|
---|
| 1901 | LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3\n", i, pThis));
|
---|
| 1902 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves),
|
---|
| 1903 | ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE);
|
---|
| 1904 | pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] = pThis->s.pSelfR3;
|
---|
| 1905 | VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
|
---|
| 1906 | RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i])
|
---|
[93554] | 1907 | && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] & HOST_PAGE_OFFSET_MASK)
|
---|
| 1908 | == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK),
|
---|
[90658] | 1909 | ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i], pThis),
|
---|
| 1910 | pdmCritSectRwCorrupted(pThis, "Invalid self pointer on queue (excl)"));
|
---|
| 1911 | VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
|
---|
| 1912 | VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
|
---|
| 1913 | STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
|
---|
| 1914 | STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveExcl);
|
---|
[25663] | 1915 | return VINF_SUCCESS;
|
---|
[90658] | 1916 | #endif
|
---|
[25431] | 1917 | }
|
---|
| 1918 |
|
---|
| 1919 |
|
---|
[45152] | 1920 | /**
|
---|
[45299] | 1921 | * Leave a critical section held exclusively.
|
---|
| 1922 | *
|
---|
| 1923 | * @returns VBox status code.
|
---|
| 1924 | * @retval VERR_SEM_DESTROYED if the critical section is delete before or
|
---|
| 1925 | * during the operation.
|
---|
[90347] | 1926 | * @param pVM The cross context VM structure.
|
---|
[45299] | 1927 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1928 | * @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
|
---|
| 1929 | */
|
---|
[90347] | 1930 | VMMDECL(int) PDMCritSectRwLeaveExcl(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[45299] | 1931 | {
|
---|
[90347] | 1932 | return pdmCritSectRwLeaveExclWorker(pVM, pThis, false /*fNoVal*/);
|
---|
[45299] | 1933 | }
|
---|
| 1934 |
|
---|
| 1935 |
|
---|
| 1936 | #if defined(IN_RING3) || defined(IN_RING0)
|
---|
| 1937 | /**
|
---|
| 1938 | * PDMCritSectBothFF interface.
|
---|
| 1939 | *
|
---|
[90347] | 1940 | * @param pVM The cross context VM structure.
|
---|
[45299] | 1941 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1942 | */
|
---|
[90347] | 1943 | void pdmCritSectRwLeaveExclQueued(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[45299] | 1944 | {
|
---|
[90347] | 1945 | pdmCritSectRwLeaveExclWorker(pVM, pThis, true /*fNoVal*/);
|
---|
[45299] | 1946 | }
|
---|
| 1947 | #endif
|
---|
| 1948 |
|
---|
| 1949 |
|
---|
| 1950 | /**
|
---|
[45152] | 1951 | * Checks the caller is the exclusive (write) owner of the critical section.
|
---|
| 1952 | *
|
---|
[58116] | 1953 | * @retval true if owner.
|
---|
| 1954 | * @retval false if not owner.
|
---|
[90347] | 1955 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1956 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1957 | * @sa PDMCritSectRwIsReadOwner, PDMCritSectIsOwner,
|
---|
| 1958 | * RTCritSectRwIsWriteOwner.
|
---|
| 1959 | */
|
---|
[90347] | 1960 | VMMDECL(bool) PDMCritSectRwIsWriteOwner(PVMCC pVM, PPDMCRITSECTRW pThis)
|
---|
[25426] | 1961 | {
|
---|
[25431] | 1962 | /*
|
---|
[25663] | 1963 | * Validate handle.
|
---|
[25431] | 1964 | */
|
---|
[45110] | 1965 | AssertPtr(pThis);
|
---|
[45152] | 1966 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
|
---|
[25431] | 1967 |
|
---|
[25663] | 1968 | /*
|
---|
| 1969 | * Check ownership.
|
---|
| 1970 | */
|
---|
| 1971 | RTNATIVETHREAD hNativeWriter;
|
---|
[90637] | 1972 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
|
---|
[45152] | 1973 | if (hNativeWriter == NIL_RTNATIVETHREAD)
|
---|
| 1974 | return false;
|
---|
[90347] | 1975 | return hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis);
|
---|
[25426] | 1976 | }
|
---|
| 1977 |
|
---|
| 1978 |
|
---|
[45152] | 1979 | /**
|
---|
| 1980 | * Checks if the caller is one of the read owners of the critical section.
|
---|
| 1981 | *
|
---|
| 1982 | * @note !CAUTION! This API doesn't work reliably if lock validation isn't
|
---|
| 1983 | * enabled. Meaning, the answer is not trustworhty unless
|
---|
| 1984 | * RT_LOCK_STRICT or PDMCRITSECTRW_STRICT was defined at build time.
|
---|
| 1985 | * Also, make sure you do not use RTCRITSECTRW_FLAGS_NO_LOCK_VAL when
|
---|
| 1986 | * creating the semaphore. And finally, if you used a locking class,
|
---|
| 1987 | * don't disable deadlock detection by setting cMsMinDeadlock to
|
---|
| 1988 | * RT_INDEFINITE_WAIT.
|
---|
| 1989 | *
|
---|
| 1990 | * In short, only use this for assertions.
|
---|
| 1991 | *
|
---|
| 1992 | * @returns @c true if reader, @c false if not.
|
---|
[90347] | 1993 | * @param pVM The cross context VM structure.
|
---|
[45152] | 1994 | * @param pThis Pointer to the read/write critical section.
|
---|
| 1995 | * @param fWannaHear What you'd like to hear when lock validation is not
|
---|
| 1996 | * available. (For avoiding asserting all over the place.)
|
---|
| 1997 | * @sa PDMCritSectRwIsWriteOwner, RTCritSectRwIsReadOwner.
|
---|
| 1998 | */
|
---|
[90347] | 1999 | VMMDECL(bool) PDMCritSectRwIsReadOwner(PVMCC pVM, PPDMCRITSECTRW pThis, bool fWannaHear)
|
---|
[25908] | 2000 | {
|
---|
| 2001 | /*
|
---|
| 2002 | * Validate handle.
|
---|
| 2003 | */
|
---|
[45110] | 2004 | AssertPtr(pThis);
|
---|
[45152] | 2005 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
|
---|
[25908] | 2006 |
|
---|
| 2007 | /*
|
---|
| 2008 | * Inspect the state.
|
---|
| 2009 | */
|
---|
[90654] | 2010 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[45110] | 2011 | if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
|
---|
[25908] | 2012 | {
|
---|
| 2013 | /*
|
---|
| 2014 | * It's in write mode, so we can only be a reader if we're also the
|
---|
| 2015 | * current writer.
|
---|
| 2016 | */
|
---|
| 2017 | RTNATIVETHREAD hWriter;
|
---|
[90637] | 2018 | ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hWriter);
|
---|
[45152] | 2019 | if (hWriter == NIL_RTNATIVETHREAD)
|
---|
| 2020 | return false;
|
---|
[90347] | 2021 | return hWriter == pdmCritSectRwGetNativeSelf(pVM, pThis);
|
---|
[25908] | 2022 | }
|
---|
| 2023 |
|
---|
| 2024 | /*
|
---|
| 2025 | * Read mode. If there are no current readers, then we cannot be a reader.
|
---|
| 2026 | */
|
---|
[45110] | 2027 | if (!(u64State & RTCSRW_CNT_RD_MASK))
|
---|
[25908] | 2028 | return false;
|
---|
| 2029 |
|
---|
[45152] | 2030 | #if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
|
---|
[25908] | 2031 | /*
|
---|
| 2032 | * Ask the lock validator.
|
---|
[45152] | 2033 | * Note! It doesn't know everything, let's deal with that if it becomes an issue...
|
---|
[25908] | 2034 | */
|
---|
[62659] | 2035 | NOREF(fWannaHear);
|
---|
[45152] | 2036 | return RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
|
---|
[25908] | 2037 | #else
|
---|
| 2038 | /*
|
---|
| 2039 | * Ok, we don't know, just tell the caller what he want to hear.
|
---|
| 2040 | */
|
---|
| 2041 | return fWannaHear;
|
---|
| 2042 | #endif
|
---|
| 2043 | }
|
---|
| 2044 |
|
---|
| 2045 |
|
---|
[45152] | 2046 | /**
|
---|
| 2047 | * Gets the write recursion count.
|
---|
| 2048 | *
|
---|
| 2049 | * @returns The write recursion count (0 if bad critsect).
|
---|
| 2050 | * @param pThis Pointer to the read/write critical section.
|
---|
| 2051 | * @sa PDMCritSectRwGetWriterReadRecursion, PDMCritSectRwGetReadCount,
|
---|
| 2052 | * RTCritSectRwGetWriteRecursion.
|
---|
| 2053 | */
|
---|
| 2054 | VMMDECL(uint32_t) PDMCritSectRwGetWriteRecursion(PPDMCRITSECTRW pThis)
|
---|
[25426] | 2055 | {
|
---|
[25431] | 2056 | /*
|
---|
[25663] | 2057 | * Validate handle.
|
---|
[25431] | 2058 | */
|
---|
[45110] | 2059 | AssertPtr(pThis);
|
---|
[45152] | 2060 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
|
---|
[25431] | 2061 |
|
---|
[25663] | 2062 | /*
|
---|
| 2063 | * Return the requested data.
|
---|
| 2064 | */
|
---|
[45152] | 2065 | return pThis->s.Core.cWriteRecursions;
|
---|
[25426] | 2066 | }
|
---|
| 2067 |
|
---|
| 2068 |
|
---|
[45152] | 2069 | /**
|
---|
| 2070 | * Gets the read recursion count of the current writer.
|
---|
| 2071 | *
|
---|
| 2072 | * @returns The read recursion count (0 if bad critsect).
|
---|
| 2073 | * @param pThis Pointer to the read/write critical section.
|
---|
| 2074 | * @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetReadCount,
|
---|
| 2075 | * RTCritSectRwGetWriterReadRecursion.
|
---|
| 2076 | */
|
---|
| 2077 | VMMDECL(uint32_t) PDMCritSectRwGetWriterReadRecursion(PPDMCRITSECTRW pThis)
|
---|
[25426] | 2078 | {
|
---|
[25431] | 2079 | /*
|
---|
[25663] | 2080 | * Validate handle.
|
---|
[25431] | 2081 | */
|
---|
[45110] | 2082 | AssertPtr(pThis);
|
---|
[45152] | 2083 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
|
---|
[25431] | 2084 |
|
---|
[25663] | 2085 | /*
|
---|
| 2086 | * Return the requested data.
|
---|
| 2087 | */
|
---|
[45152] | 2088 | return pThis->s.Core.cWriterReads;
|
---|
[25426] | 2089 | }
|
---|
| 2090 |
|
---|
| 2091 |
|
---|
[45152] | 2092 | /**
|
---|
| 2093 | * Gets the current number of reads.
|
---|
| 2094 | *
|
---|
| 2095 | * This includes all read recursions, so it might be higher than the number of
|
---|
| 2096 | * read owners. It does not include reads done by the current writer.
|
---|
| 2097 | *
|
---|
| 2098 | * @returns The read count (0 if bad critsect).
|
---|
| 2099 | * @param pThis Pointer to the read/write critical section.
|
---|
| 2100 | * @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetWriterReadRecursion,
|
---|
| 2101 | * RTCritSectRwGetReadCount.
|
---|
| 2102 | */
|
---|
| 2103 | VMMDECL(uint32_t) PDMCritSectRwGetReadCount(PPDMCRITSECTRW pThis)
|
---|
[25426] | 2104 | {
|
---|
[25431] | 2105 | /*
|
---|
| 2106 | * Validate input.
|
---|
| 2107 | */
|
---|
[45110] | 2108 | AssertPtr(pThis);
|
---|
[45152] | 2109 | AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
|
---|
[25431] | 2110 |
|
---|
[25663] | 2111 | /*
|
---|
| 2112 | * Return the requested data.
|
---|
| 2113 | */
|
---|
[90654] | 2114 | uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
|
---|
[45110] | 2115 | if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
|
---|
[25663] | 2116 | return 0;
|
---|
[45110] | 2117 | return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
|
---|
[25426] | 2118 | }
|
---|
| 2119 |
|
---|
[45110] | 2120 |
|
---|
[45152] | 2121 | /**
|
---|
| 2122 | * Checks if the read/write critical section is initialized or not.
|
---|
| 2123 | *
|
---|
[58116] | 2124 | * @retval true if initialized.
|
---|
| 2125 | * @retval false if not initialized.
|
---|
[45152] | 2126 | * @param pThis Pointer to the read/write critical section.
|
---|
| 2127 | * @sa PDMCritSectIsInitialized, RTCritSectRwIsInitialized.
|
---|
| 2128 | */
|
---|
| 2129 | VMMDECL(bool) PDMCritSectRwIsInitialized(PCPDMCRITSECTRW pThis)
|
---|
[45110] | 2130 | {
|
---|
| 2131 | AssertPtr(pThis);
|
---|
[45152] | 2132 | return pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC;
|
---|
[45110] | 2133 | }
|
---|
| 2134 |
|
---|