VirtualBox

source: vbox/trunk/src/VBox/Runtime/generic/semrw-lockless-generic.cpp

Last change on this file was 98103, checked in by vboxsync, 16 months ago

Copyright year updates by scm.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 34.4 KB
Line 
1/* $Id: semrw-lockless-generic.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2/** @file
3 * IPRT - Read-Write Semaphore, Generic, lockless variant.
4 */
5
6/*
7 * Copyright (C) 2009-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/*********************************************************************************************************************************
39* Header Files *
40*********************************************************************************************************************************/
41#define RTSEMRW_WITHOUT_REMAPPING
42#define RTASSERT_QUIET
43#include <iprt/semaphore.h>
44#include "internal/iprt.h"
45
46#include <iprt/asm.h>
47#include <iprt/assert.h>
48#include <iprt/err.h>
49#include <iprt/lockvalidator.h>
50#include <iprt/mem.h>
51#include <iprt/thread.h>
52
53#include "internal/magics.h"
54#include "internal/strict.h"
55
56
57/*********************************************************************************************************************************
58* Structures and Typedefs *
59*********************************************************************************************************************************/
60typedef struct RTSEMRWINTERNAL
61{
62 /** Magic value (RTSEMRW_MAGIC). */
63 uint32_t volatile u32Magic;
64 /** Indicates whether hEvtRead needs resetting. */
65 bool volatile fNeedReset;
66
67 /** The state variable.
68 * All accesses are atomic and it bits are defined like this:
69 * Bits 0..14 - cReads.
70 * Bit 15 - Unused.
71 * Bits 16..31 - cWrites. - doesn't make sense here
72 * Bit 31 - fDirection; 0=Read, 1=Write.
73 * Bits 32..46 - cWaitingReads
74 * Bit 47 - Unused.
75 * Bits 48..62 - cWaitingWrites
76 * Bit 63 - Unused.
77 */
78 uint64_t volatile u64State;
79 /** The write owner. */
80 RTNATIVETHREAD volatile hNativeWriter;
81 /** The number of reads made by the current writer. */
82 uint32_t volatile cWriterReads;
83 /** The number of recursions made by the current writer. (The initial grabbing
84 * of the lock counts as the first one.) */
85 uint32_t volatile cWriteRecursions;
86
87 /** What the writer threads are blocking on. */
88 RTSEMEVENT hEvtWrite;
89 /** What the read threads are blocking on when waiting for the writer to
90 * finish. */
91 RTSEMEVENTMULTI hEvtRead;
92
93#ifdef RTSEMRW_STRICT
94 /** The validator record for the writer. */
95 RTLOCKVALRECEXCL ValidatorWrite;
96 /** The validator record for the readers. */
97 RTLOCKVALRECSHRD ValidatorRead;
98#endif
99} RTSEMRWINTERNAL;
100
101
102/*********************************************************************************************************************************
103* Defined Constants And Macros *
104*********************************************************************************************************************************/
105#define RTSEMRW_CNT_BITS 15
106#define RTSEMRW_CNT_MASK UINT64_C(0x00007fff)
107
108#define RTSEMRW_CNT_RD_SHIFT 0
109#define RTSEMRW_CNT_RD_MASK (RTSEMRW_CNT_MASK << RTSEMRW_CNT_RD_SHIFT)
110#define RTSEMRW_CNT_WR_SHIFT 16
111#define RTSEMRW_CNT_WR_MASK (RTSEMRW_CNT_MASK << RTSEMRW_CNT_WR_SHIFT)
112#define RTSEMRW_DIR_SHIFT 31
113#define RTSEMRW_DIR_MASK RT_BIT_64(RTSEMRW_DIR_SHIFT)
114#define RTSEMRW_DIR_READ UINT64_C(0)
115#define RTSEMRW_DIR_WRITE UINT64_C(1)
116
117#define RTSEMRW_WAIT_CNT_RD_SHIFT 32
118#define RTSEMRW_WAIT_CNT_RD_MASK (RTSEMRW_CNT_MASK << RTSEMRW_WAIT_CNT_RD_SHIFT)
119//#define RTSEMRW_WAIT_CNT_WR_SHIFT 48
120//#define RTSEMRW_WAIT_CNT_WR_MASK (RTSEMRW_CNT_MASK << RTSEMRW_WAIT_CNT_WR_SHIFT)
121
122
123RTDECL(int) RTSemRWCreate(PRTSEMRW phRWSem)
124{
125 return RTSemRWCreateEx(phRWSem, 0 /*fFlags*/, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "RTSemRW");
126}
127RT_EXPORT_SYMBOL(RTSemRWCreate);
128
129
130RTDECL(int) RTSemRWCreateEx(PRTSEMRW phRWSem, uint32_t fFlags,
131 RTLOCKVALCLASS hClass, uint32_t uSubClass, const char *pszNameFmt, ...)
132{
133 AssertReturn(!(fFlags & ~RTSEMRW_FLAGS_NO_LOCK_VAL), VERR_INVALID_PARAMETER);
134
135 RTSEMRWINTERNAL *pThis = (RTSEMRWINTERNAL *)RTMemAlloc(sizeof(*pThis));
136 if (!pThis)
137 return VERR_NO_MEMORY;
138
139 int rc = RTSemEventMultiCreate(&pThis->hEvtRead);
140 if (RT_SUCCESS(rc))
141 {
142 rc = RTSemEventCreate(&pThis->hEvtWrite);
143 if (RT_SUCCESS(rc))
144 {
145 pThis->u32Magic = RTSEMRW_MAGIC;
146 pThis->u32Padding = 0;
147 pThis->u64State = 0;
148 pThis->hNativeWriter = NIL_RTNATIVETHREAD;
149 pThis->cWriterReads = 0;
150 pThis->cWriteRecursions = 0;
151 pThis->fNeedReset = false;
152#ifdef RTSEMRW_STRICT
153 bool const fLVEnabled = !(fFlags & RTSEMRW_FLAGS_NO_LOCK_VAL);
154 if (!pszNameFmt)
155 {
156 static uint32_t volatile s_iSemRWAnon = 0;
157 uint32_t i = ASMAtomicIncU32(&s_iSemRWAnon) - 1;
158 RTLockValidatorRecExclInit(&pThis->ValidatorWrite, hClass, uSubClass, pThis,
159 fLVEnabled, "RTSemRW-%u", i);
160 RTLockValidatorRecSharedInit(&pThis->ValidatorRead, hClass, uSubClass, pThis,
161 false /*fSignaller*/, fLVEnabled, "RTSemRW-%u", i);
162 }
163 else
164 {
165 va_list va;
166 va_start(va, pszNameFmt);
167 RTLockValidatorRecExclInitV(&pThis->ValidatorWrite, hClass, uSubClass, pThis,
168 fLVEnabled, pszNameFmt, va);
169 va_end(va);
170 va_start(va, pszNameFmt);
171 RTLockValidatorRecSharedInitV(&pThis->ValidatorRead, hClass, uSubClass, pThis,
172 false /*fSignaller*/, fLVEnabled, pszNameFmt, va);
173 va_end(va);
174 }
175 RTLockValidatorRecMakeSiblings(&pThis->ValidatorWrite.Core, &pThis->ValidatorRead.Core);
176#endif
177
178 *phRWSem = pThis;
179 return VINF_SUCCESS;
180 }
181 RTSemEventMultiDestroy(pThis->hEvtRead);
182 }
183 return rc;
184}
185RT_EXPORT_SYMBOL(RTSemRWCreateEx);
186
187
188RTDECL(int) RTSemRWDestroy(RTSEMRW hRWSem)
189{
190 /*
191 * Validate input.
192 */
193 RTSEMRWINTERNAL *pThis = hRWSem;
194 if (pThis == NIL_RTSEMRW)
195 return VINF_SUCCESS;
196 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
197 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
198 Assert(!(ASMAtomicReadU64(&pThis->u64State) & (RTSEMRW_CNT_RD_MASK | RTSEMRW_CNT_WR_MASK)));
199
200 /*
201 * Invalidate the object and free up the resources.
202 */
203 AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, ~RTSEMRW_MAGIC, RTSEMRW_MAGIC), VERR_INVALID_HANDLE);
204
205 RTSEMEVENTMULTI hEvtRead;
206 ASMAtomicXchgHandle(&pThis->hEvtRead, NIL_RTSEMEVENTMULTI, &hEvtRead);
207 int rc = RTSemEventMultiDestroy(hEvtRead);
208 AssertRC(rc);
209
210 RTSEMEVENT hEvtWrite;
211 ASMAtomicXchgHandle(&pThis->hEvtWrite, NIL_RTSEMEVENT, &hEvtWrite);
212 rc = RTSemEventDestroy(hEvtWrite);
213 AssertRC(rc);
214
215#ifdef RTSEMRW_STRICT
216 RTLockValidatorRecSharedDelete(&pThis->ValidatorRead);
217 RTLockValidatorRecExclDelete(&pThis->ValidatorWrite);
218#endif
219 RTMemFree(pThis);
220 return VINF_SUCCESS;
221}
222RT_EXPORT_SYMBOL(RTSemRWDestroy);
223
224
225RTDECL(uint32_t) RTSemRWSetSubClass(RTSEMRW hRWSem, uint32_t uSubClass)
226{
227#ifdef RTSEMRW_STRICT
228 /*
229 * Validate handle.
230 */
231 struct RTSEMRWINTERNAL *pThis = hRWSem;
232 AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
233 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
234
235 RTLockValidatorRecSharedSetSubClass(&pThis->ValidatorRead, uSubClass);
236 return RTLockValidatorRecExclSetSubClass(&pThis->ValidatorWrite, uSubClass);
237#else
238 return RTLOCKVAL_SUB_CLASS_INVALID;
239#endif
240}
241RT_EXPORT_SYMBOL(RTSemRWSetSubClass);
242
243
244static int rtSemRWRequestRead(RTSEMRW hRWSem, RTMSINTERVAL cMillies, bool fInterruptible, PCRTLOCKVALSRCPOS pSrcPos)
245{
246 /*
247 * Validate input.
248 */
249 RTSEMRWINTERNAL *pThis = hRWSem;
250 if (pThis == NIL_RTSEMRW)
251 return VINF_SUCCESS;
252 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
253 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
254
255#ifdef RTSEMRW_STRICT
256 RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
257 if (cMillies > 0)
258 {
259 int rc9;
260 RTNATIVETHREAD hNativeWriter;
261 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
262 if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == RTThreadNativeSelf())
263 rc9 = RTLockValidatorRecExclCheckOrder(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, cMillies);
264 else
265 rc9 = RTLockValidatorRecSharedCheckOrder(&pThis->ValidatorRead, hThreadSelf, pSrcPos, cMillies);
266 if (RT_FAILURE(rc9))
267 return rc9;
268 }
269#endif
270
271 /*
272 * Get cracking...
273 */
274 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
275 uint64_t u64OldState = u64State;
276
277 for (;;)
278 {
279 if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
280 {
281 /* It flows in the right direction, try follow it before it changes. */
282 uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
283 c++;
284 Assert(c < RTSEMRW_CNT_MASK / 2);
285 u64State &= ~RTSEMRW_CNT_RD_MASK;
286 u64State |= c << RTSEMRW_CNT_RD_SHIFT;
287 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
288 {
289#ifdef RTSEMRW_STRICT
290 RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
291#endif
292 break;
293 }
294 }
295 else if ((u64State & (RTSEMRW_CNT_RD_MASK | RTSEMRW_CNT_WR_MASK)) == 0)
296 {
297 /* Wrong direction, but we're alone here and can simply try switch the direction. */
298 u64State &= ~(RTSEMRW_CNT_RD_MASK | RTSEMRW_CNT_WR_MASK | RTSEMRW_DIR_MASK);
299 u64State |= (UINT64_C(1) << RTSEMRW_CNT_RD_SHIFT) | (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT);
300 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
301 {
302 Assert(!pThis->fNeedReset);
303#ifdef RTSEMRW_STRICT
304 RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
305#endif
306 break;
307 }
308 }
309 else
310 {
311 /* Is the writer perhaps doing a read recursion? */
312 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
313 RTNATIVETHREAD hNativeWriter;
314 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
315 if (hNativeSelf == hNativeWriter)
316 {
317#ifdef RTSEMRW_STRICT
318 int rc9 = RTLockValidatorRecExclRecursionMixed(&pThis->ValidatorWrite, &pThis->ValidatorRead.Core, pSrcPos);
319 if (RT_FAILURE(rc9))
320 return rc9;
321#endif
322 Assert(pThis->cWriterReads < UINT32_MAX / 2);
323 ASMAtomicIncU32(&pThis->cWriterReads);
324 return VINF_SUCCESS; /* don't break! */
325 }
326
327 /* If the timeout is 0, return already. */
328 if (!cMillies)
329 return VERR_TIMEOUT;
330
331 /* Add ourselves to the queue and wait for the direction to change. */
332 uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
333 c++;
334 Assert(c < RTSEMRW_CNT_MASK / 2);
335
336 uint64_t cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT;
337 cWait++;
338 Assert(cWait <= c);
339 Assert(cWait < RTSEMRW_CNT_MASK / 2);
340
341 u64State &= ~(RTSEMRW_CNT_RD_MASK | RTSEMRW_WAIT_CNT_RD_MASK);
342 u64State |= (c << RTSEMRW_CNT_RD_SHIFT) | (cWait << RTSEMRW_WAIT_CNT_RD_SHIFT);
343
344 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
345 {
346 for (uint32_t iLoop = 0; ; iLoop++)
347 {
348 int rc;
349#ifdef RTSEMRW_STRICT
350 rc = RTLockValidatorRecSharedCheckBlocking(&pThis->ValidatorRead, hThreadSelf, pSrcPos, true,
351 cMillies, RTTHREADSTATE_RW_READ, false);
352 if (RT_SUCCESS(rc))
353#else
354 RTTHREAD hThreadSelf = RTThreadSelf();
355 RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
356#endif
357 {
358 if (fInterruptible)
359 rc = RTSemEventMultiWaitNoResume(pThis->hEvtRead, cMillies);
360 else
361 rc = RTSemEventMultiWait(pThis->hEvtRead, cMillies);
362 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
363 if (pThis->u32Magic != RTSEMRW_MAGIC)
364 return VERR_SEM_DESTROYED;
365 }
366 if (RT_FAILURE(rc))
367 {
368 /* Decrement the counts and return the error. */
369 for (;;)
370 {
371 u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
372 c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT; Assert(c > 0);
373 c--;
374 cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0);
375 cWait--;
376 u64State &= ~(RTSEMRW_CNT_RD_MASK | RTSEMRW_WAIT_CNT_RD_MASK);
377 u64State |= (c << RTSEMRW_CNT_RD_SHIFT) | (cWait << RTSEMRW_WAIT_CNT_RD_SHIFT);
378 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
379 break;
380 }
381 return rc;
382 }
383
384 Assert(pThis->fNeedReset);
385 u64State = ASMAtomicReadU64(&pThis->u64State);
386 if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
387 break;
388 AssertMsg(iLoop < 1, ("%u\n", iLoop));
389 }
390
391 /* Decrement the wait count and maybe reset the semaphore (if we're last). */
392 for (;;)
393 {
394 u64OldState = u64State;
395
396 cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT;
397 Assert(cWait > 0);
398 cWait--;
399 u64State &= ~RTSEMRW_WAIT_CNT_RD_MASK;
400 u64State |= cWait << RTSEMRW_WAIT_CNT_RD_SHIFT;
401
402 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
403 {
404 if (cWait == 0)
405 {
406 if (ASMAtomicXchgBool(&pThis->fNeedReset, false))
407 {
408 int rc = RTSemEventMultiReset(pThis->hEvtRead);
409 AssertRCReturn(rc, rc);
410 }
411 }
412 break;
413 }
414 u64State = ASMAtomicReadU64(&pThis->u64State);
415 }
416
417#ifdef RTSEMRW_STRICT
418 RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
419#endif
420 break;
421 }
422 }
423
424 if (pThis->u32Magic != RTSEMRW_MAGIC)
425 return VERR_SEM_DESTROYED;
426
427 ASMNopPause();
428 u64State = ASMAtomicReadU64(&pThis->u64State);
429 u64OldState = u64State;
430 }
431
432 /* got it! */
433 Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT));
434 return VINF_SUCCESS;
435
436}
437
438
439RTDECL(int) RTSemRWRequestRead(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
440{
441#ifndef RTSEMRW_STRICT
442 return rtSemRWRequestRead(hRWSem, cMillies, false, NULL);
443#else
444 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
445 return rtSemRWRequestRead(hRWSem, cMillies, false, &SrcPos);
446#endif
447}
448RT_EXPORT_SYMBOL(RTSemRWRequestRead);
449
450
451RTDECL(int) RTSemRWRequestReadDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
452{
453 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
454 return rtSemRWRequestRead(hRWSem, cMillies, false, &SrcPos);
455}
456RT_EXPORT_SYMBOL(RTSemRWRequestReadDebug);
457
458
459RTDECL(int) RTSemRWRequestReadNoResume(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
460{
461#ifndef RTSEMRW_STRICT
462 return rtSemRWRequestRead(hRWSem, cMillies, true, NULL);
463#else
464 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
465 return rtSemRWRequestRead(hRWSem, cMillies, true, &SrcPos);
466#endif
467}
468RT_EXPORT_SYMBOL(RTSemRWRequestReadNoResume);
469
470
471RTDECL(int) RTSemRWRequestReadNoResumeDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
472{
473 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
474 return rtSemRWRequestRead(hRWSem, cMillies, true, &SrcPos);
475}
476RT_EXPORT_SYMBOL(RTSemRWRequestReadNoResumeDebug);
477
478
479
480RTDECL(int) RTSemRWReleaseRead(RTSEMRW hRWSem)
481{
482 /*
483 * Validate handle.
484 */
485 RTSEMRWINTERNAL *pThis = hRWSem;
486 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
487 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
488
489 /*
490 * Check the direction and take action accordingly.
491 */
492 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
493 uint64_t u64OldState = u64State;
494 if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
495 {
496#ifdef RTSEMRW_STRICT
497 int rc9 = RTLockValidatorRecSharedCheckAndRelease(&pThis->ValidatorRead, NIL_RTTHREAD);
498 if (RT_FAILURE(rc9))
499 return rc9;
500#endif
501 for (;;)
502 {
503 uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
504 AssertReturn(c > 0, VERR_NOT_OWNER);
505 c--;
506
507 if ( c > 0
508 || (u64State & RTSEMRW_CNT_WD_MASK) == 0)
509 {
510 /* Don't change the direction. */
511 u64State &= ~RTSEMRW_CNT_RD_MASK;
512 u64State |= c << RTSEMRW_CNT_RD_SHIFT;
513 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
514 break;
515 }
516 else
517 {
518 /* Reverse the direction and signal the reader threads. */
519 u64State &= ~(RTSEMRW_CNT_RD_MASK | RTSEMRW_DIR_MASK);
520 u64State |= RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT;
521 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
522 {
523 int rc = RTSemEventSignal(pThis->hEvtWrite);
524 AssertRC(rc);
525 break;
526 }
527 }
528
529 ASMNopPause();
530 u64State = ASMAtomicReadU64(&pThis->u64State);
531 u64OldState = u64State;
532 }
533 }
534 else
535 {
536 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
537 RTNATIVETHREAD hNativeWriter;
538 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
539 AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
540 AssertReturn(pThis->cWriterReads > 0, VERR_NOT_OWNER);
541#ifdef RTSEMRW_STRICT
542 int rc = RTLockValidatorRecExclUnwindMixed(&pThis->ValidatorWrite, &pThis->ValidatorRead.Core);
543 if (RT_FAILURE(rc))
544 return rc;
545#endif
546 ASMAtomicDecU32(&pThis->cWriterReads);
547 }
548
549 return VINF_SUCCESS;
550}
551RT_EXPORT_SYMBOL(RTSemRWReleaseRead);
552
553
554DECL_FORCE_INLINE(int) rtSemRWRequestWrite(RTSEMRW hRWSem, RTMSINTERVAL cMillies, bool fInterruptible, PCRTLOCKVALSRCPOS pSrcPos)
555{
556 /*
557 * Validate input.
558 */
559 RTSEMRWINTERNAL *pThis = hRWSem;
560 if (pThis == NIL_RTSEMRW)
561 return VINF_SUCCESS;
562 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
563 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
564
565#ifdef RTSEMRW_STRICT
566 RTTHREAD hThreadSelf = NIL_RTTHREAD;
567 if (cMillies)
568 {
569 hThreadSelf = RTThreadSelfAutoAdopt();
570 int rc9 = RTLockValidatorRecExclCheckOrder(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, cMillies);
571 if (RT_FAILURE(rc9))
572 return rc9;
573 }
574#endif
575
576 /*
577 * Check if we're already the owner and just recursing.
578 */
579 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
580 RTNATIVETHREAD hNativeWriter;
581 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
582 if (hNativeSelf == hNativeWriter)
583 {
584 Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT));
585#ifdef RTSEMRW_STRICT
586 int rc9 = RTLockValidatorRecExclRecursion(&pThis->ValidatorWrite, pSrcPos);
587 if (RT_FAILURE(rc9))
588 return rc9;
589#endif
590 Assert(pThis->cWriteRecursions < UINT32_MAX / 2);
591 ASMAtomicIncU32(&pThis->cWriteRecursions);
592 return VINF_SUCCESS;
593 }
594
595 /*
596 * Get cracking.
597 */
598 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
599 uint64_t u64OldState = u64State;
600
601 for (;;)
602 {
603 if ( (u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT)
604 || (u64State & (RTSEMRW_CNT_RD_MASK | RTSEMRW_CNT_WR_MASK)) != 0)
605 {
606 /* It flows in the right direction, try follow it before it changes. */
607 uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
608 c++;
609 Assert(c < RTSEMRW_CNT_MASK / 2);
610 u64State &= ~RTSEMRW_CNT_WR_MASK;
611 u64State |= c << RTSEMRW_CNT_WR_SHIFT;
612 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
613 break;
614 }
615 else if ((u64State & (RTSEMRW_CNT_RD_MASK | RTSEMRW_CNT_WR_MASK)) == 0)
616 {
617 /* Wrong direction, but we're alone here and can simply try switch the direction. */
618 u64State &= ~(RTSEMRW_CNT_RD_MASK | RTSEMRW_CNT_WR_MASK | RTSEMRW_DIR_MASK);
619 u64State |= (UINT64_C(1) << RTSEMRW_CNT_WR_SHIFT) | (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT);
620 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
621 break;
622 }
623 else if (!cMillies)
624 /* Wrong direction and we're not supposed to wait, just return. */
625 return VERR_TIMEOUT;
626 else
627 {
628 /* Add ourselves to the write count and break out to do the wait. */
629 uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
630 c++;
631 Assert(c < RTSEMRW_CNT_MASK / 2);
632 u64State &= ~RTSEMRW_CNT_WR_MASK;
633 u64State |= c << RTSEMRW_CNT_WR_SHIFT;
634 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
635 break;
636 }
637
638 if (pThis->u32Magic != RTSEMRW_MAGIC)
639 return VERR_SEM_DESTROYED;
640
641 ASMNopPause();
642 u64State = ASMAtomicReadU64(&pThis->u64State);
643 u64OldState = u64State;
644 }
645
646 /*
647 * If we're in write mode now try grab the ownership. Play fair if there
648 * are threads already waiting.
649 */
650 bool fDone = (u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT)
651 && ( ((u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT) == 1
652 || cMillies == 0);
653 if (fDone)
654 ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
655 if (!fDone)
656 {
657 /*
658 * Wait for our turn.
659 */
660 for (uint32_t iLoop = 0; ; iLoop++)
661 {
662 int rc;
663#ifdef RTSEMRW_STRICT
664 if (cMillies)
665 {
666 if (hThreadSelf == NIL_RTTHREAD)
667 hThreadSelf = RTThreadSelfAutoAdopt();
668 rc = RTLockValidatorRecExclCheckBlocking(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, true,
669 cMillies, RTTHREADSTATE_RW_WRITE, false);
670 }
671 else
672 rc = VINF_SUCCESS;
673 if (RT_SUCCESS(rc))
674#else
675 RTTHREAD hThreadSelf = RTThreadSelf();
676 RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
677#endif
678 {
679 if (fInterruptible)
680 rc = RTSemEventWaitNoResume(pThis->hEvtWrite, cMillies);
681 else
682 rc = RTSemEventWait(pThis->hEvtWrite, cMillies);
683 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
684 if (pThis->u32Magic != RTSEMRW_MAGIC)
685 return VERR_SEM_DESTROYED;
686 }
687 if (RT_FAILURE(rc))
688 {
689 /* Decrement the counts and return the error. */
690 for (;;)
691 {
692 u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
693 uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT; Assert(c > 0);
694 c--;
695 u64State &= ~RTSEMRW_CNT_WR_MASK;
696 u64State |= c << RTSEMRW_CNT_WR_SHIFT;
697 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
698 break;
699 }
700 return rc;
701 }
702
703 u64State = ASMAtomicReadU64(&pThis->u64State);
704 if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT))
705 {
706 ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
707 if (fDone)
708 break;
709 }
710 AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
711 }
712 }
713
714 /*
715 * Got it!
716 */
717 Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT));
718 ASMAtomicWriteU32(&pThis->cWriteRecursions, 1);
719 Assert(pThis->cWriterReads == 0);
720#ifdef RTSEMRW_STRICT
721 RTLockValidatorRecExclSetOwner(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, true);
722#endif
723
724 return VINF_SUCCESS;
725}
726
727
728RTDECL(int) RTSemRWRequestWrite(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
729{
730#ifndef RTSEMRW_STRICT
731 return rtSemRWRequestWrite(hRWSem, cMillies, false, NULL);
732#else
733 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
734 return rtSemRWRequestWrite(hRWSem, cMillies, false, &SrcPos);
735#endif
736}
737RT_EXPORT_SYMBOL(RTSemRWRequestWrite);
738
739
740RTDECL(int) RTSemRWRequestWriteDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
741{
742 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
743 return rtSemRWRequestWrite(hRWSem, cMillies, false, &SrcPos);
744}
745RT_EXPORT_SYMBOL(RTSemRWRequestWriteDebug);
746
747
748RTDECL(int) RTSemRWRequestWriteNoResume(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
749{
750#ifndef RTSEMRW_STRICT
751 return rtSemRWRequestWrite(hRWSem, cMillies, true, NULL);
752#else
753 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
754 return rtSemRWRequestWrite(hRWSem, cMillies, true, &SrcPos);
755#endif
756}
757RT_EXPORT_SYMBOL(RTSemRWRequestWriteNoResume);
758
759
760RTDECL(int) RTSemRWRequestWriteNoResumeDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
761{
762 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
763 return rtSemRWRequestWrite(hRWSem, cMillies, true, &SrcPos);
764}
765RT_EXPORT_SYMBOL(RTSemRWRequestWriteNoResumeDebug);
766
767
768RTDECL(int) RTSemRWReleaseWrite(RTSEMRW hRWSem)
769{
770
771 /*
772 * Validate handle.
773 */
774 struct RTSEMRWINTERNAL *pThis = hRWSem;
775 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
776 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
777
778 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
779 RTNATIVETHREAD hNativeWriter;
780 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
781 AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
782
783 /*
784 * Unwind a recursion.
785 */
786 if (pThis->cWriteRecursions == 1)
787 {
788 AssertReturn(pThis->cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
789#ifdef RTSEMRW_STRICT
790 int rc9 = RTLockValidatorRecExclReleaseOwner(&pThis->ValidatorWrite, true);
791 if (RT_FAILURE(rc9))
792 return rc9;
793#endif
794 /*
795 * Update the state.
796 */
797 ASMAtomicWriteU32(&pThis->cWriteRecursions, 0);
798 ASMAtomicWriteHandle(&pThis->hNativeWriter, NIL_RTNATIVETHREAD);
799
800 for (;;)
801 {
802 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
803 uint64_t u64OldState = u64State;
804
805 uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
806 Assert(c > 0);
807 c--;
808
809 if ( c > 0
810 || (u64State & RTSEMRW_CNT_RD_MASK) == 0)
811 {
812 /* Don't change the direction, wait up the next writer if any. */
813 u64State &= ~RTSEMRW_CNT_WR_MASK;
814 u64State |= c << RTSEMRW_CNT_WR_SHIFT;
815 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
816 {
817 if (c > 0)
818 {
819 int rc = RTSemEventSignal(pThis->hEvtWrite);
820 AssertRC(rc);
821 }
822 break;
823 }
824 }
825 else
826 {
827 /* Reverse the direction and signal the reader threads. */
828 u64State &= ~(RTSEMRW_CNT_WR_MASK | RTSEMRW_DIR_MASK);
829 u64State |= RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT;
830 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
831 {
832 Assert(!pThis->fNeedReset);
833 ASMAtomicWriteBool(&pThis->fNeedReset, true);
834 int rc = RTSemEventMultiSignal(pThis->hEvtRead);
835 AssertRC(rc);
836 break;
837 }
838 }
839
840 ASMNopPause();
841 if (pThis->u32Magic != RTSEMRW_MAGIC)
842 return VERR_SEM_DESTROYED;
843 }
844 }
845 else
846 {
847 Assert(pThis->cWriteRecursions != 0);
848#ifdef RTSEMRW_STRICT
849 int rc9 = RTLockValidatorRecExclUnwind(&pThis->ValidatorWrite);
850 if (RT_FAILURE(rc9))
851 return rc9;
852#endif
853 ASMAtomicDecU32(&pThis->cWriteRecursions);
854 }
855
856 return VINF_SUCCESS;
857}
858RT_EXPORT_SYMBOL(RTSemRWReleaseWrite);
859
860
861RTDECL(bool) RTSemRWIsWriteOwner(RTSEMRW hRWSem)
862{
863 /*
864 * Validate handle.
865 */
866 struct RTSEMRWINTERNAL *pThis = hRWSem;
867 AssertPtrReturn(pThis, false);
868 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, false);
869
870 /*
871 * Check ownership.
872 */
873 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
874 RTNATIVETHREAD hNativeWriter;
875 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
876 return hNativeWriter == hNativeSelf;
877}
878RT_EXPORT_SYMBOL(RTSemRWIsWriteOwner);
879
880
881RTDECL(bool) RTSemRWIsReadOwner(RTSEMRW hRWSem, bool fWannaHear)
882{
883 /*
884 * Validate handle.
885 */
886 struct RTSEMRWINTERNAL *pThis = hRWSem;
887 AssertPtrReturn(pThis, false);
888 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, false);
889
890 /*
891 * Inspect the state.
892 */
893 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
894 if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT))
895 {
896 /*
897 * It's in write mode, so we can only be a reader if we're also the
898 * current writer.
899 */
900 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
901 RTNATIVETHREAD hWriter;
902 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hWriter);
903 return hWriter == hNativeSelf;
904 }
905
906 /*
907 * Read mode. If there are no current readers, then we cannot be a reader.
908 */
909 if (!(u64State & RTSEMRW_CNT_RD_MASK))
910 return false;
911
912#ifdef RTSEMRW_STRICT
913 /*
914 * Ask the lock validator.
915 */
916 return RTLockValidatorRecSharedIsOwner(&pThis->ValidatorRead, NIL_RTTHREAD);
917#else
918 /*
919 * Ok, we don't know, just tell the caller what he want to hear.
920 */
921 return fWannaHear;
922#endif
923}
924RT_EXPORT_SYMBOL(RTSemRWIsReadOwner);
925
926
927RTDECL(uint32_t) RTSemRWGetWriteRecursion(RTSEMRW hRWSem)
928{
929 /*
930 * Validate handle.
931 */
932 struct RTSEMRWINTERNAL *pThis = hRWSem;
933 AssertPtrReturn(pThis, 0);
934 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, 0);
935
936 /*
937 * Return the requested data.
938 */
939 return pThis->cWriteRecursions;
940}
941RT_EXPORT_SYMBOL(RTSemRWGetWriteRecursion);
942
943
944RTDECL(uint32_t) RTSemRWGetWriterReadRecursion(RTSEMRW hRWSem)
945{
946 /*
947 * Validate handle.
948 */
949 struct RTSEMRWINTERNAL *pThis = hRWSem;
950 AssertPtrReturn(pThis, 0);
951 AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, 0);
952
953 /*
954 * Return the requested data.
955 */
956 return pThis->cWriterReads;
957}
958RT_EXPORT_SYMBOL(RTSemRWGetWriterReadRecursion);
959
960
961RTDECL(uint32_t) RTSemRWGetReadCount(RTSEMRW hRWSem)
962{
963 /*
964 * Validate input.
965 */
966 struct RTSEMRWINTERNAL *pThis = hRWSem;
967 AssertPtrReturn(pThis, 0);
968 AssertMsgReturn(pThis->u32Magic == RTSEMRW_MAGIC,
969 ("pThis=%p u32Magic=%#x\n", pThis, pThis->u32Magic),
970 0);
971
972 /*
973 * Return the requested data.
974 */
975 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
976 if ((u64State & RTSEMRW_DIR_MASK) != (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
977 return 0;
978 return (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
979}
980RT_EXPORT_SYMBOL(RTSemRWGetReadCount);
981
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use