1 /*
2 * Copyright (c) 2025 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include "rwlock.h"
30 #include "random.h"
31
32 #include <sys/errno.h>
33 #include <sys/types.h>
34
35 #ifdef __BUILDING_WITH_TSAN__
36 #include <sanitizer/tsan_interface.h>
37 #endif
38
39 void
fibers_rwlock_init(fibers_rwlock_t * rwlock)40 fibers_rwlock_init(fibers_rwlock_t *rwlock)
41 {
42 rwlock->writer_active = NULL;
43 rwlock->reader_count = 0;
44 rwlock->reader_wait_queue = (struct fibers_queue){0, 0};
45 rwlock->writer_wait_queue = (struct fibers_queue){0, 0};
46
47 #ifdef __BUILDING_WITH_TSAN__
48 __tsan_mutex_create(rwlock, __tsan_mutex_not_static);
49 #endif
50 }
51
52 static void
fibers_rwlock_rdlock_helper(fibers_rwlock_t * rwlock,bool check_may_yield)53 fibers_rwlock_rdlock_helper(fibers_rwlock_t *rwlock, bool check_may_yield)
54 {
55 #ifdef __BUILDING_WITH_TSAN__
56 __tsan_mutex_pre_lock(rwlock, __tsan_mutex_read_lock);
57 #endif
58
59 // stop a reader if there are writers waiting (RANGELOCKINGTODO rdar://150845975 use the PRNG to choose?)
60 if (rwlock->writer_active != NULL || rwlock->writer_wait_queue.count > 0) {
61 FIBERS_LOG(FIBERS_LOG_DEBUG, "waiting for read lock %p (writer %p active, %d writers waiting)",
62 rwlock, rwlock->writer_active, rwlock->writer_wait_queue.count);
63 if (check_may_yield) {
64 FIBERS_ASSERT(fibers_current->may_yield_disabled == 0, "fibers_rwlock_rdlock_helper: waiting on rwlock with fibers_current->may_yield_disabled not 0");
65 }
66
67 fibers_queue_push(&rwlock->reader_wait_queue, fibers_current);
68 #ifdef __BUILDING_WITH_TSAN__
69 __tsan_mutex_pre_divert(rwlock, 0);
70 #endif
71 fibers_choose_next(FIBER_WAIT);
72 #ifdef __BUILDING_WITH_TSAN__
73 __tsan_mutex_post_divert(rwlock, 0);
74 #endif
75 FIBERS_ASSERT(rwlock->writer_active == NULL, "fibers_rwlock_rdlock_helper: woken up while writer %d still active", rwlock->writer_active ? rwlock->writer_active->id : -1);
76 } else {
77 rwlock->reader_count++;
78 FIBERS_LOG(FIBERS_LOG_DEBUG, "acquired read lock %p (now %u readers)", rwlock, rwlock->reader_count);
79 }
80
81 #ifdef __BUILDING_WITH_TSAN__
82 __tsan_mutex_post_lock(rwlock, __tsan_mutex_read_lock, 0);
83 #endif
84
85 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_LOCK);
86 }
87
88 static int
fibers_rwlock_try_rdlock_helper(fibers_rwlock_t * rwlock)89 fibers_rwlock_try_rdlock_helper(fibers_rwlock_t *rwlock)
90 {
91 #ifdef __BUILDING_WITH_TSAN__
92 __tsan_mutex_pre_lock(rwlock, __tsan_mutex_try_read_lock);
93 #endif
94
95 if (rwlock->writer_active != NULL || rwlock->writer_wait_queue.count > 0) {
96 #ifdef __BUILDING_WITH_TSAN__
97 __tsan_mutex_post_lock(rwlock, __tsan_mutex_try_read_lock | __tsan_mutex_try_read_lock_failed, 0);
98 #endif
99 return EBUSY;
100 } else {
101 rwlock->reader_count++;
102 FIBERS_LOG(FIBERS_LOG_DEBUG, "try acquired read lock %p (now %u readers)", rwlock, rwlock->reader_count);
103 #ifdef __BUILDING_WITH_TSAN__
104 __tsan_mutex_post_lock(rwlock, __tsan_mutex_try_read_lock, 0);
105 #endif
106 return 0;
107 }
108 }
109
110 static void
fibers_rwlock_wrlock_helper(fibers_rwlock_t * rwlock,bool check_may_yield)111 fibers_rwlock_wrlock_helper(fibers_rwlock_t *rwlock, bool check_may_yield)
112 {
113 #ifdef __BUILDING_WITH_TSAN__
114 __tsan_mutex_pre_lock(rwlock, 0);
115 #endif
116
117 if (rwlock->writer_active != NULL || rwlock->reader_count > 0) {
118 FIBERS_ASSERT(rwlock->writer_active != fibers_current, "fibers_rwlock_wrlock_helper: recursive write lock attempted by %d", fibers_current->id);
119 FIBERS_LOG(FIBERS_LOG_DEBUG, "waiting for write lock %p (writer %p active, %u readers active)",
120 rwlock, rwlock->writer_active, rwlock->reader_count);
121 if (check_may_yield) {
122 FIBERS_ASSERT(fibers_current->may_yield_disabled == 0, "fibers_rwlock_wrlock_helper: waiting on rwlock with fibers_current->may_yield_disabled not 0");
123 }
124
125 fibers_queue_push(&rwlock->writer_wait_queue, fibers_current);
126 #ifdef __BUILDING_WITH_TSAN__
127 __tsan_mutex_pre_divert(rwlock, 0);
128 #endif
129 fibers_choose_next(FIBER_WAIT);
130 #ifdef __BUILDING_WITH_TSAN__
131 __tsan_mutex_post_divert(rwlock, 0);
132 #endif
133 FIBERS_ASSERT(rwlock->writer_active == fibers_current, "fibers_rwlock_wrlock_helper: woken up but not writer holder (%p != %p)", rwlock->writer_active, fibers_current);
134 FIBERS_ASSERT(rwlock->reader_count == 0, "fibers_rwlock_wrlock_helper: woken up as writer but %u readers still active?", rwlock->reader_count);
135 } else {
136 FIBERS_LOG(FIBERS_LOG_DEBUG, "acquired write lock %p", rwlock);
137 rwlock->writer_active = fibers_current;
138 }
139
140 #ifdef __BUILDING_WITH_TSAN__
141 __tsan_mutex_post_lock(rwlock, 0, 0);
142 #endif
143
144 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_LOCK);
145 }
146
147 static int
fibers_rwlock_try_wrlock_helper(fibers_rwlock_t * rwlock)148 fibers_rwlock_try_wrlock_helper(fibers_rwlock_t *rwlock)
149 {
150 #ifdef __BUILDING_WITH_TSAN__
151 __tsan_mutex_pre_lock(rwlock, __tsan_mutex_try_lock);
152 #endif
153
154 if (rwlock->writer_active != NULL || rwlock->reader_count > 0) {
155 #ifdef __BUILDING_WITH_TSAN__
156 __tsan_mutex_post_lock(rwlock, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0);
157 #endif
158 return EBUSY;
159 } else {
160 // Acquire write lock
161 FIBERS_LOG(FIBERS_LOG_DEBUG, "try acquired write lock %p", rwlock);
162 rwlock->writer_active = fibers_current;
163 #ifdef __BUILDING_WITH_TSAN__
164 __tsan_mutex_post_lock(rwlock, __tsan_mutex_try_lock, 0);
165 #endif
166 return 0;
167 }
168 }
169
170 static void
fibers_rwlock_rdunlock_helper(fibers_rwlock_t * rwlock)171 fibers_rwlock_rdunlock_helper(fibers_rwlock_t *rwlock)
172 {
173 FIBERS_ASSERT(rwlock->writer_active == NULL, "fibers_rwlock_rdunlock_helper: trying to read-unlock while writer %d active", rwlock->writer_active ? rwlock->writer_active->id : -1);
174 FIBERS_ASSERT(rwlock->reader_count > 0, "fibers_rwlock_rdunlock_helper: trying to read-unlock with zero readers");
175
176 #ifdef __BUILDING_WITH_TSAN__
177 __tsan_mutex_pre_unlock(rwlock, __tsan_mutex_read_lock);
178 #endif
179
180 rwlock->reader_count--;
181 FIBERS_LOG(FIBERS_LOG_DEBUG, "released read lock %p (readers remaining %u)", rwlock, rwlock->reader_count);
182
183 // if last reader out and writers are waiting, wake one writer
184 if (rwlock->reader_count == 0 && rwlock->writer_wait_queue.count > 0) {
185 fiber_t new_writer = fibers_queue_pop(&rwlock->writer_wait_queue, random_below(rwlock->writer_wait_queue.count));
186 FIBERS_ASSERT(new_writer->state == FIBER_WAIT, "fibers_rwlock_rdunlock_helper: woken writer %d is not FIBER_WAIT", new_writer->id);
187 FIBERS_LOG(FIBERS_LOG_DEBUG, "waking up writer %d waiting on rwlock %p", new_writer->id, rwlock);
188 rwlock->writer_active = new_writer;
189
190 fibers_queue_push(&fibers_run_queue, new_writer);
191 }
192
193 #ifdef __BUILDING_WITH_TSAN__
194 __tsan_mutex_post_unlock(rwlock, __tsan_mutex_read_lock);
195 #endif
196
197 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_UNLOCK);
198 }
199
200 static void
fibers_rwlock_wrunlock_helper(fibers_rwlock_t * rwlock)201 fibers_rwlock_wrunlock_helper(fibers_rwlock_t *rwlock)
202 {
203 FIBERS_ASSERT(rwlock->writer_active == fibers_current, "fibers_rwlock_wrunlock_helper: trying to write-unlock lock not held by current fiber %d (holder %d)", fibers_current->id, rwlock->writer_active ? rwlock->writer_active->id : -1);
204 FIBERS_ASSERT(rwlock->reader_count == 0, "fibers_rwlock_wrunlock_helper: trying to write-unlock while %u readers active?", rwlock->reader_count);
205
206 #ifdef __BUILDING_WITH_TSAN__
207 __tsan_mutex_pre_unlock(rwlock, 0);
208 #endif
209
210 FIBERS_LOG(FIBERS_LOG_DEBUG, "releasing write lock %p", rwlock);
211 rwlock->writer_active = NULL;
212
213 if (rwlock->writer_wait_queue.count > 0) {
214 fiber_t new_writer = fibers_queue_pop(&rwlock->writer_wait_queue, random_below(rwlock->writer_wait_queue.count));
215 FIBERS_ASSERT(new_writer->state == FIBER_WAIT, "fibers_rwlock_wrunlock_helper: woken writer %d is not FIBER_WAIT", new_writer->id);
216 FIBERS_LOG(FIBERS_LOG_DEBUG, "waking up writer %d waiting on rwlock %p", new_writer->id, rwlock);
217 rwlock->writer_active = new_writer;
218
219 fibers_queue_push(&fibers_run_queue, new_writer);
220 } else if (rwlock->reader_wait_queue.count > 0) {
221 FIBERS_LOG(FIBERS_LOG_DEBUG, "waking up %d readers waiting on rwlock %p", rwlock->reader_wait_queue.count, rwlock);
222
223 unsigned int initial_count = rwlock->reader_wait_queue.count;
224 while (rwlock->reader_wait_queue.count > 0) {
225 fiber_t new_reader = fibers_queue_pop(&rwlock->reader_wait_queue, random_below(rwlock->reader_wait_queue.count));
226 FIBERS_ASSERT(new_reader->state == FIBER_WAIT, "fibers_rwlock_wrunlock_helper: woken reader %d is not FIBER_WAIT", new_reader->id);
227 rwlock->reader_count++;
228
229 fibers_queue_push(&fibers_run_queue, new_reader);
230 }
231 FIBERS_ASSERT(rwlock->reader_count == initial_count, "fibers_rwlock_wrunlock_helper: reader count mismatch after waking readers (%u != %u)", rwlock->reader_count, initial_count);
232 FIBERS_LOG(FIBERS_LOG_DEBUG, "rwlock %p now held by %u readers", rwlock, rwlock->reader_count);
233 }
234
235 #ifdef __BUILDING_WITH_TSAN__
236 __tsan_mutex_post_unlock(rwlock, 0);
237 #endif
238
239 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_UNLOCK);
240 }
241
242 void
fibers_rwlock_rdlock(fibers_rwlock_t * rwlock,bool check_may_yield)243 fibers_rwlock_rdlock(fibers_rwlock_t *rwlock, bool check_may_yield)
244 {
245 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_LOCK);
246 fibers_rwlock_rdlock_helper(rwlock, check_may_yield);
247 }
248
249 void
fibers_rwlock_wrlock(fibers_rwlock_t * rwlock,bool check_may_yield)250 fibers_rwlock_wrlock(fibers_rwlock_t *rwlock, bool check_may_yield)
251 {
252 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_LOCK);
253 fibers_rwlock_wrlock_helper(rwlock, check_may_yield);
254 }
255
256 int
fibers_rwlock_try_rdlock(fibers_rwlock_t * rwlock)257 fibers_rwlock_try_rdlock(fibers_rwlock_t *rwlock)
258 {
259 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_LOCK);
260 int err = fibers_rwlock_try_rdlock_helper(rwlock);
261 fibers_may_yield_internal_with_reason(
262 FIBERS_YIELD_REASON_MUTEX |
263 FIBERS_YIELD_REASON_ERROR_IF(err != 0));
264 return err;
265 }
266
267 int
fibers_rwlock_try_wrlock(fibers_rwlock_t * rwlock)268 fibers_rwlock_try_wrlock(fibers_rwlock_t *rwlock)
269 {
270 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_LOCK);
271 int err = fibers_rwlock_try_wrlock_helper(rwlock);
272 fibers_may_yield_internal_with_reason(
273 FIBERS_YIELD_REASON_MUTEX |
274 FIBERS_YIELD_REASON_ERROR_IF(err != 0));
275 return err;
276 }
277
278 void
fibers_rwlock_rdunlock(fibers_rwlock_t * rwlock)279 fibers_rwlock_rdunlock(fibers_rwlock_t *rwlock)
280 {
281 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_UNLOCK);
282 fibers_rwlock_rdunlock_helper(rwlock);
283 }
284
285 void
fibers_rwlock_wrunlock(fibers_rwlock_t * rwlock)286 fibers_rwlock_wrunlock(fibers_rwlock_t *rwlock)
287 {
288 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_UNLOCK);
289 fibers_rwlock_wrunlock_helper(rwlock);
290 }
291
292 void
fibers_rwlock_unlock(fibers_rwlock_t * rwlock)293 fibers_rwlock_unlock(fibers_rwlock_t *rwlock)
294 {
295 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_UNLOCK);
296 if (rwlock->writer_active) {
297 fibers_rwlock_wrunlock_helper(rwlock);
298 } else {
299 fibers_rwlock_rdunlock_helper(rwlock);
300 }
301 }
302
303 void
fibers_rwlock_destroy(fibers_rwlock_t * rwlock)304 fibers_rwlock_destroy(fibers_rwlock_t *rwlock)
305 {
306 FIBERS_ASSERT(rwlock->writer_active == NULL, "fibers_rwlock_destroy: tried to destroy rwlock with active writer %d", rwlock->writer_active ? rwlock->writer_active->id : -1);
307 FIBERS_ASSERT(rwlock->reader_count == 0, "fibers_rwlock_destroy: tried to destroy rwlock with %u active readers", rwlock->reader_count);
308 FIBERS_ASSERT(rwlock->reader_wait_queue.count == 0, "fibers_rwlock_destroy: tried to destroy rwlock with %d waiting readers", rwlock->reader_wait_queue.count);
309 FIBERS_ASSERT(rwlock->writer_wait_queue.count == 0, "fibers_rwlock_destroy: tried to destroy rwlock with %d waiting writers", rwlock->writer_wait_queue.count);
310
311 #ifdef __BUILDING_WITH_TSAN__
312 __tsan_mutex_destroy(rwlock, __tsan_mutex_not_static);
313 #endif
314
315 fibers_may_yield_internal_with_reason(
316 FIBERS_YIELD_REASON_MUTEX |
317 FIBERS_YIELD_REASON_MUTEX_DESTROY |
318 FIBERS_YIELD_REASON_ORDER_POST);
319 }
320
321 bool
fibers_rwlock_upgrade(fibers_rwlock_t * rwlock)322 fibers_rwlock_upgrade(fibers_rwlock_t *rwlock)
323 {
324 fibers_may_yield_with_prob(FIBERS_INTERNAL_YIELD_PROB);
325
326 FIBERS_ASSERT(rwlock->writer_active == NULL, "fibers_rwlock_upgrade: trying to upgrade lock while writer %d active", rwlock->writer_active ? rwlock->writer_active->id : -1);
327 FIBERS_ASSERT(rwlock->reader_count > 0, "fibers_rwlock_upgrade: trying to upgrade with zero readers");
328
329 // if another fiber want to upgrade fail, release the lock and bail out
330 if (rwlock->flags & FIBERS_RWLOCK_WANT_UPGRADE) {
331 fibers_rwlock_rdunlock_helper(rwlock);
332 return false;
333 }
334
335 #ifdef __BUILDING_WITH_TSAN__
336 __tsan_mutex_pre_unlock(rwlock, __tsan_mutex_read_lock);
337 #endif
338
339 // mark that we want to upgrade and we arrived here first
340 rwlock->flags |= FIBERS_RWLOCK_WANT_UPGRADE;
341 rwlock->reader_count--;
342
343 // wait for the other readers to finish
344 if (rwlock->reader_count > 0) {
345 FIBERS_LOG(FIBERS_LOG_DEBUG, "fibers_rwlock_upgrade: waiting for remaining readers (%u) to finish on rwlock %p", rwlock->reader_count, rwlock);
346
347 fibers_queue_push(&rwlock->writer_wait_queue, fibers_current);
348
349 #ifdef __BUILDING_WITH_TSAN__
350 __tsan_mutex_pre_divert(rwlock, 0);
351 #endif
352 fibers_choose_next(FIBER_WAIT);
353 #ifdef __BUILDING_WITH_TSAN__
354 __tsan_mutex_post_divert(rwlock, 0);
355 #endif
356
357 // when we wake up, we should be the only ones holding the lock.
358 FIBERS_ASSERT(rwlock->writer_active == fibers_current, "fibers_rwlock_upgrade: woken up but not writer holder (%p != %p)", rwlock->writer_active, fibers_current);
359 FIBERS_ASSERT(rwlock->reader_count == 0, "fibers_rwlock_upgrade: woken up as writer but %u readers still active?", rwlock->reader_count);
360 } else {
361 // we were the only reader, so we can immediately become the writer.
362 FIBERS_LOG(FIBERS_LOG_DEBUG, "fibers_rwlock_upgrade: no other readers, acquiring write lock %p", rwlock);
363 rwlock->writer_active = fibers_current;
364 }
365
366 rwlock->flags &= ~FIBERS_RWLOCK_WANT_UPGRADE;
367
368 #ifdef __BUILDING_WITH_TSAN__
369 __tsan_mutex_post_unlock(rwlock, __tsan_mutex_read_lock);
370 __tsan_mutex_pre_lock(rwlock, 0);
371 __tsan_mutex_post_lock(rwlock, 0, 0);
372 #endif
373 fibers_may_yield_with_prob(FIBERS_INTERNAL_YIELD_PROB);
374
375 return true;
376 }
377
378 void
fibers_rwlock_downgrade(fibers_rwlock_t * rwlock)379 fibers_rwlock_downgrade(fibers_rwlock_t *rwlock)
380 {
381 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_UNLOCK);
382
383 FIBERS_ASSERT(rwlock->writer_active == fibers_current, "fibers_rwlock_downgrade: trying to downgrade lock not held exclusively by current fiber %d (holder %d)", fibers_current->id, rwlock->writer_active ? rwlock->writer_active->id : -1);
384 FIBERS_ASSERT(rwlock->reader_count == 0, "fibers_rwlock_downgrade: trying to downgrade while %u readers unexpectedly active?", rwlock->reader_count);
385
386 #ifdef __BUILDING_WITH_TSAN__
387 __tsan_mutex_pre_unlock(rwlock, 0);
388 #endif
389
390 FIBERS_LOG(FIBERS_LOG_DEBUG, "downgrading write lock %p to read lock", rwlock);
391
392 // release the write hold, acquire a read hold for the current fiber
393 rwlock->writer_active = NULL;
394 rwlock->reader_count = 1;
395
396 #ifdef __BUILDING_WITH_TSAN__
397 __tsan_mutex_post_unlock(rwlock, 0);
398 __tsan_mutex_pre_lock(rwlock, __tsan_mutex_read_lock);
399 #endif
400
401 if (rwlock->reader_wait_queue.count > 0) {
402 FIBERS_LOG(FIBERS_LOG_DEBUG, "downgrade: waking up %d readers waiting on rwlock %p", rwlock->reader_wait_queue.count, rwlock);
403 unsigned int initial_woken_count = rwlock->reader_wait_queue.count;
404 unsigned int readers_woken = 0;
405 while (rwlock->reader_wait_queue.count > 0) {
406 fiber_t new_reader = fibers_queue_pop(&rwlock->reader_wait_queue, random_below(rwlock->reader_wait_queue.count));
407 FIBERS_ASSERT(new_reader->state == FIBER_WAIT, "fibers_rwlock_downgrade: woken reader %d is not FIBER_WAIT", new_reader->id);
408 rwlock->reader_count++;
409 readers_woken++;
410 fibers_queue_push(&fibers_run_queue, new_reader);
411 // TSan: Each woken reader will execute its post_lock upon resuming.
412 }
413 FIBERS_ASSERT(readers_woken == initial_woken_count, "fibers_rwlock_downgrade: reader wakeup count mismatch (%u != %u)", readers_woken, initial_woken_count);
414 FIBERS_LOG(FIBERS_LOG_DEBUG, "rwlock %p now held by %u readers after downgrade", rwlock, rwlock->reader_count);
415 } else {
416 FIBERS_LOG(FIBERS_LOG_DEBUG, "rwlock %p now held by 1 reader (self) after downgrade", rwlock);
417 }
418
419 #ifdef __BUILDING_WITH_TSAN__
420 __tsan_mutex_post_lock(rwlock, __tsan_mutex_read_lock, 0);
421 #endif
422
423 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_UNLOCK);
424 }
425
426 void
fibers_rwlock_assert(fibers_rwlock_t * rwlock,unsigned int type)427 fibers_rwlock_assert(fibers_rwlock_t *rwlock, unsigned int type)
428 {
429 fiber_t current = fibers_current;
430 bool condition_met = false;
431 const char *fail_msg = "Unknown assertion failure";
432
433 switch (type) {
434 case FIBERS_RWLOCK_ASSERT_SHARED:
435 if (rwlock->reader_count > 0 && rwlock->writer_active == NULL) {
436 condition_met = true;
437 } else {
438 fail_msg = "Lock not held in shared mode";
439 }
440 break;
441
442 case FIBERS_RWLOCK_ASSERT_EXCLUSIVE:
443 if (rwlock->writer_active == current && rwlock->reader_count == 0) {
444 condition_met = true;
445 } else {
446 fail_msg = "Lock not held exclusively by current fiber";
447 }
448 break;
449
450 case FIBERS_RWLOCK_ASSERT_HELD:
451 if ((rwlock->reader_count > 0 && rwlock->writer_active == NULL) ||
452 (rwlock->writer_active == current && rwlock->reader_count == 0)) {
453 condition_met = true;
454 } else {
455 fail_msg = "Lock not held by current fiber (exclusively) or any fiber (shared)";
456 }
457 break;
458
459 case FIBERS_RWLOCK_ASSERT_NOTHELD:
460 if (rwlock->reader_count == 0 && rwlock->writer_active == NULL) {
461 condition_met = true;
462 } else {
463 fail_msg = "Lock is held";
464 }
465 break;
466
467 case FIBERS_RWLOCK_ASSERT_NOT_OWNED:
468 if (rwlock->writer_active != current) {
469 condition_met = true;
470 } else {
471 fail_msg = "Lock is held exclusively by current fiber";
472 }
473 break;
474
475 default:
476 fail_msg = "Unknown assertion type requested";
477 break;
478 }
479
480 FIBERS_ASSERT(
481 condition_met,
482 "fibers_rwlock_assert(%p) failed: type=0x%x (%s). State: writer=%d, readers=%u", (void *)rwlock, type, fail_msg,
483 rwlock->writer_active ? rwlock->writer_active->id : -1,
484 rwlock->reader_count
485 );
486 }
487