1 /*
2 * Copyright (c) 2025 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include "mutex.h"
30 #include "random.h"
31
32 #include <sys/errno.h>
33 #include <sys/types.h>
34 #include <sys/signal.h>
35
36 #ifdef __BUILDING_WITH_TSAN__
37 #include <sanitizer/tsan_interface.h>
38 #endif
39
40 void
fibers_mutex_init(fibers_mutex_t * mtx)41 fibers_mutex_init(fibers_mutex_t *mtx)
42 {
43 mtx->holder = 0;
44 mtx->wait_queue = (struct fibers_queue){0, 0};
45 #ifdef __BUILDING_WITH_TSAN__
46 __tsan_mutex_create(mtx, __tsan_mutex_not_static);
47 #endif
48 }
49
50 static void
fibers_mutex_lock_helper(fibers_mutex_t * mtx,bool check_may_yield)51 fibers_mutex_lock_helper(fibers_mutex_t *mtx, bool check_may_yield)
52 {
53 #ifdef __BUILDING_WITH_TSAN__
54 __tsan_mutex_pre_lock(mtx, 0);
55 #endif
56
57 if (mtx->holder) {
58 FIBERS_ASSERT(mtx->holder != fibers_current, "fibers_mutex_lock_helper: tried to lock mutex already held by %d", mtx->holder->id);
59 // TODO rdar://150846598 add support for recursive locks
60 FIBERS_LOG(FIBERS_LOG_DEBUG, "waiting on mutex %p locked by %d", mtx, mtx->holder->id);
61 if (check_may_yield) {
62 // check for mutexes but not spinlocks
63 FIBERS_ASSERT(fibers_current->may_yield_disabled == 0, "fibers_mutex_lock_helper: waiting on a mutex with fibers_current->may_yield_disabled not 0");
64 }
65
66 fibers_queue_push(&mtx->wait_queue, fibers_current);
67 #ifdef __BUILDING_WITH_TSAN__
68 __tsan_mutex_pre_divert(mtx, 0);
69 #endif
70 fibers_choose_next(FIBER_WAIT);
71 #ifdef __BUILDING_WITH_TSAN__
72 __tsan_mutex_post_divert(mtx, 0);
73 #endif
74 FIBERS_ASSERT(mtx->holder == fibers_current, "fibers_mutex_lock_helper: waken up without being the holder of %p", mtx);
75 } else {
76 FIBERS_LOG(FIBERS_LOG_DEBUG, "locking mutex %p", mtx);
77 mtx->holder = fibers_current;
78 }
79
80 #ifdef __BUILDING_WITH_TSAN__
81 __tsan_mutex_post_lock(mtx, 0, 0);
82 #endif
83
84 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_LOCK);
85 }
86
87 static void
fibers_mutex_unlock_helper(fibers_mutex_t * mtx)88 fibers_mutex_unlock_helper(fibers_mutex_t *mtx)
89 {
90 FIBERS_ASSERT(mtx->holder == fibers_current, "fibers_mutex_unlock_helper: tried to unlock mutex held by %d", mtx->holder ? mtx->holder->id : -1);
91 FIBERS_LOG(FIBERS_LOG_DEBUG, "unlocking mutex %p", mtx);
92
93 #ifdef __BUILDING_WITH_TSAN__
94 __tsan_mutex_pre_unlock(mtx, 0);
95 #endif
96
97 mtx->holder = NULL;
98
99 #ifdef __BUILDING_WITH_TSAN__
100 __tsan_mutex_post_unlock(mtx, 0);
101 #endif
102
103 if (mtx->wait_queue.count) {
104 fiber_t new_holder = fibers_queue_pop(&mtx->wait_queue, random_below(mtx->wait_queue.count));
105 FIBERS_ASSERT(new_holder->state == FIBER_WAIT, "fibers_mutex_unlock_helper: new holder %d is not FIBER_WAIT", new_holder->id);
106 FIBERS_LOG(FIBERS_LOG_DEBUG, "waking up %d waiting on mutex %p", new_holder->id, mtx);
107 mtx->holder = new_holder;
108
109 fibers_queue_push(&fibers_run_queue, new_holder);
110 }
111
112 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_DID_UNLOCK);
113 }
114
115 static int
fibers_mutex_try_lock_helper(fibers_mutex_t * mtx)116 fibers_mutex_try_lock_helper(fibers_mutex_t *mtx)
117 {
118 #ifdef __BUILDING_WITH_TSAN__
119 __tsan_mutex_pre_lock(mtx, __tsan_mutex_try_lock);
120 #endif
121
122 if (mtx->holder) {
123 #ifdef __BUILDING_WITH_TSAN__
124 __tsan_mutex_post_lock(mtx, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0);
125 #endif
126 return EBUSY;
127 } else {
128 FIBERS_LOG(FIBERS_LOG_DEBUG, "locking mutex %p", mtx);
129 mtx->holder = fibers_current;
130 }
131
132 #ifdef __BUILDING_WITH_TSAN__
133 __tsan_mutex_post_lock(mtx, __tsan_mutex_try_lock, 0);
134 #endif
135 return 0;
136 }
137
138 void
fibers_mutex_lock(fibers_mutex_t * mtx,bool check_may_yield)139 fibers_mutex_lock(fibers_mutex_t *mtx, bool check_may_yield)
140 {
141 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_LOCK);
142 fibers_mutex_lock_helper(mtx, check_may_yield);
143 }
144
145 void
fibers_mutex_unlock(fibers_mutex_t * mtx)146 fibers_mutex_unlock(fibers_mutex_t *mtx)
147 {
148 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_UNLOCK);
149 fibers_mutex_unlock_helper(mtx);
150 }
151
152 int
fibers_mutex_try_lock(fibers_mutex_t * mtx)153 fibers_mutex_try_lock(fibers_mutex_t *mtx)
154 {
155 fibers_may_yield_internal_with_reason(FIBERS_YIELD_REASON_MUTEX_WILL_LOCK);
156 int err = fibers_mutex_try_lock_helper(mtx);
157 fibers_may_yield_internal_with_reason(err == 0 ? FIBERS_YIELD_REASON_MUTEX_DID_LOCK : FIBERS_YIELD_REASON_MUTEX_TRY_LOCK_FAIL);
158 return err;
159 }
160
161 void
fibers_mutex_destroy(fibers_mutex_t * mtx)162 fibers_mutex_destroy(fibers_mutex_t *mtx)
163 {
164 FIBERS_ASSERT(mtx->holder == NULL, "fibers_mutex_destroy: tried to destroy mutex held by %d", mtx->holder->id);
165 FIBERS_ASSERT(mtx->wait_queue.count == 0, "fibers_mutex_destroy: tried to destroy mutex with non empty wait queue");
166
167 #ifdef __BUILDING_WITH_TSAN__
168 __tsan_mutex_destroy(mtx, __tsan_mutex_not_static);
169 #endif
170
171 fibers_may_yield_internal();
172 }
173