1 /*
2 * Copyright (c) 2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 #ifndef _KERN_LOCK_PTR_H_
31 #define _KERN_LOCK_PTR_H_
32
33 #include <kern/lock_types.h>
34 #include <kern/lock_group.h>
35 #include <kern/lock_attr.h>
36 #if KASAN_TBI
37 #include <san/kasan-tbi.h>
38 #endif /* KASAN_TBI */
39
40 __BEGIN_DECLS
41 #pragma GCC visibility push(hidden)
42
43 #if KASAN_TBI
44 #define HW_LCK_PTR_BITS (64 - 4 - 1 - 1 - 16)
45 #else /* !KASAN_TBI */
46 #define HW_LCK_PTR_BITS (64 - 1 - 1 - 16)
47 #endif /* !KASAN_TBI */
48
49 /*!
50 * @typedef hw_lck_ptr_t
51 *
52 * @brief
53 * Low level lock that can share the slot of a pointer value.
54 *
55 * @discussion
56 * An @c hw_lck_ptr_t is a fair spinlock fitting in a single
57 * pointer sized word, and allows for that word to retain
58 * its original pointer value without any loss.
59 *
60 * It uses the top bits of the pointer which in the kernel
61 * aren't significant to store an MCS queue as well as various
62 * state bits.
63 *
64 * The pointer can't be PAC signed, but the MTESAN tag
65 * is preserved (when the feature is on).
66 *
67 * No assumption is made on the alignment of the pointer,
68 * and clients may use the low bits of the pointer to encode
69 * state as they see fit.
70 *
71 * It is intended to be used for fine grained locking
72 * in hash table bucket heads or similar structures.
73 *
74 * The pointer value can be changed by taking the lock,
75 * (which returns the current value of the pointer),
76 * and then unlock with the new pointer value.
77 */
78 typedef union hw_lck_ptr {
79 struct {
80 intptr_t lck_ptr_bits : HW_LCK_PTR_BITS;
81 #if KASAN_TBI
82 uintptr_t lck_ptr_tag : 4;
83 #endif /* KASAN_TBI */
84 uintptr_t lck_ptr_locked : 1;
85 uintptr_t lck_ptr_stats : 1;
86 uint16_t lck_ptr_mcs_tail __kernel_ptr_semantics;
87 };
88 uintptr_t lck_ptr_value;
89 } hw_lck_ptr_t;
90
91
92 /* init/destroy */
93
94 #if !LCK_GRP_USE_ARG
95 #define hw_lck_ptr_init(lck, val, grp) hw_lck_ptr_init(lck, val)
96 #define hw_lck_ptr_destroy(lck, grp) hw_lck_ptr_destroy(lck)
97 #endif /* !LCK_GRP_USE_ARG */
98
99 /*!
100 * @function hw_lck_ptr_init()
101 *
102 * @brief
103 * Initializes an hw_lck_ptr_t with the specified pointer value.
104 *
105 * @discussion
106 * hw_lck_ptr_destroy() must be called to destroy this lock.
107 *
108 * @param lck the lock to initialize
109 * @param val the pointer value to store in the lock
110 * @param grp the lock group associated with this lock
111 */
112 extern void hw_lck_ptr_init(
113 hw_lck_ptr_t *lck,
114 void *val,
115 lck_grp_t *grp);
116
117 /*!
118 * @function hw_lck_ptr_destroy()
119 *
120 * @brief
121 * Detroys an hw_lck_ptr_t initialized with hw_lck_ptr_init().
122 *
123 * @param lck the lock to destroy
124 * @param grp the lock group associated with this lock
125 */
126 extern void hw_lck_ptr_destroy(
127 hw_lck_ptr_t *lck,
128 lck_grp_t *grp);
129
130 /*!
131 * @function hw_lck_ptr_held()
132 *
133 * @brief
134 * Returns whether the pointer lock is currently held by anyone.
135 *
136 * @param lck the lock to check.
137 */
138 extern bool hw_lck_ptr_held(
139 hw_lck_ptr_t *lck) __result_use_check;
140
141
142 /* lock */
143
144 #if !LCK_GRP_USE_ARG
145 #define hw_lck_ptr_lock(lck, grp) hw_lck_ptr_lock(lck)
146 #define hw_lck_ptr_lock_nopreempt(lck, grp) hw_lck_ptr_lock_nopreempt(lck)
147 #endif /* !LCK_GRP_USE_ARG */
148
149
150 /*!
151 * @function hw_lck_ptr_lock()
152 *
153 * @brief
154 * Locks a pointer lock, and returns its current value.
155 *
156 * @discussion
157 * This call will disable preemption.
158 *
159 * @param lck the lock to lock
160 * @param grp the lock group associated with this lock
161 */
162 extern void *hw_lck_ptr_lock(
163 hw_lck_ptr_t *lck,
164 lck_grp_t *grp) __result_use_check;
165
166 /*!
167 * @function hw_lck_ptr_lock_nopreempt()
168 *
169 * @brief
170 * Locks a pointer lock, and returns its current value.
171 *
172 * @discussion
173 * Preemption must be disabled to make this call,
174 * and must stay disabled until @c hw_lck_ptr_unlock()
175 * or @c hw_lck_ptr_unlock_nopreempt() is called
176 *
177 * @param lck the lock to lock
178 * @param grp the lock group associated with this lock
179 */
180 extern void *hw_lck_ptr_lock_nopreempt(
181 hw_lck_ptr_t *lck,
182 lck_grp_t *grp) __result_use_check;
183
184
185 /* unlock */
186
187 #if !LCK_GRP_USE_ARG
188 #define hw_lck_ptr_unlock(lck, val, grp) \
189 hw_lck_ptr_unlock(lck, val)
190 #define hw_lck_ptr_unlock_nopreempt(lck, val, grp) \
191 hw_lck_ptr_unlock_nopreempt(lck, val)
192 #endif /* !LCK_GRP_USE_ARG */
193
194
195 /*!
196 * @function hw_lck_ptr_unlock()
197 *
198 * @brief
199 * Unlocks a pointer lock, and update its currently held value.
200 *
201 * @discussion
202 * This call will reenable preemption.
203 *
204 * @param lck the lock to unlock
205 * @param val the value to update the pointer to
206 * @param grp the lock group associated with this lock
207 */
208 extern void hw_lck_ptr_unlock(
209 hw_lck_ptr_t *lck,
210 void *val,
211 lck_grp_t *grp);
212
213 /*!
214 * @function hw_lck_ptr_unlock_nopreempt()
215 *
216 * @brief
217 * Unlocks a pointer lock, and update its currently held value.
218 *
219 * @discussion
220 * This call will not reenable preemption.
221 *
222 * @param lck the lock to unlock
223 * @param val the value to update the pointer to
224 * @param grp the lock group associated with this lock
225 */
226 extern void hw_lck_ptr_unlock_nopreempt(
227 hw_lck_ptr_t *lck,
228 void *val,
229 lck_grp_t *grp);
230
231
232 /* wait_for_value */
233
234 #if !LCK_GRP_USE_ARG
235 #define hw_lck_ptr_wait_for_value(lck, val, grp) \
236 hw_lck_ptr_wait_for_value(lck, val)
237 #endif /* !LCK_GRP_USE_ARG */
238
239 static inline void *
__hw_lck_ptr_value(hw_lck_ptr_t val)240 __hw_lck_ptr_value(hw_lck_ptr_t val)
241 {
242 vm_offset_t ptr = val.lck_ptr_bits;
243
244 #if KASAN_TBI
245 if (ptr) {
246 ptr = kasan_tbi_tag_ptr(ptr, val.lck_ptr_tag);
247 }
248 #endif
249 return (void *)ptr;
250 }
251
252
253 /*!
254 * @function hw_lck_ptr_value()
255 *
256 * @brief
257 * Returns the pointer value currently held by this lock.
258 *
259 * @param lck the lock to get the pointer value of.
260 */
261 static inline void *
hw_lck_ptr_value(hw_lck_ptr_t * lck)262 hw_lck_ptr_value(hw_lck_ptr_t *lck)
263 {
264 hw_lck_ptr_t tmp;
265
266 tmp = atomic_load_explicit((hw_lck_ptr_t _Atomic *)lck,
267 memory_order_relaxed);
268
269 return __hw_lck_ptr_value(tmp);
270 }
271
272 /*!
273 * @function hw_lck_ptr_wait_for_value()
274 *
275 * @brief
276 * Spins until the pointer in the lock has the specified value.
277 *
278 * @discussion
279 * This function has an implicit acquire barrier pairing
280 * with the hw_lck_ptr_unlock() which sets the observed value.
281 *
282 * @param lck the lock to spin on
283 * @param val the value to wait for
284 * @param grp the lock group associated with this lock
285 */
286 extern void hw_lck_ptr_wait_for_value(
287 hw_lck_ptr_t *lck,
288 void *val,
289 lck_grp_t *grp);
290
291
292 #pragma GCC visibility pop
293 __END_DECLS
294
295 #endif /* _KERN_LOCK_PTR_H_ */
296