1 /*
2 * Copyright (c) 2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 #ifndef _KERN_LOCK_PTR_H_
31 #define _KERN_LOCK_PTR_H_
32
33 #include <kern/lock_types.h>
34 #include <kern/lock_group.h>
35 #include <kern/lock_attr.h>
36 #include <vm/vm_memtag.h>
37
38 __BEGIN_DECLS
39 #pragma GCC visibility push(hidden)
40
41 #if CONFIG_KERNEL_TAGGING
42 #define HW_LCK_PTR_BITS (64 - 4 - 1 - 1 - 16)
43 #else /* !CONFIG_KERNEL_TAGGING */
44 #define HW_LCK_PTR_BITS (64 - 1 - 1 - 16)
45 #endif /* !CONFIG_KERNEL_TAGGING */
46
47 /*!
48 * @typedef hw_lck_ptr_t
49 *
50 * @brief
51 * Low level lock that can share the slot of a pointer value.
52 *
53 * @discussion
54 * An @c hw_lck_ptr_t is a fair spinlock fitting in a single
55 * pointer sized word, and allows for that word to retain
56 * its original pointer value without any loss.
57 *
58 * It uses the top bits of the pointer which in the kernel
59 * aren't significant to store an MCS queue as well as various
60 * state bits.
61 *
62 * The pointer can't be PAC signed, but the MTESAN tag
63 * is preserved (when the feature is on).
64 *
65 * No assumption is made on the alignment of the pointer,
66 * and clients may use the low bits of the pointer to encode
67 * state as they see fit.
68 *
69 * It is intended to be used for fine grained locking
70 * in hash table bucket heads or similar structures.
71 *
72 * The pointer value can be changed by taking the lock,
73 * (which returns the current value of the pointer),
74 * and then unlock with the new pointer value.
75 */
76 typedef union hw_lck_ptr {
77 struct {
78 intptr_t lck_ptr_bits : HW_LCK_PTR_BITS;
79 #if CONFIG_KERNEL_TAGGING
80 uintptr_t lck_ptr_tag : 4;
81 #endif /* CONFIG_KERNEL_TAGGING */
82 uintptr_t lck_ptr_locked : 1;
83 uintptr_t lck_ptr_stats : 1;
84 uint16_t lck_ptr_mcs_tail __kernel_ptr_semantics;
85 };
86 uintptr_t lck_ptr_value;
87 } hw_lck_ptr_t;
88
89
90 /* init/destroy */
91
92 #if !LCK_GRP_USE_ARG
93 #define hw_lck_ptr_init(lck, val, grp) hw_lck_ptr_init(lck, val)
94 #define hw_lck_ptr_destroy(lck, grp) hw_lck_ptr_destroy(lck)
95 #endif /* !LCK_GRP_USE_ARG */
96
97 /*!
98 * @function hw_lck_ptr_init()
99 *
100 * @brief
101 * Initializes an hw_lck_ptr_t with the specified pointer value.
102 *
103 * @discussion
104 * hw_lck_ptr_destroy() must be called to destroy this lock.
105 *
106 * @param lck the lock to initialize
107 * @param val the pointer value to store in the lock
108 * @param grp the lock group associated with this lock
109 */
110 extern void hw_lck_ptr_init(
111 hw_lck_ptr_t *lck,
112 void *val,
113 lck_grp_t *grp);
114
115 /*!
116 * @function hw_lck_ptr_destroy()
117 *
118 * @brief
119 * Detroys an hw_lck_ptr_t initialized with hw_lck_ptr_init().
120 *
121 * @param lck the lock to destroy
122 * @param grp the lock group associated with this lock
123 */
124 extern void hw_lck_ptr_destroy(
125 hw_lck_ptr_t *lck,
126 lck_grp_t *grp);
127
128 /*!
129 * @function hw_lck_ptr_held()
130 *
131 * @brief
132 * Returns whether the pointer lock is currently held by anyone.
133 *
134 * @param lck the lock to check.
135 */
136 extern bool hw_lck_ptr_held(
137 hw_lck_ptr_t *lck) __result_use_check;
138
139
140 /* lock */
141
142 #if !LCK_GRP_USE_ARG
143 #define hw_lck_ptr_lock(lck, grp) hw_lck_ptr_lock(lck)
144 #define hw_lck_ptr_lock_nopreempt(lck, grp) hw_lck_ptr_lock_nopreempt(lck)
145 #endif /* !LCK_GRP_USE_ARG */
146
147
148 /*!
149 * @function hw_lck_ptr_lock()
150 *
151 * @brief
152 * Locks a pointer lock, and returns its current value.
153 *
154 * @discussion
155 * This call will disable preemption.
156 *
157 * @param lck the lock to lock
158 * @param grp the lock group associated with this lock
159 */
160 extern void *hw_lck_ptr_lock(
161 hw_lck_ptr_t *lck,
162 lck_grp_t *grp) __result_use_check;
163
164 /*!
165 * @function hw_lck_ptr_lock_nopreempt()
166 *
167 * @brief
168 * Locks a pointer lock, and returns its current value.
169 *
170 * @discussion
171 * Preemption must be disabled to make this call,
172 * and must stay disabled until @c hw_lck_ptr_unlock()
173 * or @c hw_lck_ptr_unlock_nopreempt() is called
174 *
175 * @param lck the lock to lock
176 * @param grp the lock group associated with this lock
177 */
178 extern void *hw_lck_ptr_lock_nopreempt(
179 hw_lck_ptr_t *lck,
180 lck_grp_t *grp) __result_use_check;
181
182
183 /* unlock */
184
185 #if !LCK_GRP_USE_ARG
186 #define hw_lck_ptr_unlock(lck, val, grp) \
187 hw_lck_ptr_unlock(lck, val)
188 #define hw_lck_ptr_unlock_nopreempt(lck, val, grp) \
189 hw_lck_ptr_unlock_nopreempt(lck, val)
190 #endif /* !LCK_GRP_USE_ARG */
191
192
193 /*!
194 * @function hw_lck_ptr_unlock()
195 *
196 * @brief
197 * Unlocks a pointer lock, and update its currently held value.
198 *
199 * @discussion
200 * This call will reenable preemption.
201 *
202 * @param lck the lock to unlock
203 * @param val the value to update the pointer to
204 * @param grp the lock group associated with this lock
205 */
206 extern void hw_lck_ptr_unlock(
207 hw_lck_ptr_t *lck,
208 void *val,
209 lck_grp_t *grp);
210
211 /*!
212 * @function hw_lck_ptr_unlock_nopreempt()
213 *
214 * @brief
215 * Unlocks a pointer lock, and update its currently held value.
216 *
217 * @discussion
218 * This call will not reenable preemption.
219 *
220 * @param lck the lock to unlock
221 * @param val the value to update the pointer to
222 * @param grp the lock group associated with this lock
223 */
224 extern void hw_lck_ptr_unlock_nopreempt(
225 hw_lck_ptr_t *lck,
226 void *val,
227 lck_grp_t *grp);
228
229
230 /* wait_for_value */
231
232 #if !LCK_GRP_USE_ARG
233 #define hw_lck_ptr_wait_for_value(lck, val, grp) \
234 hw_lck_ptr_wait_for_value(lck, val)
235 #endif /* !LCK_GRP_USE_ARG */
236
237 static inline void *
__hw_lck_ptr_value(hw_lck_ptr_t val)238 __hw_lck_ptr_value(hw_lck_ptr_t val)
239 {
240 vm_offset_t ptr = val.lck_ptr_bits;
241
242 #if CONFIG_KERNEL_TAGGING
243 if (ptr) {
244 ptr = vm_memtag_add_ptr_tag(ptr, val.lck_ptr_tag);
245 }
246 #endif /* CONFIG_KERNEL_TAGGING */
247 return (void *)ptr;
248 }
249
250
251 /*!
252 * @function hw_lck_ptr_value()
253 *
254 * @brief
255 * Returns the pointer value currently held by this lock.
256 *
257 * @param lck the lock to get the pointer value of.
258 */
259 static inline void *
hw_lck_ptr_value(hw_lck_ptr_t * lck)260 hw_lck_ptr_value(hw_lck_ptr_t *lck)
261 {
262 hw_lck_ptr_t tmp;
263
264 tmp = atomic_load_explicit((hw_lck_ptr_t _Atomic *)lck,
265 memory_order_relaxed);
266
267 return __hw_lck_ptr_value(tmp);
268 }
269
270 /*!
271 * @function hw_lck_ptr_wait_for_value()
272 *
273 * @brief
274 * Spins until the pointer in the lock has the specified value.
275 *
276 * @discussion
277 * This function has an implicit acquire barrier pairing
278 * with the hw_lck_ptr_unlock() which sets the observed value.
279 *
280 * @param lck the lock to spin on
281 * @param val the value to wait for
282 * @param grp the lock group associated with this lock
283 */
284 extern void hw_lck_ptr_wait_for_value(
285 hw_lck_ptr_t *lck,
286 void *val,
287 lck_grp_t *grp);
288
289
290 #pragma GCC visibility pop
291 __END_DECLS
292
293 #endif /* _KERN_LOCK_PTR_H_ */
294