1 /* 2 * Copyright (c) 2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _KERN_APFS_REFLOCK_H_ 30 #define _KERN_APFS_REFLOCK_H_ 31 32 #include <sys/cdefs.h> 33 #include <kern/kern_types.h> 34 #include <kern/locks.h> 35 36 /* 37 * kern_apfs_reflock_t is an object that provides a refcount protected by an embedded lock. 38 * Manipulating the refcount is expected to be the most common operation on this object; 39 * the refcount can be changed (incremented or decremented) when the lock is not held. 40 * Some users might require to halt the refcount manipulation while some operations 41 * are in progress. To express this, kern_apfs_reflock_t allow to lock the object providing 42 * mutual exclusion between those operations and the refcount. 43 * When the object is locked all new lock requests, increments and decrements of the kern_apfs_reflock_t 44 * will fail, and the user can choose to wait for the object to be unlocked. 45 * The thread that locked the object will inherit the priority push of all the 46 * threads waiting for it to be unlocked. 47 * Further, refcount transitions 0->1 and 1->0, allow to atomically lock the reflock 48 * providing the possibility to cleanup/initialize the state. 49 */ 50 51 #ifdef KERNEL_PRIVATE 52 53 #if XNU_KERNEL_PRIVATE 54 #define KERN_APFS_REFLOCK_WAITERS_BIT 16 55 #define KERN_APFS_REFLOCK_REFCOUNT_BIT (64 - (SWI_COND_OWNER_BITS + KERN_APFS_REFLOCK_WAITERS_BIT + 4)) 56 #define KERN_APFS_REFLOCK_MAXREFCOUNT ((1ull << KERN_APFS_REFLOCK_REFCOUNT_BIT) - 1) 57 #define KERN_APFS_REFLOCK_MAXWAITERS ((1ull << KERN_APFS_REFLOCK_WAITERS_BIT) - 1) 58 #define KERN_APFS_REFLOCK_DESTROYED (~(0ull)) 59 60 /* 61 * Mask to debounce the sleep. Needs to be kept up to date with kern_apfs_reflock. 62 * Equivalent to: 63 * mask = {.kern_apfs_rl_owner = ((1 << SWI_COND_OWNER_BITS) - 1), 64 * .kern_apfs_rl_delayed_free = 1, 65 * .kern_apfs_rl_wake = 1, 66 * .kern_apfs_rl_allocated = 1, 67 * .kern_apfs_rl_allow_force = 1}; 68 */ 69 70 #define KERN_APFS_SLEEP_DEBOUNCE_MASK ((uint64_t)0xf0000fffff) 71 72 typedef struct kern_apfs_reflock { 73 union { 74 cond_swi_var64_s kern_apfs_rl_data; 75 struct { 76 uint64_t kern_apfs_rl_owner: SWI_COND_OWNER_BITS, 77 kern_apfs_rl_waiters: KERN_APFS_REFLOCK_WAITERS_BIT, 78 kern_apfs_rl_delayed_free: 1, 79 kern_apfs_rl_wake: 1, 80 kern_apfs_rl_allocated: 1, 81 kern_apfs_rl_allow_force: 1, 82 kern_apfs_rl_count: KERN_APFS_REFLOCK_REFCOUNT_BIT; 83 }; 84 }; 85 } *kern_apfs_reflock_t; 86 87 88 #else /* XNU_KERNEL_PRIVATE */ 89 typedef struct kern_apfs_reflock { 90 uint64_t opaque; 91 } *kern_apfs_reflock_t; 92 #endif /* XNU_KERNEL_PRIVATE */ 93 94 __options_decl(kern_apfs_reflock_in_flags_t, uint32_t, { 95 KERN_APFS_REFLOCK_IN_DEFAULT = 0x0, 96 KERN_APFS_REFLOCK_IN_LOCK_IF_LAST = 0x1, 97 KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST = 0x2, 98 KERN_APFS_REFLOCK_IN_WILL_WAIT = 0x4, 99 KERN_APFS_REFLOCK_IN_FORCE = 0x8, 100 KERN_APFS_REFLOCK_IN_ALLOW_FORCE = 0x10, 101 }); 102 103 __options_decl(kern_apfs_reflock_out_flags_t, uint32_t, { 104 KERN_APFS_REFLOCK_OUT_DEFAULT = 0x0, 105 KERN_APFS_REFLOCK_OUT_LOCKED = 0x1, 106 }); 107 108 __BEGIN_DECLS 109 110 /* 111 * Name: kern_apfs_reflock_data 112 * 113 * Description: declares a kern_apfs_reflock variable with specified storage class. 114 * The reflock will be stored in this variable and it is the caller's responsibility 115 * to ensure that this variable's memory is going to be accessible by all threads that will use 116 * the kern_apfs_reflock. 117 * Every kern_apfs_reflock function will require a pointer to this variable as parameter. 118 * 119 * The variable needs to be initialized once with kern_apfs_reflock_init() and destroyed once with 120 * kern_apfs_reflock_destroy() when not needed anymore. 121 * 122 * Args: 123 * Arg1: storage class. 124 * Arg2: variable name. 125 */ 126 #define kern_apfs_reflock_data(class, name) class struct kern_apfs_reflock name 127 128 /* 129 * Name: kern_apfs_reflock_init 130 * 131 * Description: initializes a kern_apfs_reflock_t. 132 * 133 * Args: 134 * Arg1: kern_apfs_reflock object. 135 * 136 * Conditions: the memory pointed by kern_apfs_reflock_t needs to be available 137 * while any of the other functions are executed. 138 * It is the caller responsibility to guarantee that all functions call 139 * executed on the kern_apfs_reflock have terminated before freeing it, 140 * including possible kern_apfs_reflock_wait_for_unlock(). If it is not possible 141 * to safely synchronize kern_apfs_reflock_wait_for_unlock() calls 142 * kern_apfs_reflock_alloc_init() should be used instead. 143 */ 144 void kern_apfs_reflock_init(kern_apfs_reflock_t reflock); 145 146 /* 147 * Name: kern_apfs_reflock_destroy 148 * 149 * Description: destroys a kern_apfs_reflock_t. 150 * 151 * Args: 152 * Arg1: kern_apfs_reflock object. 153 * 154 * Conditions: the object must have been previously initialized with kern_apfs_reflock_init. 155 * Any access past this point to the kern_apfs_reflock will be considered invalid. 156 */ 157 void kern_apfs_reflock_destroy(kern_apfs_reflock_t reflock); 158 159 /* 160 * Name: kern_apfs_reflock_alloc_init 161 * 162 * Description: allocates a kern_apfs_reflock_t. 163 * 164 * Returns: allocated kern_apfs_reflock_t. 165 * 166 * Conditions: It is the caller responsibility to guarantee that all functions call 167 * executed on the kern_apfs_reflock_t returned by this function 168 * (except for kern_apfs_reflock_wait_for_unlock()) have terminated before freeing it. 169 * It is safe to execute concurrent or later kern_apfs_reflock_wait_for_unlock() 170 * calls as long as the matching kern_apfs_reflock_try_get_ref(), 171 * kern_apfs_reflock_try_put_ref() or kern_apfs_reflock_try_lock() was executed before 172 * the call to kern_apfs_reflock_free(). In this case the free of the object will be delegated 173 * to the last concurrent kern_apfs_reflock_wait_for_unlock() executed. 174 */ 175 kern_apfs_reflock_t kern_apfs_reflock_alloc_init(void); 176 177 /* 178 * Name: kern_apfs_reflock_free 179 * 180 * Description: frees and destroys a kern_apfs_reflock_t. 181 * 182 * Args: 183 * Arg1: kern_apfs_reflock object. 184 * 185 * Conditions: It is the caller responsability to guarantee that all functions call 186 * executed on the kern_apfs_reflock_t (except kern_apfs_reflock_wait_for_unlock()) 187 * have terminated before freeing it. 188 * It is safe to execute concurrent or later kern_apfs_reflock_wait_for_unlock() 189 * calls as long as the matching kern_apfs_reflock_try_get_ref(), 190 * kern_apfs_reflock_try_put_ref() or kern_apfs_reflock_try_lock() was executed before 191 * the call to kern_apfs_reflock_free(). In this case the free of the object will be delegated 192 * to the last concurrent kern_apfs_reflock_wait_for_unlock() executed. 193 */ 194 void kern_apfs_reflock_free(kern_apfs_reflock_t reflock); 195 196 /* 197 * Name: kern_apfs_reflock_try_get_ref 198 * 199 * Description: tries to get a reference on the kern_apfs_reflock. 200 * The operation will succeed if the lock on the object is not held. 201 * In case of failure the caller can choose to wait for the lock to unlock 202 * with a subsequent call to kern_apfs_reflock_wait_for_unlock(). 203 * 204 * Args: 205 * Arg1: kern_apfs_reflock object. 206 * Arg2: in flags can be a combination of: 207 * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour. 208 * - KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST will lock the reflock if the refcount was incremented 209 * in the "init" transition, so from 0->1. 210 * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_get() fails, then the thread will call kern_apfs_reflock_wait_for_unlock(). 211 * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this 212 * flag was not set. 213 * - KERN_APFS_REFLOCK_IN_FORCE if the reflock was locked from a kern_apfs_reflock_try_lock() with KERN_APFS_REFLOCK_IN_ALLOW_FORCE 214 * this flag will allow to get the reference even if the object is locked. 215 * Even with this flag set the function might fail if the reflock was locked from a 216 * kern_apfs_reflock_try_get_ref() or kern_apfs_reflock_try_put_ref(). 217 * NOTE: KERN_APFS_REFLOCK_IN_FORCE and KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST cannot be used together. 218 * Arg3: out flags: 219 * - KERN_APFS_REFLOCK_OUT_DEFAULT if the lock was not acquired. 220 * - KERN_APFS_REFLOCK_OUT_LOCKED if the lock was acquired. 221 * 222 * Returns: true if the reference was acquired, false otherwise. 223 * If KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST was set and the reference was successfully acquired, out_flags will indicate if the 224 * lock was acquired. 225 * 226 * 227 * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock() 228 * needs to be called in case of failure. 229 * If KERN_APFS_REFLOCK_OUT_LOCKED is returned on the out_flags a corresponding kern_apfs_reflock_wait_for_unlock() needs to be called 230 * by the same thread and the thread cannot execute in userspace until the unlock is called. 231 */ 232 bool kern_apfs_reflock_try_get_ref(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, kern_apfs_reflock_out_flags_t *out_flags); 233 234 /* 235 * Name: kern_apfs_reflock_try_put_ref 236 * 237 * Description: tries to put a reference on the kern_apfs_reflock. 238 * The operation will succeed if the lock on the object is not held. 239 * In case of failure the caller can choose to wait for the lock to unlock 240 * with a subsequent call to kern_apfs_reflock_wait_for_unlock(). 241 * 242 * Args: 243 * Arg1: kern_apfs_reflock object. 244 * Arg2: in flags can be a combination of: 245 * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour. 246 * - KERN_APFS_REFLOCK_IN_LOCK_IF_LAST will lock the reflock if the refcount was decremented 247 * in the "cleanup" transition, so from 1->0. 248 * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_put() fails, then the thread will call kern_apfs_reflock_wait_for_unlock(). 249 * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this 250 * flag was not set. 251 * - KERN_APFS_REFLOCK_IN_FORCE if the reflock was locked from a kern_apfs_reflock_try_lock() with KERN_APFS_REFLOCK_IN_ALLOW_FORCE 252 * this flag will allow to put the reference even if the object is locked. 253 * Even with this flag set the function might fail if the reflock was locked from a 254 * kern_apfs_reflock_try_get_ref() or kern_apfs_reflock_try_put_ref(). 255 * NOTE: KERN_APFS_REFLOCK_IN_FORCE and KERN_APFS_REFLOCK_IN_LOCK_IF_LAST cannot be used together. 256 * Arg3: out flags: 257 * - KERN_APFS_REFLOCK_OUT_DEFAULT if the lock was not acquired. 258 * - KERN_APFS_REFLOCK_OUT_LOCKED if the lock was acquired. 259 * 260 * Returns: true if the reference was successfully decremented, false otherwise. 261 * If KERN_APFS_REFLOCK_IN_LOCK_IF_LAST was set and the reference was successfully decremented, out_flags will indicate if the 262 * lock was acquired. 263 * 264 * 265 * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock() 266 * needs to be called in case of failure. 267 * If KERN_APFS_REFLOCK_OUT_LOCKED is returned on the out_flags a corresponding kern_apfs_reflock_wait_for_unlock() needs to be called 268 * by the same theread and the thread cannot execute in userspace until the unlock is called. 269 * 270 */ 271 bool kern_apfs_reflock_try_put_ref(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, kern_apfs_reflock_out_flags_t *out_flags); 272 273 /* 274 * Name: kern_apfs_reflock_try_lock 275 * 276 * Description: tries to acquire the lock on the kern_apfs_reflock. 277 * The operation will succeed if the lock on the object is not held. 278 * In case of failure the caller can choose to wait for the lock to unlock 279 * with a subsequent call to kern_apfs_reflock_wait_for_unlock(). 280 * 281 * Args: 282 * Arg1: kern_apfs_reflock object. 283 * Arg2: in flags can be a combination of: 284 * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour. 285 * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_lock() fails, then the thread will call kern_apfs_reflock_wait_for_unlock(). 286 * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this 287 * flag was not set. 288 * - KERN_APFS_REFLOCK_IN_ALLOW_FORCE if this flag is set, kern_apfs_reflock_try_put_ref() and kern_apfs_reflock_try_get_ref() with 289 * flag KERN_APFS_REFLOCK_IN_FORCE set will succed even after this call locked the reflock. 290 * 291 * Arg3: refcount_when_lock pointer into which return the value of the refcount at the moment of lock. 292 * 293 * Returns: true if the lock was acquired, false otherwise. 294 * 295 * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock() 296 * needs to be called in case of failure. 297 * If the lock was acquired a subsequent kern_apfs_reflock_unlock() by the same theread and 298 * the thread cannot execute in userspace until the unlock is called. 299 * Recursive locking is not allowed. 300 */ 301 bool kern_apfs_reflock_try_lock(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, uint32_t *refcount_when_lock); 302 303 /* 304 * Name: kern_apfs_reflock_wait_for_unlock 305 * 306 * Description: waits for the lock to be unlocked. 307 * While waiting the priority of this thread will contribute 308 * to the priority push of the owner of the reflock. 309 * NOTE: it is not guaranteed that by the time this calls 310 * returns the reflock is unlocked, as it might have been re-locked 311 * after the current thread has been woken up. 312 * If needed, the matching kern_apfs_reflock_try_get_ref(), kern_apfs_reflock_try_put_ref() or 313 * kern_apfs_reflock_try_lock() should be re-driven after this function. 314 * 315 * Args: 316 * Arg1: reflock object. 317 * Arg2: interruptible flag for wait. 318 * Arg3: deadline for wait. 319 * 320 * Returns: result of the wait. 321 * THREAD_AWAKENED - normal wakeup 322 * THREAD_TIMED_OUT - timeout expired 323 * THREAD_INTERRUPTED - aborted/interrupted 324 * THREAD_NOT_WAITING - thread didn't need to wait 325 */ 326 wait_result_t kern_apfs_reflock_wait_for_unlock(kern_apfs_reflock_t reflock, wait_interrupt_t interruptible, uint64_t deadline); 327 328 /* 329 * Name: kern_apfs_reflock_unlock 330 * 331 * Description: unlocks a reflock obj. 332 * 333 * Args: 334 * Arg1: reflock object. 335 * 336 * Conditions: the same thread that locked the object needs to unlock it. 337 */ 338 void kern_apfs_reflock_unlock(kern_apfs_reflock_t reflock); 339 340 /* 341 * Name: kern_apfs_reflock_read_ref 342 * 343 * Description: reads the refcount counter. 344 * Note: using this function is racy, as the refcount can change 345 * after this function reads it. Its usage is discouraged. 346 * 347 * Args: 348 * Arg1: reflock object. 349 * 350 * Returns: refcount value. 351 */ 352 uint64_t kern_apfs_reflock_read_ref(kern_apfs_reflock_t reflock); 353 __END_DECLS 354 355 #endif /* KERNEL_PRIVATE */ 356 #endif /* _KERN_APFS_REFLOCK_H_ */ 357