1 /* 2 * Copyright (c) 2021 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 #ifndef _KERN_LOCK_TYPES_H 29 #define _KERN_LOCK_TYPES_H 30 31 #if XNU_KERNEL_PRIVATE 32 #include <machine/static_if.h> 33 #endif /* XNU_KERNEL_PRIVATE */ 34 #include <kern/kern_types.h> 35 36 __BEGIN_DECLS 37 38 #define LCK_SLEEP_MASK 0x3f /* Valid actions */ 39 40 /*! 41 * @enum lck_sleep_action_t 42 * 43 * @abstract 44 * An action to pass to the @c lck_*_sleep* family of functions. 45 */ 46 __options_decl(lck_sleep_action_t, unsigned int, { 47 LCK_SLEEP_DEFAULT = 0x00, /**< Release the lock while waiting for the event, then reclaim */ 48 LCK_SLEEP_UNLOCK = 0x01, /**< Release the lock and return unheld */ 49 LCK_SLEEP_SHARED = 0x02, /**< Reclaim the lock in shared mode (RW only) */ 50 LCK_SLEEP_EXCLUSIVE = 0x04, /**< Reclaim the lock in exclusive mode (RW only) */ 51 LCK_SLEEP_SPIN = 0x08, /**< Reclaim the lock in spin mode (mutex only) */ 52 LCK_SLEEP_PROMOTED_PRI = 0x10, /**< Sleep at a promoted priority */ 53 LCK_SLEEP_SPIN_ALWAYS = 0x20, /**< Reclaim the lock in spin-always mode (mutex only) */ 54 }); 55 56 __options_decl(lck_wake_action_t, unsigned int, { 57 LCK_WAKE_DEFAULT = 0x00, /* If waiters are present, transfer their push to the wokenup thread */ 58 LCK_WAKE_DO_NOT_TRANSFER_PUSH = 0x01, /* Do not transfer waiters push when waking up */ 59 }); 60 61 typedef const struct hw_spin_policy *hw_spin_policy_t; 62 63 #if XNU_KERNEL_PRIVATE 64 65 /*! 66 * @enum lck_option_t 67 * 68 * @abstract 69 * Lock options to pass to "lcks=" boot-arg 70 */ 71 __options_decl(lck_option_t, unsigned int, { 72 LCK_OPTION_ENABLE_DEBUG = 0x01, /**< Request debug in default attribute */ 73 LCK_OPTION_ENABLE_STAT = 0x02, /**< Request lock group statistics in default attribute */ 74 LCK_OPTION_DISABLE_RW_PRIO = 0x04, /**< Disable RW lock priority promotion */ 75 LCK_OPTION_ENABLE_TIME_STAT = 0x08, /**< Request time lock group statistics in default attribute */ 76 LCK_OPTION_DISABLE_RW_DEBUG = 0x10, /**< Disable RW lock best-effort debugging. */ 77 }); 78 79 #endif // XNU_KERNEL_PRIVATE 80 81 #if MACH_KERNEL_PRIVATE 82 83 /* 84 * The "hardware lock". Low-level locking primitives that 85 * MUST be exported by machine-dependent code; this abstraction 86 * must provide atomic, non-blocking mutual exclusion that 87 * is invulnerable to uniprocessor or SMP races, interrupts, 88 * traps or any other events. 89 * 90 * hw_lock_data_t machine-specific lock data structure 91 * hw_lock_t pointer to hw_lock_data_t 92 * 93 * An implementation must export these data types and must 94 * also provide routines to manipulate them (see prototypes, 95 * below). These routines may be external, inlined, optimized, 96 * or whatever, based on the kernel configuration. In the event 97 * that the implementation wishes to define its own prototypes, 98 * macros, or inline functions, it may define LOCK_HW_PROTOS 99 * to disable the definitions below. 100 * 101 * Mach does not expect these locks to support statistics, 102 * debugging, tracing or any other complexity. In certain 103 * configurations, Mach will build other locking constructs 104 * on top of this one. A correctly functioning Mach port need 105 * only implement these locks to be successful. However, 106 * greater efficiency may be gained with additional machine- 107 * dependent optimizations for the locking constructs defined 108 * later in kern/lock.h.. 109 */ 110 struct hslock { 111 uintptr_t lock_data __kernel_data_semantics; 112 }; 113 typedef struct hslock hw_lock_data_t, *hw_lock_t; 114 115 /*! 116 * @enum hw_lock_status_t 117 * 118 * @abstract 119 * Used to pass information about very low level locking primitives. 120 * 121 */ 122 __enum_closed_decl(hw_lock_status_t, int, { 123 /** 124 * The lock was not taken because it is in an invalid state, 125 * or the memory was unmapped. 126 * 127 * This is only valid for @c *_allow_invalid() variants. 128 * 129 * Preemption is preserved to the caller level for all variants. 130 */ 131 HW_LOCK_INVALID = -1, 132 133 /** 134 * the lock wasn't acquired and is contended / timed out. 135 * 136 * - @c *_nopreempt() variants: preemption level preserved 137 * - @c *_trylock() variants: preemption level preserved 138 * - other variants: preemption is disabled 139 */ 140 HW_LOCK_CONTENDED = 0, 141 142 /** 143 * the lock was acquired successfully 144 * 145 * - @c *_nopreempt() variants: preemption level preserved 146 * - other variants: preemption is disabled 147 */ 148 HW_LOCK_ACQUIRED = 1, 149 }); 150 151 /*! 152 * @enum hw_spin_timeout_status_t 153 * 154 * @abstract 155 * Used by spinlock timeout handlers. 156 * 157 * @const HW_LOCK_TIMEOUT_RETURN 158 * Tell the @c hw_lock*_to* caller to break out of the wait 159 * and return HW_LOCK_CONTENDED. 160 * 161 * @const HW_LOCK_TIMEOUT_CONTINUE 162 * Keep spinning for another "timeout". 163 */ 164 __enum_closed_decl(hw_spin_timeout_status_t, _Bool, { 165 HW_LOCK_TIMEOUT_RETURN, /**< return without taking the lock */ 166 HW_LOCK_TIMEOUT_CONTINUE, /**< keep spinning */ 167 }); 168 169 170 /*! 171 * @typedef hw_spin_timeout_t 172 * 173 * @abstract 174 * Describes the timeout used for a given spinning session. 175 */ 176 typedef struct { 177 uint64_t hwst_timeout; 178 #if SCHED_HYGIENE_DEBUG 179 bool hwst_in_ppl; 180 bool hwst_interruptible; 181 #endif /* SCHED_HYGIENE_DEBUG */ 182 } hw_spin_timeout_t; 183 184 185 /*! 186 * @typedef hw_spin_state_t 187 * 188 * @abstract 189 * Keeps track of the various timings used for spinning 190 */ 191 typedef struct { 192 uint64_t hwss_start; 193 uint64_t hwss_now; 194 uint64_t hwss_deadline; 195 #if SCHED_HYGIENE_DEBUG 196 uint64_t hwss_irq_start; 197 uint64_t hwss_irq_end; 198 #endif /* SCHED_HYGIENE_DEBUG */ 199 } hw_spin_state_t; 200 201 202 /*! 203 * @typedef hw_spin_timeout_fn_t 204 * 205 * @abstract 206 * The type of the timeout handlers for low level locking primitives. 207 * 208 * @discussion 209 * Typical handlers are written to just panic and not return 210 * unless some very specific conditions are met (debugging, ...). 211 * 212 * For formatting purposes, we provide HW_SPIN_TIMEOUT{,_DETAILS}{_FMT,_ARG} 213 * 214 * Those are meant to be used inside an hw_spin_timeout_fn_t function 215 * to form informative panic strings, like this: 216 * 217 * panic("MyLock[%p] " HW_SPIN_TIMEOUT_FMT "; " 218 * "<lock specific things> " HW_SPIN_TIMEOUT_DETAILS_FMT, 219 * lock_address, HW_SPIN_TIMEOUT_ARG(to, st), 220 * <lock specific args>, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); 221 * 222 * This ensures consistent panic string style, and transparent adoption 223 * for any new diagnostic/debugging features at all call-sites. 224 */ 225 typedef hw_spin_timeout_status_t (hw_spin_timeout_fn_t)(void *lock, 226 hw_spin_timeout_t to, hw_spin_state_t st); 227 228 #define HW_SPIN_TIMEOUT_FMT \ 229 "timeout after %llu ticks" 230 #define HW_SPIN_TIMEOUT_ARG(to, st) \ 231 ((st).hwss_now - (st).hwss_start) 232 233 #if SCHED_HYGIENE_DEBUG 234 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT \ 235 ", irq time: %llu" 236 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) \ 237 , ((st).hwss_irq_end - (st).hwss_irq_start) 238 #else 239 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT 240 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) 241 #endif 242 243 #define HW_SPIN_TIMEOUT_DETAILS_FMT \ 244 "start time: %llu, now: %llu, timeout: %llu" \ 245 HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT 246 #define HW_SPIN_TIMEOUT_DETAILS_ARG(to, st) \ 247 (st).hwss_start, (st).hwss_now, (to).hwst_timeout \ 248 HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) 249 250 /*! 251 * @struct hw_spin_policy 252 * 253 * @abstract 254 * Describes the spinning policy for a given lock. 255 */ 256 struct hw_spin_policy { 257 const char *hwsp_name; 258 union { 259 const uint64_t *hwsp_timeout; 260 const _Atomic uint64_t *hwsp_timeout_atomic; 261 }; 262 uint16_t hwsp_timeout_shift; 263 uint16_t hwsp_lock_offset; 264 265 hw_spin_timeout_fn_t *hwsp_op_timeout; 266 }; 267 268 #if __x86_64__ 269 #define LCK_MTX_USE_ARCH 1 270 #else 271 #define LCK_MTX_USE_ARCH 0 272 #endif 273 #endif /* MACH_KERNEL_PRIVATE */ 274 275 __END_DECLS 276 277 #endif /* _KERN_LOCK_TYPES_H */ 278