1 /* 2 * Copyright (c) 2021 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 #ifndef _KERN_LOCK_TYPES_H 29 #define _KERN_LOCK_TYPES_H 30 31 #include <kern/kern_types.h> 32 33 __BEGIN_DECLS 34 35 #define LCK_SLEEP_MASK 0x3f /* Valid actions */ 36 37 /*! 38 * @enum lck_sleep_action_t 39 * 40 * @abstract 41 * An action to pass to the @c lck_*_sleep* family of functions. 42 */ 43 __options_decl(lck_sleep_action_t, unsigned int, { 44 LCK_SLEEP_DEFAULT = 0x00, /**< Release the lock while waiting for the event, then reclaim */ 45 LCK_SLEEP_UNLOCK = 0x01, /**< Release the lock and return unheld */ 46 LCK_SLEEP_SHARED = 0x02, /**< Reclaim the lock in shared mode (RW only) */ 47 LCK_SLEEP_EXCLUSIVE = 0x04, /**< Reclaim the lock in exclusive mode (RW only) */ 48 LCK_SLEEP_SPIN = 0x08, /**< Reclaim the lock in spin mode (mutex only) */ 49 LCK_SLEEP_PROMOTED_PRI = 0x10, /**< Sleep at a promoted priority */ 50 LCK_SLEEP_SPIN_ALWAYS = 0x20, /**< Reclaim the lock in spin-always mode (mutex only) */ 51 }); 52 53 __options_decl(lck_wake_action_t, unsigned int, { 54 LCK_WAKE_DEFAULT = 0x00, /* If waiters are present, transfer their push to the wokenup thread */ 55 LCK_WAKE_DO_NOT_TRANSFER_PUSH = 0x01, /* Do not transfer waiters push when waking up */ 56 }); 57 58 typedef const struct hw_spin_policy *hw_spin_policy_t; 59 60 #if XNU_KERNEL_PRIVATE 61 62 /*! 63 * @enum lck_option_t 64 * 65 * @abstract 66 * Lock options to pass to "lcks=" boot-arg 67 */ 68 __options_decl(lck_option_t, unsigned int, { 69 LCK_OPTION_ENABLE_DEBUG = 0x01, /**< Request debug in default attribute */ 70 LCK_OPTION_ENABLE_STAT = 0x02, /**< Request lock group statistics in default attribute */ 71 LCK_OPTION_DISABLE_RW_PRIO = 0x04, /**< Disable RW lock priority promotion */ 72 LCK_OPTION_ENABLE_TIME_STAT = 0x08, /**< Request time lock group statistics in default attribute */ 73 LCK_OPTION_DISABLE_RW_DEBUG = 0x10, /**< Disable RW lock best-effort debugging. */ 74 }); 75 76 #endif // XNU_KERNEL_PRIVATE 77 78 #if MACH_KERNEL_PRIVATE 79 80 /* 81 * The "hardware lock". Low-level locking primitives that 82 * MUST be exported by machine-dependent code; this abstraction 83 * must provide atomic, non-blocking mutual exclusion that 84 * is invulnerable to uniprocessor or SMP races, interrupts, 85 * traps or any other events. 86 * 87 * hw_lock_data_t machine-specific lock data structure 88 * hw_lock_t pointer to hw_lock_data_t 89 * 90 * An implementation must export these data types and must 91 * also provide routines to manipulate them (see prototypes, 92 * below). These routines may be external, inlined, optimized, 93 * or whatever, based on the kernel configuration. In the event 94 * that the implementation wishes to define its own prototypes, 95 * macros, or inline functions, it may define LOCK_HW_PROTOS 96 * to disable the definitions below. 97 * 98 * Mach does not expect these locks to support statistics, 99 * debugging, tracing or any other complexity. In certain 100 * configurations, Mach will build other locking constructs 101 * on top of this one. A correctly functioning Mach port need 102 * only implement these locks to be successful. However, 103 * greater efficiency may be gained with additional machine- 104 * dependent optimizations for the locking constructs defined 105 * later in kern/lock.h.. 106 */ 107 struct hslock { 108 uintptr_t lock_data __kernel_data_semantics; 109 }; 110 typedef struct hslock hw_lock_data_t, *hw_lock_t; 111 112 /*! 113 * @enum hw_lock_status_t 114 * 115 * @abstract 116 * Used to pass information about very low level locking primitives. 117 * 118 */ 119 __enum_closed_decl(hw_lock_status_t, int, { 120 /** 121 * The lock was not taken because it is in an invalid state, 122 * or the memory was unmapped. 123 * 124 * This is only valid for @c *_allow_invalid() variants. 125 * 126 * Preemption is preserved to the caller level for all variants. 127 */ 128 HW_LOCK_INVALID = -1, 129 130 /** 131 * the lock wasn't acquired and is contended / timed out. 132 * 133 * - @c *_nopreempt() variants: preemption level preserved 134 * - @c *_trylock() variants: preemption level preserved 135 * - other variants: preemption is disabled 136 */ 137 HW_LOCK_CONTENDED = 0, 138 139 /** 140 * the lock was acquired successfully 141 * 142 * - @c *_nopreempt() variants: preemption level preserved 143 * - other variants: preemption is disabled 144 */ 145 HW_LOCK_ACQUIRED = 1, 146 }); 147 148 /*! 149 * @enum hw_spin_timeout_status_t 150 * 151 * @abstract 152 * Used by spinlock timeout handlers. 153 * 154 * @const HW_LOCK_TIMEOUT_RETURN 155 * Tell the @c hw_lock*_to* caller to break out of the wait 156 * and return HW_LOCK_CONTENDED. 157 * 158 * @const HW_LOCK_TIMEOUT_CONTINUE 159 * Keep spinning for another "timeout". 160 */ 161 __enum_closed_decl(hw_spin_timeout_status_t, _Bool, { 162 HW_LOCK_TIMEOUT_RETURN, /**< return without taking the lock */ 163 HW_LOCK_TIMEOUT_CONTINUE, /**< keep spinning */ 164 }); 165 166 167 /*! 168 * @typedef hw_spin_timeout_t 169 * 170 * @abstract 171 * Describes the timeout used for a given spinning session. 172 */ 173 typedef struct { 174 uint64_t hwst_timeout; 175 #if SCHED_HYGIENE_DEBUG 176 bool hwst_in_ppl; 177 bool hwst_interruptible; 178 #endif /* SCHED_HYGIENE_DEBUG */ 179 } hw_spin_timeout_t; 180 181 182 /*! 183 * @typedef hw_spin_state_t 184 * 185 * @abstract 186 * Keeps track of the various timings used for spinning 187 */ 188 typedef struct { 189 uint64_t hwss_start; 190 uint64_t hwss_now; 191 uint64_t hwss_deadline; 192 #if SCHED_HYGIENE_DEBUG 193 uint64_t hwss_irq_start; 194 uint64_t hwss_irq_end; 195 #endif /* SCHED_HYGIENE_DEBUG */ 196 } hw_spin_state_t; 197 198 199 /*! 200 * @typedef hw_spin_timeout_fn_t 201 * 202 * @abstract 203 * The type of the timeout handlers for low level locking primitives. 204 * 205 * @discussion 206 * Typical handlers are written to just panic and not return 207 * unless some very specific conditions are met (debugging, ...). 208 * 209 * For formatting purposes, we provide HW_SPIN_TIMEOUT{,_DETAILS}{_FMT,_ARG} 210 * 211 * Those are meant to be used inside an hw_spin_timeout_fn_t function 212 * to form informative panic strings, like this: 213 * 214 * panic("MyLock[%p] " HW_SPIN_TIMEOUT_FMT "; " 215 * "<lock specific things> " HW_SPIN_TIMEOUT_DETAILS_FMT, 216 * lock_address, HW_SPIN_TIMEOUT_ARG(to, st), 217 * <lock specific args>, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); 218 * 219 * This ensures consistent panic string style, and transparent adoption 220 * for any new diagnostic/debugging features at all call-sites. 221 */ 222 typedef hw_spin_timeout_status_t (hw_spin_timeout_fn_t)(void *lock, 223 hw_spin_timeout_t to, hw_spin_state_t st); 224 225 #define HW_SPIN_TIMEOUT_FMT \ 226 "timeout after %llu ticks" 227 #define HW_SPIN_TIMEOUT_ARG(to, st) \ 228 ((st).hwss_now - (st).hwss_start) 229 230 #if SCHED_HYGIENE_DEBUG 231 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT \ 232 ", irq time: %llu" 233 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) \ 234 , ((st).hwss_irq_end - (st).hwss_irq_start) 235 #else 236 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT 237 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) 238 #endif 239 240 #define HW_SPIN_TIMEOUT_DETAILS_FMT \ 241 "start time: %llu, now: %llu, timeout: %llu" \ 242 HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT 243 #define HW_SPIN_TIMEOUT_DETAILS_ARG(to, st) \ 244 (st).hwss_start, (st).hwss_now, (to).hwst_timeout \ 245 HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) 246 247 /*! 248 * @struct hw_spin_policy 249 * 250 * @abstract 251 * Describes the spinning policy for a given lock. 252 */ 253 struct hw_spin_policy { 254 const char *hwsp_name; 255 union { 256 const uint64_t *hwsp_timeout; 257 const _Atomic uint64_t *hwsp_timeout_atomic; 258 }; 259 uint16_t hwsp_timeout_shift; 260 uint16_t hwsp_lock_offset; 261 262 hw_spin_timeout_fn_t *hwsp_op_timeout; 263 }; 264 265 #if __x86_64__ 266 #define LCK_MTX_USE_ARCH 1 267 #else 268 #define LCK_MTX_USE_ARCH 0 269 #endif 270 #endif /* MACH_KERNEL_PRIVATE */ 271 272 __END_DECLS 273 274 #endif /* _KERN_LOCK_TYPES_H */ 275