1 /* 2 * Copyright (c) 2021 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 #ifndef _KERN_LOCK_TYPES_H 29 #define _KERN_LOCK_TYPES_H 30 31 #include <kern/kern_types.h> 32 33 __BEGIN_DECLS 34 35 #define LCK_SLEEP_MASK 0x3f /* Valid actions */ 36 37 /*! 38 * @enum lck_sleep_action_t 39 * 40 * @abstract 41 * An action to pass to the @c lck_*_sleep* family of functions. 42 */ 43 __options_decl(lck_sleep_action_t, unsigned int, { 44 LCK_SLEEP_DEFAULT = 0x00, /**< Release the lock while waiting for the event, then reclaim */ 45 LCK_SLEEP_UNLOCK = 0x01, /**< Release the lock and return unheld */ 46 LCK_SLEEP_SHARED = 0x02, /**< Reclaim the lock in shared mode (RW only) */ 47 LCK_SLEEP_EXCLUSIVE = 0x04, /**< Reclaim the lock in exclusive mode (RW only) */ 48 LCK_SLEEP_SPIN = 0x08, /**< Reclaim the lock in spin mode (mutex only) */ 49 LCK_SLEEP_PROMOTED_PRI = 0x10, /**< Sleep at a promoted priority */ 50 LCK_SLEEP_SPIN_ALWAYS = 0x20, /**< Reclaim the lock in spin-always mode (mutex only) */ 51 }); 52 53 __options_decl(lck_wake_action_t, unsigned int, { 54 LCK_WAKE_DEFAULT = 0x00, /* If waiters are present, transfer their push to the wokenup thread */ 55 LCK_WAKE_DO_NOT_TRANSFER_PUSH = 0x01, /* Do not transfer waiters push when waking up */ 56 }); 57 58 typedef const struct hw_spin_policy *hw_spin_policy_t; 59 60 #if MACH_KERNEL_PRIVATE 61 62 /* 63 * The "hardware lock". Low-level locking primitives that 64 * MUST be exported by machine-dependent code; this abstraction 65 * must provide atomic, non-blocking mutual exclusion that 66 * is invulnerable to uniprocessor or SMP races, interrupts, 67 * traps or any other events. 68 * 69 * hw_lock_data_t machine-specific lock data structure 70 * hw_lock_t pointer to hw_lock_data_t 71 * 72 * An implementation must export these data types and must 73 * also provide routines to manipulate them (see prototypes, 74 * below). These routines may be external, inlined, optimized, 75 * or whatever, based on the kernel configuration. In the event 76 * that the implementation wishes to define its own prototypes, 77 * macros, or inline functions, it may define LOCK_HW_PROTOS 78 * to disable the definitions below. 79 * 80 * Mach does not expect these locks to support statistics, 81 * debugging, tracing or any other complexity. In certain 82 * configurations, Mach will build other locking constructs 83 * on top of this one. A correctly functioning Mach port need 84 * only implement these locks to be successful. However, 85 * greater efficiency may be gained with additional machine- 86 * dependent optimizations for the locking constructs defined 87 * later in kern/lock.h.. 88 */ 89 struct hslock { 90 uintptr_t lock_data __kernel_data_semantics; 91 }; 92 typedef struct hslock hw_lock_data_t, *hw_lock_t; 93 94 /*! 95 * @enum hw_lock_status_t 96 * 97 * @abstract 98 * Used to pass information about very low level locking primitives. 99 * 100 */ 101 __enum_closed_decl(hw_lock_status_t, int, { 102 /** 103 * The lock was not taken because it is in an invalid state, 104 * or the memory was unmapped. 105 * 106 * This is only valid for @c *_allow_invalid() variants. 107 * 108 * Preemption is preserved to the caller level for all variants. 109 */ 110 HW_LOCK_INVALID = -1, 111 112 /** 113 * the lock wasn't acquired and is contended / timed out. 114 * 115 * - @c *_nopreempt() variants: preemption level preserved 116 * - @c *_trylock() variants: preemption level preserved 117 * - other variants: preemption is disabled 118 */ 119 HW_LOCK_CONTENDED = 0, 120 121 /** 122 * the lock was acquired successfully 123 * 124 * - @c *_nopreempt() variants: preemption level preserved 125 * - other variants: preemption is disabled 126 */ 127 HW_LOCK_ACQUIRED = 1, 128 }); 129 130 /*! 131 * @enum hw_spin_timeout_status_t 132 * 133 * @abstract 134 * Used by spinlock timeout handlers. 135 * 136 * @const HW_LOCK_TIMEOUT_RETURN 137 * Tell the @c hw_lock*_to* caller to break out of the wait 138 * and return HW_LOCK_CONTENDED. 139 * 140 * @const HW_LOCK_TIMEOUT_CONTINUE 141 * Keep spinning for another "timeout". 142 */ 143 __enum_closed_decl(hw_spin_timeout_status_t, _Bool, { 144 HW_LOCK_TIMEOUT_RETURN, /**< return without taking the lock */ 145 HW_LOCK_TIMEOUT_CONTINUE, /**< keep spinning */ 146 }); 147 148 149 /*! 150 * @typedef hw_spin_timeout_t 151 * 152 * @abstract 153 * Describes the timeout used for a given spinning session. 154 */ 155 typedef struct { 156 uint64_t hwst_timeout; 157 #if SCHED_HYGIENE_DEBUG 158 bool hwst_in_ppl; 159 bool hwst_interruptible; 160 #endif /* SCHED_HYGIENE_DEBUG */ 161 } hw_spin_timeout_t; 162 163 164 /*! 165 * @typedef hw_spin_state_t 166 * 167 * @abstract 168 * Keeps track of the various timings used for spinning 169 */ 170 typedef struct { 171 uint64_t hwss_start; 172 uint64_t hwss_now; 173 uint64_t hwss_deadline; 174 #if SCHED_HYGIENE_DEBUG 175 uint64_t hwss_irq_start; 176 uint64_t hwss_irq_end; 177 #endif /* SCHED_HYGIENE_DEBUG */ 178 } hw_spin_state_t; 179 180 181 /*! 182 * @typedef hw_spin_timeout_fn_t 183 * 184 * @abstract 185 * The type of the timeout handlers for low level locking primitives. 186 * 187 * @discussion 188 * Typical handlers are written to just panic and not return 189 * unless some very specific conditions are met (debugging, ...). 190 * 191 * For formatting purposes, we provide HW_SPIN_TIMEOUT{,_DETAILS}{_FMT,_ARG} 192 * 193 * Those are meant to be used inside an hw_spin_timeout_fn_t function 194 * to form informative panic strings, like this: 195 * 196 * panic("MyLock[%p] " HW_SPIN_TIMEOUT_FMT "; " 197 * "<lock specific things> " HW_SPIN_TIMEOUT_DETAILS_FMT, 198 * lock_address, HW_SPIN_TIMEOUT_ARG(to, st), 199 * <lock specific args>, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); 200 * 201 * This ensures consistent panic string style, and transparent adoption 202 * for any new diagnostic/debugging features at all call-sites. 203 */ 204 typedef hw_spin_timeout_status_t (hw_spin_timeout_fn_t)(void *lock, 205 hw_spin_timeout_t to, hw_spin_state_t st); 206 207 #define HW_SPIN_TIMEOUT_FMT \ 208 "timeout after %llu ticks" 209 #define HW_SPIN_TIMEOUT_ARG(to, st) \ 210 ((st).hwss_now - (st).hwss_start) 211 212 #if SCHED_HYGIENE_DEBUG 213 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT \ 214 ", irq time: %llu" 215 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) \ 216 , ((st).hwss_irq_end - (st).hwss_irq_start) 217 #else 218 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT 219 #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) 220 #endif 221 222 #define HW_SPIN_TIMEOUT_DETAILS_FMT \ 223 "start time: %llu, now: %llu, timeout: %llu" \ 224 HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT 225 #define HW_SPIN_TIMEOUT_DETAILS_ARG(to, st) \ 226 (st).hwss_start, (st).hwss_now, (to).hwst_timeout \ 227 HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) 228 229 /*! 230 * @struct hw_spin_policy 231 * 232 * @abstract 233 * Describes the spinning policy for a given lock. 234 */ 235 struct hw_spin_policy { 236 const char *hwsp_name; 237 union { 238 const uint64_t *hwsp_timeout; 239 const _Atomic uint64_t *hwsp_timeout_atomic; 240 }; 241 uint16_t hwsp_timeout_shift; 242 uint16_t hwsp_lock_offset; 243 244 hw_spin_timeout_fn_t *hwsp_op_timeout; 245 }; 246 247 #if __x86_64__ 248 #define LCK_MTX_USE_ARCH 1 249 #else 250 #define LCK_MTX_USE_ARCH 0 251 #endif 252 #endif /* MACH_KERNEL_PRIVATE */ 253 254 __END_DECLS 255 256 #endif /* _KERN_LOCK_TYPES_H */ 257