/* * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * Copyright (C) 1998 Apple Computer * All Rights Reserved */ /* * @OSF_COPYRIGHT@ */ /* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ /* * File: kern/simple_lock.h (derived from kern/lock.h) * Author: Avadis Tevanian, Jr., Michael Wayne Young * Date: 1985 * * Atomic primitives and Simple Locking primitives definitions */ #ifdef KERNEL_PRIVATE #ifndef _KERN_SIMPLE_LOCK_H_ #define _KERN_SIMPLE_LOCK_H_ #include #include #include #include #ifdef XNU_KERNEL_PRIVATE #if MACH_KERNEL_PRIVATE #include #include #endif __BEGIN_DECLS #pragma GCC visibility push(hidden) #ifdef MACH_KERNEL_PRIVATE #define HW_LOCK_STATE_TO_THREAD(state) ((thread_t)(state)) #define HW_LOCK_THREAD_TO_STATE(thread) ((uintptr_t)(thread)) extern void hw_lock_init( hw_lock_t); extern void hw_lock_lock( hw_lock_t LCK_GRP_ARG(lck_grp_t*)); extern void hw_lock_lock_nopreempt( hw_lock_t LCK_GRP_ARG(lck_grp_t*)); extern unsigned int hw_lock_to( hw_lock_t, hw_spin_policy_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; extern unsigned int hw_lock_to_nopreempt( hw_lock_t, hw_spin_policy_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; extern unsigned int hw_lock_try( hw_lock_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; extern unsigned int hw_lock_try_nopreempt( hw_lock_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; #if !LCK_GRP_USE_ARG #define hw_lock_lock(lck, grp) \ hw_lock_lock(lck) #define hw_lock_lock_nopreempt(lck, grp) \ hw_lock_lock_nopreempt(lck) #define hw_lock_to(lck, spec, grp) \ hw_lock_to(lck, spec) #define hw_lock_to_nopreempt(lck, spec, grp) \ hw_lock_to_nopreempt(lck, spec) #define hw_lock_try(lck, grp) \ hw_lock_try(lck) #define hw_lock_try_nopreempt(lck, grp) \ hw_lock_try_nopreempt(lck) #endif /* !LCK_GRP_USE_ARG */ extern void hw_lock_unlock( hw_lock_t); extern void hw_lock_unlock_nopreempt( hw_lock_t); extern void hw_lock_assert( hw_lock_t lock, unsigned int type); extern unsigned int hw_lock_held( hw_lock_t) __result_use_check; extern boolean_t hw_atomic_test_and_set32( uint32_t *target, uint32_t test_mask, uint32_t set_mask, enum memory_order ord, boolean_t wait); extern boolean_t atomic_test_and_set32( uint32_t *target, uint32_t test_mask, uint32_t set_mask, enum memory_order ord, boolean_t wait); extern void atomic_exchange_abort( void); extern boolean_t atomic_exchange_complete32( uint32_t *target, uint32_t previous, uint32_t newval, enum memory_order ord); extern uint32_t atomic_exchange_begin32( uint32_t *target, uint32_t *previous, enum memory_order ord); #if defined(__arm__) || defined(__arm64__) uint32_t load_exclusive32( uint32_t *target, enum memory_order ord); boolean_t store_exclusive32( uint32_t *target, uint32_t value, enum memory_order ord); #endif /* defined(__arm__)||defined(__arm64__) */ extern void usimple_unlock_nopreempt( usimple_lock_t); extern hw_spin_timeout_t hw_spin_compute_timeout( hw_spin_policy_t policy); extern bool hw_spin_in_ppl( hw_spin_timeout_t to) __pure2; extern bool hw_spin_should_keep_spinning( void *lock, hw_spin_policy_t policy, hw_spin_timeout_t to, hw_spin_state_t *state); #endif /* MACH_KERNEL_PRIVATE */ struct usimple_lock_startup_spec { usimple_lock_t lck; unsigned short lck_init_arg; }; extern void usimple_lock_startup_init( struct usimple_lock_startup_spec *spec); #define SIMPLE_LOCK_DECLARE(var, arg) \ decl_simple_lock_data(, var); \ static __startup_data struct usimple_lock_startup_spec \ __startup_usimple_lock_spec_ ## var = { &var, arg }; \ STARTUP_ARG(LOCKS, STARTUP_RANK_FOURTH, usimple_lock_startup_init, \ &__startup_usimple_lock_spec_ ## var) extern uint32_t hw_wait_while_equals32( uint32_t *address, uint32_t current); extern uint64_t hw_wait_while_equals64( uint64_t *address, uint64_t current); #if __LP64__ #define hw_wait_while_equals_long(ptr, cur) ({ \ static_assert(sizeof(*(ptr)) == sizeof(long)); \ (typeof(cur))hw_wait_while_equals64(__DEVOLATILE(uint64_t *, ptr), (uint64_t)(cur)); \ }) #else #define hw_wait_while_equals_long(ptr, cur) ({ \ static_assert(sizeof(*(ptr)) == sizeof(long)); \ (typeof(cur))hw_wait_while_equals32(__DEVOLATILE(uint32_t *, ptr), (uint32_t)(cur)); \ }) #endif extern void usimple_lock_init( usimple_lock_t, unsigned short); extern void usimple_lock( usimple_lock_t LCK_GRP_ARG(lck_grp_t*)); extern unsigned int usimple_lock_try( usimple_lock_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; extern void usimple_lock_assert( usimple_lock_t lock, unsigned int type); extern void usimple_lock_try_lock_loop( usimple_lock_t LCK_GRP_ARG(lck_grp_t*)); #if defined(__x86_64__) extern unsigned int usimple_lock_try_lock_mp_signal_safe_loop_deadline( usimple_lock_t, uint64_t LCK_GRP_ARG(lck_grp_t*)) /* __result_use_check */; extern unsigned int usimple_lock_try_lock_mp_signal_safe_loop_duration( usimple_lock_t, uint64_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; #endif extern void usimple_unlock( usimple_lock_t); #if !LCK_GRP_USE_ARG #define usimple_lock(lck, grp) \ usimple_lock(lck) #define usimple_lock_try(lck, grp) \ usimple_lock_try(lck) #define usimple_lock_try_lock_loop(lck, grp) \ usimple_lock_try_lock_loop(lck) #if defined(__x86_64__) #define usimple_lock_try_lock_mp_signal_safe_loop_deadline(lck, ddl, grp) \ usimple_lock_try_lock_mp_signal_safe_loop_deadline(lck, ddl) #define usimple_lock_try_lock_mp_signal_safe_loop_duration(lck, dur, grp) \ usimple_lock_try_lock_mp_signal_safe_loop_duration(lck, dur) #endif #endif /* !LCK_GRP_USE_ARG */ /* * If we got to here and we still don't have simple_lock_init * defined, then we must either be outside the osfmk component, * running on a true SMP, or need debug. */ #if !defined(simple_lock_init) #define simple_lock_init(l, t) usimple_lock_init(l,t) #define simple_lock(l, grp) usimple_lock(l, grp) #define simple_unlock(l) usimple_unlock(l) #define simple_lock_assert(l, x) usimple_lock_assert((l), (x)) #define simple_lock_try(l, grp) usimple_lock_try(l, grp) #define simple_lock_try_lock_loop(l, grp) usimple_lock_try_lock_loop(l, grp) #define simple_lock_try_lock_mp_signal_safe_loop_deadline(l, ddl, grp) \ usimple_lock_try_lock_mp_signal_safe_loop_deadline(l, ddl, grp) #define simple_lock_try_lock_mp_signal_safe_loop_duration(l, dur, grp) \ usimple_lock_try_lock_mp_signal_safe_loop_duration(l, dur, grp) #define simple_lock_addr(l) (&(l)) #endif /* !defined(simple_lock_init) */ #ifdef MACH_KERNEL_PRIVATE typedef uint32_t hw_lock_bit_t; extern const struct hw_spin_policy hw_lock_bit_policy; #if __arm64__ extern const struct hw_spin_policy hw_lock_bit_policy_2s; #endif extern const struct hw_spin_policy hw_lock_spin_policy; extern const struct hw_spin_policy hw_lock_spin_panic_policy; #if DEBUG || DEVELOPMENT extern const struct hw_spin_policy hw_lock_test_give_up_policy; #endif /* DEBUG || DEVELOPMENT */ extern void hw_lock_bit( hw_lock_bit_t *, unsigned int LCK_GRP_ARG(lck_grp_t*)); extern void hw_lock_bit_nopreempt( hw_lock_bit_t *, unsigned int LCK_GRP_ARG(lck_grp_t*)); extern bool hw_lock_bit_try( hw_lock_bit_t *, unsigned int LCK_GRP_ARG(lck_grp_t*)) __result_use_check; extern unsigned int hw_lock_bit_to( hw_lock_bit_t *, unsigned int, hw_spin_policy_t LCK_GRP_ARG(lck_grp_t*)) __result_use_check; extern hw_lock_status_t hw_lock_bit_to_b( hw_lock_bit_t *, unsigned int, hw_spin_policy_t, bool (^lock_pause)(void) LCK_GRP_ARG(lck_grp_t*)); extern void hw_unlock_bit( hw_lock_bit_t *, unsigned int); extern void hw_unlock_bit_nopreempt( hw_lock_bit_t *, unsigned int); #define hw_lock_bit_held(l, b) \ (((*(l)) & (1 << (b))) != 0) #if !LCK_GRP_USE_ARG #define hw_lock_bit(lck, bit, grp) \ hw_lock_bit(lck, bit) #define hw_lock_bit_nopreempt(lck, bit, grp) \ hw_lock_bit_nopreempt(lck, bit) #define hw_lock_bit_try(lck, bit, grp) \ hw_lock_bit_try(lck, bit) #define hw_lock_bit_to(lck, bit, spec, grp) \ hw_lock_bit_to(lck, bit, spec) #define hw_lock_bit_to_b(lck, bit, spec, pause, grp) \ hw_lock_bit_to_b(lck, bit, spec, pause) #endif /* !LCK_GRP_USE_ARG */ #endif /* MACH_KERNEL_PRIVATE */ __END_DECLS #pragma GCC visibility pop #endif /* XNU_KERNEL_PRIVATE */ #endif /*!_KERN_SIMPLE_LOCK_H_*/ #endif /* KERNEL_PRIVATE */