/* * Copyright (c) 2000-2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ #define LOCK_PRIVATE 1 #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__x86_64__) #include #include #endif #include #include #include #include #include #define LCK_MTX_SLEEP_CODE 0 #define LCK_MTX_SLEEP_DEADLINE_CODE 1 #define LCK_MTX_LCK_WAIT_CODE 2 #define LCK_MTX_UNLCK_WAKEUP_CODE 3 // Panic in tests that check lock usage correctness // These are undesirable when in a panic or a debugger is runnning. #define LOCK_CORRECTNESS_PANIC() (kernel_debugger_entry_count == 0) #if MACH_LDEBUG #define ALIGN_TEST(p, t) do{if((uintptr_t)p&(sizeof(t)-1)) __builtin_trap();}while(0) #else #define ALIGN_TEST(p, t) do{}while(0) #endif #define NOINLINE __attribute__((noinline)) #define ordered_load_hw(lock) os_atomic_load(&(lock)->lock_data, compiler_acq_rel) #define ordered_store_hw(lock, value) os_atomic_store(&(lock)->lock_data, (value), compiler_acq_rel) KALLOC_TYPE_DEFINE(KT_GATE, gate_t, KT_PRIV_ACCT); struct lck_spinlock_to_info PERCPU_DATA(lck_spinlock_to_info); volatile lck_spinlock_to_info_t lck_spinlock_timeout_in_progress; SECURITY_READ_ONLY_LATE(boolean_t) spinlock_timeout_panic = TRUE; struct lck_tktlock_pv_info PERCPU_DATA(lck_tktlock_pv_info); #if CONFIG_PV_TICKET SECURITY_READ_ONLY_LATE(bool) has_lock_pv = FALSE; /* used by waitq.py */ #endif #if DEBUG TUNABLE(uint32_t, LcksOpts, "lcks", LCK_OPTION_ENABLE_DEBUG); #else TUNABLE(uint32_t, LcksOpts, "lcks", 0); #endif #if CONFIG_DTRACE #if defined (__x86_64__) machine_timeout_t dtrace_spin_threshold = 500; // 500ns #elif defined(__arm64__) MACHINE_TIMEOUT(dtrace_spin_threshold, "dtrace-spin-threshold", 0xC /* 12 ticks == 500ns with 24MHz OSC */, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL); #endif #endif struct lck_mcs PERCPU_DATA(lck_mcs); __kdebug_only uintptr_t unslide_for_kdebug(const void* object) { if (__improbable(kdebug_enable)) { return VM_KERNEL_UNSLIDE_OR_PERM(object); } else { return 0; } } static __abortlike void __lck_require_preemption_disabled_panic(void *lock) { panic("Attempt to take no-preempt lock %p in preemptible context", lock); } static inline void __lck_require_preemption_disabled(void *lock, thread_t self __unused) { if (__improbable(!lock_preemption_disabled_for_thread(self))) { __lck_require_preemption_disabled_panic(lock); } } #pragma mark - HW Spin policies /* * Input and output timeouts are expressed in absolute_time for arm and TSC for Intel */ __attribute__((always_inline)) hw_spin_timeout_t hw_spin_compute_timeout(hw_spin_policy_t pol) { hw_spin_timeout_t ret = { .hwst_timeout = os_atomic_load(pol->hwsp_timeout, relaxed), }; ret.hwst_timeout <<= pol->hwsp_timeout_shift; #if SCHED_HYGIENE_DEBUG ret.hwst_in_ppl = pmap_in_ppl(); /* Note we can't check if we are interruptible if in ppl */ ret.hwst_interruptible = !ret.hwst_in_ppl && ml_get_interrupts_enabled(); #endif /* SCHED_HYGIENE_DEBUG */ #if SCHED_HYGIENE_DEBUG #ifndef KASAN if (ret.hwst_timeout > 0 && !ret.hwst_in_ppl && !ret.hwst_interruptible && interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) { uint64_t int_timeout = os_atomic_load(&interrupt_masked_timeout, relaxed); #if defined(__x86_64__) int_timeout = tmrCvt(int_timeout, tscFCvtn2t); #endif if (int_timeout < ret.hwst_timeout) { ret.hwst_timeout = int_timeout; } } #endif /* !KASAN */ #endif /* SCHED_HYGIENE_DEBUG */ return ret; } __attribute__((always_inline)) bool hw_spin_in_ppl(hw_spin_timeout_t to) { #if SCHED_HYGIENE_DEBUG return to.hwst_in_ppl; #else (void)to; return pmap_in_ppl(); #endif } bool hw_spin_should_keep_spinning( void *lock, hw_spin_policy_t pol, hw_spin_timeout_t to, hw_spin_state_t *state) { hw_spin_timeout_status_t rc; #if SCHED_HYGIENE_DEBUG uint64_t irq_time = 0; #endif uint64_t now; if (__improbable(to.hwst_timeout == 0)) { return true; } now = ml_get_timebase(); if (__probable(now < state->hwss_deadline)) { /* keep spinning */ return true; } #if SCHED_HYGIENE_DEBUG if (to.hwst_interruptible) { irq_time = current_thread()->machine.int_time_mt; } #endif /* SCHED_HYGIENE_DEBUG */ if (__probable(state->hwss_deadline == 0)) { state->hwss_start = now; state->hwss_deadline = now + to.hwst_timeout; #if SCHED_HYGIENE_DEBUG state->hwss_irq_start = irq_time; #endif return true; } /* * Update fields that the callback needs */ state->hwss_now = now; #if SCHED_HYGIENE_DEBUG state->hwss_irq_end = irq_time; #endif /* SCHED_HYGIENE_DEBUG */ rc = pol->hwsp_op_timeout((char *)lock - pol->hwsp_lock_offset, to, *state); if (rc == HW_LOCK_TIMEOUT_CONTINUE) { /* push the deadline */ state->hwss_deadline += to.hwst_timeout; } return rc == HW_LOCK_TIMEOUT_CONTINUE; } __attribute__((always_inline)) void lck_spinlock_timeout_set_orig_owner(uintptr_t owner) { #if DEBUG || DEVELOPMENT PERCPU_GET(lck_spinlock_to_info)->owner_thread_orig = owner & ~0x7ul; #else (void)owner; #endif } __attribute__((always_inline)) void lck_spinlock_timeout_set_orig_ctid(uint32_t ctid) { #if DEBUG || DEVELOPMENT PERCPU_GET(lck_spinlock_to_info)->owner_thread_orig = (uintptr_t)ctid_get_thread_unsafe(ctid); #else (void)ctid; #endif } lck_spinlock_to_info_t lck_spinlock_timeout_hit(void *lck, uintptr_t owner) { lck_spinlock_to_info_t lsti = PERCPU_GET(lck_spinlock_to_info); if (owner < (1u << CTID_SIZE_BIT)) { owner = (uintptr_t)ctid_get_thread_unsafe((uint32_t)owner); } else { /* strip possible bits used by the lock implementations */ owner &= ~0x7ul; } lsti->lock = lck; lsti->owner_thread_cur = owner; lsti->owner_cpu = ~0u; os_atomic_store(&lck_spinlock_timeout_in_progress, lsti, release); if (owner == 0) { /* if the owner isn't known, just bail */ goto out; } for (uint32_t i = 0; i <= ml_early_cpu_max_number(); i++) { cpu_data_t *data = cpu_datap(i); if (data && (uintptr_t)data->cpu_active_thread == owner) { lsti->owner_cpu = i; os_atomic_store(&lck_spinlock_timeout_in_progress, lsti, release); #if __x86_64__ if ((uint32_t)cpu_number() != i) { /* Cause NMI and panic on the owner's cpu */ NMIPI_panic(cpu_to_cpumask(i), SPINLOCK_TIMEOUT); } #endif break; } } out: return lsti; } #pragma mark - HW locks /* * Routine: hw_lock_init * * Initialize a hardware lock. */ MARK_AS_HIBERNATE_TEXT void hw_lock_init(hw_lock_t lock) { ordered_store_hw(lock, 0); } __result_use_check static inline bool hw_lock_trylock_contended(hw_lock_t lock, uintptr_t newval) { #if OS_ATOMIC_USE_LLSC uintptr_t oldval; os_atomic_rmw_loop(&lock->lock_data, oldval, newval, acquire, { if (oldval != 0) { wait_for_event(); // clears the monitor so we don't need give_up() return false; } }); return true; #else // !OS_ATOMIC_USE_LLSC #if OS_ATOMIC_HAS_LLSC uintptr_t oldval = os_atomic_load_exclusive(&lock->lock_data, relaxed); if (oldval != 0) { wait_for_event(); // clears the monitor so we don't need give_up() return false; } #endif return lock_cmpxchg(&lock->lock_data, 0, newval, acquire); #endif // !OS_ATOMIC_USE_LLSC } __result_use_check static inline bool hw_lock_trylock_bit(uint32_t *target, unsigned int bit, bool wait) { uint32_t mask = 1u << bit; #if OS_ATOMIC_USE_LLSC || !OS_ATOMIC_HAS_LLSC uint32_t oldval, newval; os_atomic_rmw_loop(target, oldval, newval, acquire, { newval = oldval | mask; if (__improbable(oldval & mask)) { #if OS_ATOMIC_HAS_LLSC if (wait) { wait_for_event(); // clears the monitor so we don't need give_up() } else { os_atomic_clear_exclusive(); } #else if (wait) { cpu_pause(); } #endif return false; } }); return true; #else uint32_t oldval = os_atomic_load_exclusive(target, relaxed); if (__improbable(oldval & mask)) { if (wait) { wait_for_event(); // clears the monitor so we don't need give_up() } else { os_atomic_clear_exclusive(); } return false; } return (os_atomic_or_orig(target, mask, acquire) & mask) == 0; #endif // !OS_ATOMIC_USE_LLSC && OS_ATOMIC_HAS_LLSC } static hw_spin_timeout_status_t hw_spin_timeout_panic(void *_lock, hw_spin_timeout_t to, hw_spin_state_t st) { hw_lock_t lock = _lock; uintptr_t owner = lock->lock_data & ~0x7ul; lck_spinlock_to_info_t lsti; if (!spinlock_timeout_panic) { /* keep spinning rather than panicing */ return HW_LOCK_TIMEOUT_CONTINUE; } if (pmap_in_ppl()) { /* * This code is used by the PPL and can't write to globals. */ panic("Spinlock[%p] " HW_SPIN_TIMEOUT_FMT "; " "current owner: %p, " HW_SPIN_TIMEOUT_DETAILS_FMT, lock, HW_SPIN_TIMEOUT_ARG(to, st), (void *)owner, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); } // Capture the actual time spent blocked, which may be higher than the timeout // if a misbehaving interrupt stole this thread's CPU time. lsti = lck_spinlock_timeout_hit(lock, owner); panic("Spinlock[%p] " HW_SPIN_TIMEOUT_FMT "; " "current owner: %p (on cpu %d), " #if DEBUG || DEVELOPMENT "initial owner: %p, " #endif /* DEBUG || DEVELOPMENT */ HW_SPIN_TIMEOUT_DETAILS_FMT, lock, HW_SPIN_TIMEOUT_ARG(to, st), (void *)lsti->owner_thread_cur, lsti->owner_cpu, #if DEBUG || DEVELOPMENT (void *)lsti->owner_thread_orig, #endif /* DEBUG || DEVELOPMENT */ HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); } const struct hw_spin_policy hw_lock_spin_policy = { .hwsp_name = "hw_lock_t", .hwsp_timeout_atomic = &lock_panic_timeout, .hwsp_op_timeout = hw_spin_timeout_panic, }; static hw_spin_timeout_status_t hw_spin_always_return(void *_lock, hw_spin_timeout_t to, hw_spin_state_t st) { #pragma unused(_lock, to, st) return HW_LOCK_TIMEOUT_RETURN; } const struct hw_spin_policy hw_lock_spin_panic_policy = { .hwsp_name = "hw_lock_t[panic]", #if defined(__x86_64__) .hwsp_timeout = &LockTimeOutTSC, #else .hwsp_timeout_atomic = &LockTimeOut, #endif .hwsp_timeout_shift = 2, .hwsp_op_timeout = hw_spin_always_return, }; #if DEBUG || DEVELOPMENT static machine_timeout_t hw_lock_test_to; const struct hw_spin_policy hw_lock_test_give_up_policy = { .hwsp_name = "testing policy", #if defined(__x86_64__) .hwsp_timeout = &LockTimeOutTSC, #else .hwsp_timeout_atomic = &LockTimeOut, #endif .hwsp_timeout_shift = 2, .hwsp_op_timeout = hw_spin_always_return, }; __startup_func static void hw_lock_test_to_init(void) { uint64_t timeout; nanoseconds_to_absolutetime(100 * NSEC_PER_USEC, &timeout); #if defined(__x86_64__) timeout = tmrCvt(timeout, tscFCvtn2t); #endif os_atomic_init(&hw_lock_test_to, timeout); } STARTUP(TIMEOUTS, STARTUP_RANK_FIRST, hw_lock_test_to_init); #endif static hw_spin_timeout_status_t hw_lock_bit_timeout_panic(void *_lock, hw_spin_timeout_t to, hw_spin_state_t st) { hw_lock_bit_t *lock = _lock; if (!spinlock_timeout_panic) { /* keep spinning rather than panicing */ return HW_LOCK_TIMEOUT_CONTINUE; } panic("Spinlock[%p] " HW_SPIN_TIMEOUT_FMT "; " "current value: 0x%08x, " HW_SPIN_TIMEOUT_DETAILS_FMT, lock, HW_SPIN_TIMEOUT_ARG(to, st), *lock, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); } const struct hw_spin_policy hw_lock_bit_policy = { .hwsp_name = "hw_lock_bit_t", .hwsp_timeout_atomic = &lock_panic_timeout, .hwsp_op_timeout = hw_lock_bit_timeout_panic, }; #if __arm64__ const uint64_t hw_lock_bit_timeout_2s = 0x3000000; const struct hw_spin_policy hw_lock_bit_policy_2s = { .hwsp_name = "hw_lock_bit_t", .hwsp_timeout = &hw_lock_bit_timeout_2s, .hwsp_op_timeout = hw_lock_bit_timeout_panic, }; #endif /* * Routine: hw_lock_lock_contended * * Spin until lock is acquired or timeout expires. * timeout is in mach_absolute_time ticks. Called with * preemption disabled. */ static hw_lock_status_t NOINLINE hw_lock_lock_contended( hw_lock_t lock, uintptr_t data, hw_spin_policy_t pol LCK_GRP_ARG(lck_grp_t *grp)) { hw_spin_timeout_t to = hw_spin_compute_timeout(pol); hw_spin_state_t state = { }; hw_lock_status_t rc = HW_LOCK_CONTENDED; if (HW_LOCK_STATE_TO_THREAD(lock->lock_data) == HW_LOCK_STATE_TO_THREAD(data) && LOCK_CORRECTNESS_PANIC()) { panic("hwlock: thread %p is trying to lock %p recursively", HW_LOCK_STATE_TO_THREAD(data), lock); } #if CONFIG_DTRACE || LOCK_STATS uint64_t begin = 0; boolean_t stat_enabled = lck_grp_spin_spin_enabled(lock LCK_GRP_ARG(grp)); if (__improbable(stat_enabled)) { begin = mach_absolute_time(); } #endif /* CONFIG_DTRACE || LOCK_STATS */ if (!hw_spin_in_ppl(to)) { /* * This code is used by the PPL and can't write to globals. */ lck_spinlock_timeout_set_orig_owner(lock->lock_data); } do { for (uint32_t i = 0; i < LOCK_SNOOP_SPINS; i++) { cpu_pause(); if (hw_lock_trylock_contended(lock, data)) { lck_grp_spin_update_held(lock LCK_GRP_ARG(grp)); rc = HW_LOCK_ACQUIRED; goto end; } } } while (hw_spin_should_keep_spinning(lock, pol, to, &state)); end: #if CONFIG_DTRACE || LOCK_STATS if (__improbable(stat_enabled)) { lck_grp_spin_update_spin(lock LCK_GRP_ARG(grp), mach_absolute_time() - begin); } lck_grp_spin_update_miss(lock LCK_GRP_ARG(grp)); #endif /* CONFIG_DTRACE || LOCK_STATS */ return rc; } static hw_spin_timeout_status_t hw_wait_while_equals32_panic(void *_lock, hw_spin_timeout_t to, hw_spin_state_t st) { uint32_t *address = _lock; if (!spinlock_timeout_panic) { /* keep spinning rather than panicing */ return HW_LOCK_TIMEOUT_CONTINUE; } panic("wait_while_equals32[%p] " HW_SPIN_TIMEOUT_FMT "; " "current value: 0x%08x, " HW_SPIN_TIMEOUT_DETAILS_FMT, address, HW_SPIN_TIMEOUT_ARG(to, st), *address, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); } static const struct hw_spin_policy hw_wait_while_equals32_policy = { .hwsp_name = "hw_wait_while_equals32", .hwsp_timeout_atomic = &lock_panic_timeout, .hwsp_op_timeout = hw_wait_while_equals32_panic, }; static hw_spin_timeout_status_t hw_wait_while_equals64_panic(void *_lock, hw_spin_timeout_t to, hw_spin_state_t st) { uint64_t *address = _lock; if (!spinlock_timeout_panic) { /* keep spinning rather than panicing */ return HW_LOCK_TIMEOUT_CONTINUE; } panic("wait_while_equals64[%p] " HW_SPIN_TIMEOUT_FMT "; " "current value: 0x%016llx, " HW_SPIN_TIMEOUT_DETAILS_FMT, address, HW_SPIN_TIMEOUT_ARG(to, st), *address, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); } static const struct hw_spin_policy hw_wait_while_equals64_policy = { .hwsp_name = "hw_wait_while_equals64", .hwsp_timeout_atomic = &lock_panic_timeout, .hwsp_op_timeout = hw_wait_while_equals64_panic, }; uint32_t hw_wait_while_equals32(uint32_t *address, uint32_t current) { hw_spin_policy_t pol = &hw_wait_while_equals32_policy; hw_spin_timeout_t to = hw_spin_compute_timeout(pol); hw_spin_state_t state = { }; uint32_t v; while (__improbable(!hw_spin_wait_until(address, v, v != current))) { hw_spin_should_keep_spinning(address, pol, to, &state); } return v; } uint64_t hw_wait_while_equals64(uint64_t *address, uint64_t current) { hw_spin_policy_t pol = &hw_wait_while_equals64_policy; hw_spin_timeout_t to = hw_spin_compute_timeout(pol); hw_spin_state_t state = { }; uint64_t v; while (__improbable(!hw_spin_wait_until(address, v, v != current))) { hw_spin_should_keep_spinning(address, pol, to, &state); } return v; } __result_use_check static inline hw_lock_status_t hw_lock_to_internal( hw_lock_t lock, thread_t thread, hw_spin_policy_t pol LCK_GRP_ARG(lck_grp_t *grp)) { uintptr_t state = HW_LOCK_THREAD_TO_STATE(thread); if (__probable(hw_lock_trylock_contended(lock, state))) { lck_grp_spin_update_held(lock LCK_GRP_ARG(grp)); return HW_LOCK_ACQUIRED; } return hw_lock_lock_contended(lock, state, pol LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_lock * * Acquire lock, spinning until it becomes available, * return with preemption disabled. */ void (hw_lock_lock)(hw_lock_t lock LCK_GRP_ARG(lck_grp_t *grp)) { thread_t thread = current_thread(); lock_disable_preemption_for_thread(thread); (void)hw_lock_to_internal(lock, thread, &hw_lock_spin_policy LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_lock_nopreempt * * Acquire lock, spinning until it becomes available. */ void (hw_lock_lock_nopreempt)(hw_lock_t lock LCK_GRP_ARG(lck_grp_t *grp)) { thread_t thread = current_thread(); __lck_require_preemption_disabled(lock, thread); (void)hw_lock_to_internal(lock, thread, &hw_lock_spin_policy LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_to * * Acquire lock, spinning until it becomes available or timeout. * Timeout is in mach_absolute_time ticks (TSC in Intel), return with * preemption disabled. */ unsigned int (hw_lock_to)(hw_lock_t lock, hw_spin_policy_t pol LCK_GRP_ARG(lck_grp_t *grp)) { thread_t thread = current_thread(); lock_disable_preemption_for_thread(thread); return (unsigned)hw_lock_to_internal(lock, thread, pol LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_to_nopreempt * * Acquire lock, spinning until it becomes available or timeout. * Timeout is in mach_absolute_time ticks, called and return with * preemption disabled. */ unsigned int (hw_lock_to_nopreempt)(hw_lock_t lock, hw_spin_policy_t pol LCK_GRP_ARG(lck_grp_t *grp)) { thread_t thread = current_thread(); __lck_require_preemption_disabled(lock, thread); return (unsigned)hw_lock_to_internal(lock, thread, pol LCK_GRP_ARG(grp)); } __result_use_check static inline unsigned int hw_lock_try_internal(hw_lock_t lock, thread_t thread LCK_GRP_ARG(lck_grp_t *grp)) { if (__probable(lock_cmpxchg(&lock->lock_data, 0, HW_LOCK_THREAD_TO_STATE(thread), acquire))) { lck_grp_spin_update_held(lock LCK_GRP_ARG(grp)); return true; } return false; } /* * Routine: hw_lock_try * * returns with preemption disabled on success. */ unsigned int (hw_lock_try)(hw_lock_t lock LCK_GRP_ARG(lck_grp_t *grp)) { thread_t thread = current_thread(); lock_disable_preemption_for_thread(thread); unsigned int success = hw_lock_try_internal(lock, thread LCK_GRP_ARG(grp)); if (!success) { lock_enable_preemption(); } return success; } unsigned int (hw_lock_try_nopreempt)(hw_lock_t lock LCK_GRP_ARG(lck_grp_t *grp)) { thread_t thread = current_thread(); __lck_require_preemption_disabled(lock, thread); return hw_lock_try_internal(lock, thread LCK_GRP_ARG(grp)); } #if DEBUG || DEVELOPMENT __abortlike static void __hw_lock_unlock_unowned_panic(hw_lock_t lock) { panic("hwlock: thread %p is trying to lock %p recursively", current_thread(), lock); } #endif /* DEBUG || DEVELOPMENT */ /* * Routine: hw_lock_unlock * * Unconditionally release lock, release preemption level. */ static inline void hw_lock_unlock_internal(hw_lock_t lock) { #if DEBUG || DEVELOPMENT if (HW_LOCK_STATE_TO_THREAD(lock->lock_data) != current_thread() && LOCK_CORRECTNESS_PANIC()) { __hw_lock_unlock_unowned_panic(lock); } #endif /* DEBUG || DEVELOPMENT */ os_atomic_store(&lock->lock_data, 0, release); #if CONFIG_DTRACE LOCKSTAT_RECORD(LS_LCK_SPIN_UNLOCK_RELEASE, lock, 0); #endif /* CONFIG_DTRACE */ } void (hw_lock_unlock)(hw_lock_t lock) { hw_lock_unlock_internal(lock); lock_enable_preemption(); } void (hw_lock_unlock_nopreempt)(hw_lock_t lock) { hw_lock_unlock_internal(lock); } void hw_lock_assert(__assert_only hw_lock_t lock, __assert_only unsigned int type) { #if MACH_ASSERT thread_t thread, holder; holder = HW_LOCK_STATE_TO_THREAD(lock->lock_data); thread = current_thread(); if (type == LCK_ASSERT_OWNED) { if (holder == 0) { panic("Lock not owned %p = %p", lock, holder); } if (holder != thread) { panic("Lock not owned by current thread %p = %p", lock, holder); } } else if (type == LCK_ASSERT_NOTOWNED) { if (holder != THREAD_NULL && holder == thread) { panic("Lock owned by current thread %p = %p", lock, holder); } } else { panic("hw_lock_assert(): invalid arg (%u)", type); } #endif /* MACH_ASSERT */ } /* * Routine hw_lock_held, doesn't change preemption state. * N.B. Racy, of course. */ unsigned int hw_lock_held(hw_lock_t lock) { return ordered_load_hw(lock) != 0; } static hw_lock_status_t NOINLINE hw_lock_bit_to_contended( hw_lock_bit_t *lock, uint32_t bit, hw_spin_policy_t pol, bool (^lock_pause)(void) LCK_GRP_ARG(lck_grp_t *grp)) { hw_spin_timeout_t to = hw_spin_compute_timeout(pol); hw_spin_state_t state = { }; hw_lock_status_t rc = HW_LOCK_CONTENDED; #if CONFIG_DTRACE || LOCK_STATS uint64_t begin = 0; boolean_t stat_enabled = lck_grp_spin_spin_enabled(lock LCK_GRP_ARG(grp)); if (__improbable(stat_enabled)) { begin = mach_absolute_time(); } #endif /* LOCK_STATS || CONFIG_DTRACE */ do { for (int i = 0; i < LOCK_SNOOP_SPINS; i++) { rc = (hw_lock_trylock_bit(lock, bit, true) ? HW_LOCK_ACQUIRED : HW_LOCK_CONTENDED); if (rc == HW_LOCK_ACQUIRED) { lck_grp_spin_update_held(lock LCK_GRP_ARG(grp)); goto end; } if (__improbable(lock_pause && lock_pause())) { goto end; } } assert(rc == HW_LOCK_CONTENDED); } while (hw_spin_should_keep_spinning(lock, pol, to, &state)); end: #if CONFIG_DTRACE || LOCK_STATS if (__improbable(stat_enabled)) { lck_grp_spin_update_spin(lock LCK_GRP_ARG(grp), mach_absolute_time() - begin); } lck_grp_spin_update_miss(lock LCK_GRP_ARG(grp)); #endif /* CONFIG_DTRACE || LCK_GRP_STAT */ return rc; } __result_use_check static inline hw_lock_status_t hw_lock_bit_to_internal( hw_lock_bit_t *lock, unsigned int bit, hw_spin_policy_t pol, bool (^lock_pause)(void) LCK_GRP_ARG(lck_grp_t *grp)) { if (__probable(hw_lock_trylock_bit(lock, bit, true))) { lck_grp_spin_update_held(lock LCK_GRP_ARG(grp)); return HW_LOCK_ACQUIRED; } return hw_lock_bit_to_contended(lock, bit, pol, lock_pause LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_bit_to * * Acquire bit lock, spinning until it becomes available or timeout. * Timeout is in mach_absolute_time ticks (TSC in Intel), return with * preemption disabled. */ unsigned int (hw_lock_bit_to)( hw_lock_bit_t * lock, uint32_t bit, hw_spin_policy_t pol LCK_GRP_ARG(lck_grp_t *grp)) { _disable_preemption(); return (unsigned int)hw_lock_bit_to_internal(lock, bit, pol, NULL LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_bit * * Acquire bit lock, spinning until it becomes available, * return with preemption disabled. */ void (hw_lock_bit)(hw_lock_bit_t * lock, unsigned int bit LCK_GRP_ARG(lck_grp_t *grp)) { _disable_preemption(); (void)hw_lock_bit_to_internal(lock, bit, &hw_lock_bit_policy, NULL LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_bit_nopreempt * * Acquire bit lock with preemption already disabled, spinning until it becomes available. */ void (hw_lock_bit_nopreempt)(hw_lock_bit_t * lock, unsigned int bit LCK_GRP_ARG(lck_grp_t *grp)) { __lck_require_preemption_disabled(lock, current_thread()); (void)hw_lock_bit_to_internal(lock, bit, &hw_lock_bit_policy, NULL LCK_GRP_ARG(grp)); } /* * Routine: hw_lock_bit_to_b * * Acquire bit lock, spinning until it becomes available, times out, * or the supplied lock_pause callout returns true. * Timeout is in mach_absolute_time ticks (TSC in Intel), return with * preemption disabled iff the lock is successfully acquired. */ hw_lock_status_t (hw_lock_bit_to_b)( hw_lock_bit_t * lock, uint32_t bit, hw_spin_policy_t pol, bool (^lock_pause) (void) LCK_GRP_ARG(lck_grp_t * grp)) { _disable_preemption(); hw_lock_status_t ret = hw_lock_bit_to_internal(lock, bit, pol, lock_pause LCK_GRP_ARG(grp)); if (ret != HW_LOCK_ACQUIRED) { lock_enable_preemption(); } return ret; } bool (hw_lock_bit_try)(hw_lock_bit_t * lock, unsigned int bit LCK_GRP_ARG(lck_grp_t *grp)) { bool success = false; _disable_preemption(); success = hw_lock_trylock_bit(lock, bit, false); if (!success) { lock_enable_preemption(); } if (success) { lck_grp_spin_update_held(lock LCK_GRP_ARG(grp)); } return success; } static inline void hw_unlock_bit_internal(hw_lock_bit_t *lock, unsigned int bit) { os_atomic_andnot(lock, 1u << bit, release); #if CONFIG_DTRACE LOCKSTAT_RECORD(LS_LCK_SPIN_UNLOCK_RELEASE, lock, bit); #endif } /* * Routine: hw_unlock_bit * * Release spin-lock. The second parameter is the bit number to test and set. * Decrement the preemption level. */ void hw_unlock_bit(hw_lock_bit_t * lock, unsigned int bit) { hw_unlock_bit_internal(lock, bit); lock_enable_preemption(); } void hw_unlock_bit_nopreempt(hw_lock_bit_t * lock, unsigned int bit) { __lck_require_preemption_disabled(lock, current_thread()); hw_unlock_bit_internal(lock, bit); } #pragma mark - lck_*_sleep /* * Routine: lck_spin_sleep */ wait_result_t lck_spin_sleep_grp( lck_spin_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, lck_grp_t *grp) { wait_result_t res; if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) { panic("Invalid lock sleep action %x", lck_sleep_action); } res = assert_wait(event, interruptible); if (res == THREAD_WAITING) { lck_spin_unlock(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { lck_spin_lock_grp(lck, grp); } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) { lck_spin_unlock(lck); } return res; } wait_result_t lck_spin_sleep( lck_spin_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible) { return lck_spin_sleep_grp(lck, lck_sleep_action, event, interruptible, LCK_GRP_NULL); } /* * Routine: lck_spin_sleep_deadline */ wait_result_t lck_spin_sleep_deadline( lck_spin_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline) { wait_result_t res; if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) { panic("Invalid lock sleep action %x", lck_sleep_action); } res = assert_wait_deadline(event, interruptible, deadline); if (res == THREAD_WAITING) { lck_spin_unlock(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { lck_spin_lock(lck); } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) { lck_spin_unlock(lck); } return res; } /* * Routine: lck_mtx_sleep */ wait_result_t lck_mtx_sleep( lck_mtx_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible) { wait_result_t res; thread_pri_floor_t token; KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START, VM_KERNEL_UNSLIDE_OR_PERM(lck), (int)lck_sleep_action, VM_KERNEL_UNSLIDE_OR_PERM(event), (int)interruptible, 0); if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) { panic("Invalid lock sleep action %x", lck_sleep_action); } if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { /* * We get a priority floor * during the time that this thread is asleep, so that when it * is re-awakened (and not yet contending on the mutex), it is * runnable at a reasonably high priority. */ token = thread_priority_floor_start(); } res = assert_wait(event, interruptible); if (res == THREAD_WAITING) { lck_mtx_unlock(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if ((lck_sleep_action & LCK_SLEEP_SPIN)) { lck_mtx_lock_spin(lck); } else if ((lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS)) { lck_mtx_lock_spin_always(lck); } else { lck_mtx_lock(lck); } } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) { lck_mtx_unlock(lck); } if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { thread_priority_floor_end(&token); } KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); return res; } /* * Routine: lck_mtx_sleep_deadline */ wait_result_t lck_mtx_sleep_deadline( lck_mtx_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline) { wait_result_t res; thread_pri_floor_t token; KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START, VM_KERNEL_UNSLIDE_OR_PERM(lck), (int)lck_sleep_action, VM_KERNEL_UNSLIDE_OR_PERM(event), (int)interruptible, 0); if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) { panic("Invalid lock sleep action %x", lck_sleep_action); } if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { /* * See lck_mtx_sleep(). */ token = thread_priority_floor_start(); } res = assert_wait_deadline(event, interruptible, deadline); if (res == THREAD_WAITING) { lck_mtx_unlock(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if ((lck_sleep_action & LCK_SLEEP_SPIN)) { lck_mtx_lock_spin(lck); } else { lck_mtx_lock(lck); } } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) { lck_mtx_unlock(lck); } if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { thread_priority_floor_end(&token); } KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); return res; } /* * sleep_with_inheritor and wakeup_with_inheritor KPI * * Functions that allow to sleep on an event and use turnstile to propagate the priority of the sleeping threads to * the latest thread specified as inheritor. * * The inheritor management is delegated to the caller, the caller needs to store a thread identifier to provide to this functions to specified upon whom * direct the push. The inheritor cannot return to user space or exit while holding a push from an event. Therefore is the caller responsibility to call a * wakeup_with_inheritor from inheritor before running in userspace or specify another inheritor before letting the old inheritor run in userspace. * * sleep_with_inheritor requires to hold a locking primitive while invoked, but wakeup_with_inheritor and change_sleep_inheritor don't require it. * * Turnstile requires a non blocking primitive as interlock to synchronize the turnstile data structure manipulation, threfore sleep_with_inheritor, change_sleep_inheritor and * wakeup_with_inheritor will require the same interlock to manipulate turnstiles. * If sleep_with_inheritor is associated with a locking primitive that can block (like lck_mtx_t or lck_rw_t), an handoff to a non blocking primitive is required before * invoking any turnstile operation. * * All functions will save the turnstile associated with the event on the turnstile kernel hash table and will use the the turnstile kernel hash table bucket * spinlock as the turnstile interlock. Because we do not want to hold interrupt disabled while holding the bucket interlock a new turnstile kernel hash table * is instantiated for this KPI to manage the hash without interrupt disabled. * Also: * - all events on the system that hash on the same bucket will contend on the same spinlock. * - every event will have a dedicated wait_queue. * * Different locking primitives can be associated with sleep_with_inheritor as long as the primitive_lock() and primitive_unlock() functions are provided to * sleep_with_inheritor_turnstile to perform the handoff with the bucket spinlock. */ typedef enum { LCK_WAKEUP_THREAD, LCK_WAKEUP_ONE, LCK_WAKEUP_ALL } lck_wakeup_type_t; static kern_return_t wakeup_with_inheritor_and_turnstile( event_t event, wait_result_t result, lck_wakeup_type_t wake_type, lck_wake_action_t action, thread_t *thread_wokenup) { turnstile_type_t type = TURNSTILE_SLEEP_INHERITOR; uint32_t index; struct turnstile *ts = NULL; kern_return_t ret = KERN_NOT_WAITING; /* * the hash bucket spinlock is used as turnstile interlock */ turnstile_hash_bucket_lock((uintptr_t)event, &index, type); ts = turnstile_prepare_hash((uintptr_t)event, type); switch (wake_type) { case LCK_WAKEUP_ONE: { waitq_wakeup_flags_t flags = WAITQ_WAKEUP_DEFAULT; if (action == LCK_WAKE_DEFAULT) { flags = WAITQ_UPDATE_INHERITOR; } else { assert(action == LCK_WAKE_DO_NOT_TRANSFER_PUSH); } /* * WAITQ_UPDATE_INHERITOR will call turnstile_update_inheritor * if it finds a thread */ if (thread_wokenup) { thread_t wokeup; wokeup = waitq_wakeup64_identify(&ts->ts_waitq, CAST_EVENT64_T(event), result, flags); *thread_wokenup = wokeup; ret = wokeup ? KERN_SUCCESS : KERN_NOT_WAITING; } else { ret = waitq_wakeup64_one(&ts->ts_waitq, CAST_EVENT64_T(event), result, flags); } if (ret == KERN_SUCCESS && action == LCK_WAKE_DO_NOT_TRANSFER_PUSH) { goto complete; } if (ret == KERN_NOT_WAITING) { turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE); } break; } case LCK_WAKEUP_ALL: { ret = waitq_wakeup64_all(&ts->ts_waitq, CAST_EVENT64_T(event), result, WAITQ_UPDATE_INHERITOR); break; } case LCK_WAKEUP_THREAD: { assert(thread_wokenup); ret = waitq_wakeup64_thread(&ts->ts_waitq, CAST_EVENT64_T(event), *thread_wokenup, result); break; } } /* * turnstile_update_inheritor_complete could be called while holding the interlock. * In this case the new inheritor or is null, or is a thread that is just been woken up * and have not blocked because it is racing with the same interlock used here * after the wait. * So there is no chain to update for the new inheritor. * * However unless the current thread is the old inheritor, * old inheritor can be blocked and requires a chain update. * * The chain should be short because kernel turnstiles cannot have user turnstiles * chained after them. * * We can anyway optimize this by asking turnstile to tell us * if old inheritor needs an update and drop the lock * just in that case. */ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type); complete: turnstile_complete_hash((uintptr_t)event, type); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); turnstile_cleanup(); return ret; } static wait_result_t sleep_with_inheritor_and_turnstile( event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, void (^primitive_lock)(void), void (^primitive_unlock)(void)) { turnstile_type_t type = TURNSTILE_SLEEP_INHERITOR; wait_result_t ret; uint32_t index; struct turnstile *ts = NULL; /* * the hash bucket spinlock is used as turnstile interlock, * lock it before releasing the primitive lock */ turnstile_hash_bucket_lock((uintptr_t)event, &index, type); primitive_unlock(); ts = turnstile_prepare_hash((uintptr_t)event, type); thread_set_pending_block_hint(current_thread(), kThreadWaitSleepWithInheritor); /* * We need TURNSTILE_DELAYED_UPDATE because we will call * waitq_assert_wait64 after. */ turnstile_update_inheritor(ts, inheritor, (TURNSTILE_DELAYED_UPDATE | TURNSTILE_INHERITOR_THREAD)); ret = waitq_assert_wait64(&ts->ts_waitq, CAST_EVENT64_T(event), interruptible, deadline); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); /* * Update new and old inheritor chains outside the interlock; */ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); if (ret == THREAD_WAITING) { ret = thread_block(THREAD_CONTINUE_NULL); } turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type); turnstile_complete_hash((uintptr_t)event, type); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); turnstile_cleanup(); primitive_lock(); return ret; } /* * change_sleep_inheritor is independent from the locking primitive. */ /* * Name: change_sleep_inheritor * * Description: Redirect the push of the waiting threads of event to the new inheritor specified. * * Args: * Arg1: event to redirect the push. * Arg2: new inheritor for event. * * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. * * Conditions: In case of success, the new inheritor cannot return to user space or exit until another inheritor is specified for the event or a * wakeup for the event is called. * NOTE: this cannot be called from interrupt context. */ kern_return_t change_sleep_inheritor(event_t event, thread_t inheritor) { uint32_t index; struct turnstile *ts = NULL; kern_return_t ret = KERN_SUCCESS; turnstile_type_t type = TURNSTILE_SLEEP_INHERITOR; /* * the hash bucket spinlock is used as turnstile interlock */ turnstile_hash_bucket_lock((uintptr_t)event, &index, type); ts = turnstile_prepare_hash((uintptr_t)event, type); if (!turnstile_has_waiters(ts)) { ret = KERN_NOT_WAITING; } /* * We will not call an assert_wait later so use TURNSTILE_IMMEDIATE_UPDATE */ turnstile_update_inheritor(ts, inheritor, (TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_THREAD)); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); /* * update the chains outside the interlock */ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type); turnstile_complete_hash((uintptr_t)event, type); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); turnstile_cleanup(); return ret; } wait_result_t lck_spin_sleep_with_inheritor( lck_spin_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline) { if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{}, ^{ lck_spin_unlock(lock); }); } else { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{ lck_spin_lock(lock); }, ^{ lck_spin_unlock(lock); }); } } wait_result_t hw_lck_ticket_sleep_with_inheritor( hw_lck_ticket_t *lock, lck_grp_t *grp __unused, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline) { if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{}, ^{ hw_lck_ticket_unlock(lock); }); } else { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{ hw_lck_ticket_lock(lock, grp); }, ^{ hw_lck_ticket_unlock(lock); }); } } wait_result_t lck_ticket_sleep_with_inheritor( lck_ticket_t *lock, lck_grp_t *grp, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline) { if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{}, ^{ lck_ticket_unlock(lock); }); } else { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{ lck_ticket_lock(lock, grp); }, ^{ lck_ticket_unlock(lock); }); } } wait_result_t lck_mtx_sleep_with_inheritor( lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{;}, ^{lck_mtx_unlock(lock);}); } else if (lck_sleep_action & LCK_SLEEP_SPIN) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{lck_mtx_lock_spin(lock);}, ^{lck_mtx_unlock(lock);}); } else if (lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{lck_mtx_lock_spin_always(lock);}, ^{lck_mtx_unlock(lock);}); } else { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{lck_mtx_lock(lock);}, ^{lck_mtx_unlock(lock);}); } } /* * sleep_with_inheritor functions with lck_rw_t as locking primitive. */ wait_result_t lck_rw_sleep_with_inheritor( lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline) { __block lck_rw_type_t lck_rw_type = LCK_RW_TYPE_EXCLUSIVE; LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{;}, ^{lck_rw_type = lck_rw_done(lock);}); } else if (!(lck_sleep_action & (LCK_SLEEP_SHARED | LCK_SLEEP_EXCLUSIVE))) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{lck_rw_lock(lock, lck_rw_type);}, ^{lck_rw_type = lck_rw_done(lock);}); } else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{lck_rw_lock_exclusive(lock);}, ^{lck_rw_type = lck_rw_done(lock);}); } else { return sleep_with_inheritor_and_turnstile(event, inheritor, interruptible, deadline, ^{lck_rw_lock_shared(lock);}, ^{lck_rw_type = lck_rw_done(lock);}); } } /* * wakeup_with_inheritor functions are independent from the locking primitive. */ kern_return_t wakeup_thread_with_inheritor(event_t event, wait_result_t result, lck_wake_action_t action, thread_t thread_towake) { return wakeup_with_inheritor_and_turnstile(event, result, LCK_WAKEUP_THREAD, action, &thread_towake); } kern_return_t wakeup_one_with_inheritor(event_t event, wait_result_t result, lck_wake_action_t action, thread_t *thread_wokenup) { return wakeup_with_inheritor_and_turnstile(event, result, LCK_WAKEUP_ONE, action, thread_wokenup); } kern_return_t wakeup_all_with_inheritor(event_t event, wait_result_t result) { return wakeup_with_inheritor_and_turnstile(event, result, LCK_WAKEUP_ALL, 0, NULL); } void kdp_sleep_with_inheritor_find_owner(struct waitq * waitq, __unused event64_t event, thread_waitinfo_t * waitinfo) { assert(waitinfo->wait_type == kThreadWaitSleepWithInheritor); assert(waitq_type(waitq) == WQT_TURNSTILE); waitinfo->owner = 0; waitinfo->context = 0; if (waitq_held(waitq)) { return; } struct turnstile *turnstile = waitq_to_turnstile(waitq); assert(turnstile->ts_inheritor_flags & TURNSTILE_INHERITOR_THREAD); waitinfo->owner = thread_tid(turnstile->ts_inheritor); } static_assert(SWI_COND_OWNER_BITS == CTID_SIZE_BIT); static_assert(sizeof(cond_swi_var32_s) == sizeof(uint32_t)); static_assert(sizeof(cond_swi_var64_s) == sizeof(uint64_t)); static wait_result_t cond_sleep_with_inheritor_and_turnstile_type( cond_swi_var_t cond, bool (^cond_sleep_check)(ctid_t*), wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type) { wait_result_t ret; uint32_t index; struct turnstile *ts = NULL; ctid_t ctid = 0; thread_t inheritor; /* * the hash bucket spinlock is used as turnstile interlock, * lock it before checking the sleep condition */ turnstile_hash_bucket_lock((uintptr_t)cond, &index, type); /* * In case the sleep check succeeds, the block will * provide us the ctid observed on the variable. */ if (!cond_sleep_check(&ctid)) { turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); return THREAD_NOT_WAITING; } /* * We can translate the ctid to a thread_t only * if cond_sleep_check succeded. */ inheritor = ctid_get_thread(ctid); assert(inheritor != NULL); ts = turnstile_prepare_hash((uintptr_t)cond, type); thread_set_pending_block_hint(current_thread(), kThreadWaitSleepWithInheritor); /* * We need TURNSTILE_DELAYED_UPDATE because we will call * waitq_assert_wait64 after. */ turnstile_update_inheritor(ts, inheritor, (TURNSTILE_DELAYED_UPDATE | TURNSTILE_INHERITOR_THREAD)); ret = waitq_assert_wait64(&ts->ts_waitq, CAST_EVENT64_T(cond), interruptible, deadline); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); /* * Update new and old inheritor chains outside the interlock; */ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); if (ret == THREAD_WAITING) { ret = thread_block(THREAD_CONTINUE_NULL); } turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type); turnstile_complete_hash((uintptr_t)cond, type); turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0); turnstile_cleanup(); return ret; } /* * Name: cond_sleep_with_inheritor32_mask * * Description: Conditionally sleeps with inheritor, with condition variable of 32bits. * Allows a thread to conditionally sleep while indicating which thread should * inherit the priority push associated with the condition. * The condition should be expressed through a cond_swi_var32_s pointer. * The condition needs to be populated by the caller with the ctid of the * thread that should inherit the push. The remaining bits of the condition * can be used by the caller to implement its own synchronization logic. * A copy of the condition value observed by the caller when it decided to call * this function should be provided to prevent races with matching wakeups. * This function will atomically check the value stored in the condition against * the expected/observed one provided only for the bits that are set in the mask. * If the check doesn't pass the thread will not sleep and the function will return. * The ctid provided in the condition will be used only after a successful * check. * * Args: * Arg1: cond_swi_var32_s pointer that stores the condition to check. * Arg2: cond_swi_var32_s observed value to check for conditionally sleep. * Arg3: mask to apply to the condition to check. * Arg4: interruptible flag for wait. * Arg5: deadline for wait. * * Conditions: The inheritor specified cannot return to user space or exit until another inheritor is specified for the cond or a * wakeup for the cond is called. * * Returns: result of the wait. */ static wait_result_t cond_sleep_with_inheritor32_mask(cond_swi_var_t cond, cond_swi_var32_s expected_cond, uint32_t check_mask, wait_interrupt_t interruptible, uint64_t deadline) { bool (^cond_sleep_check)(uint32_t*) = ^(ctid_t *ctid) { cond_swi_var32_s cond_val = {.cond32_data = os_atomic_load((uint32_t*) cond, relaxed)}; bool ret; if ((cond_val.cond32_data & check_mask) == (expected_cond.cond32_data & check_mask)) { ret = true; *ctid = cond_val.cond32_owner; } else { ret = false; } return ret; }; return cond_sleep_with_inheritor_and_turnstile_type(cond, cond_sleep_check, interruptible, deadline, TURNSTILE_SLEEP_INHERITOR); } /* * Name: cond_sleep_with_inheritor64_mask * * Description: Conditionally sleeps with inheritor, with condition variable of 64bits. * Allows a thread to conditionally sleep while indicating which thread should * inherit the priority push associated with the condition. * The condition should be expressed through a cond_swi_var64_s pointer. * The condition needs to be populated by the caller with the ctid of the * thread that should inherit the push. The remaining bits of the condition * can be used by the caller to implement its own synchronization logic. * A copy of the condition value observed by the caller when it decided to call * this function should be provided to prevent races with matching wakeups. * This function will atomically check the value stored in the condition against * the expected/observed one provided only for the bits that are set in the mask. * If the check doesn't pass the thread will not sleep and the function will return. * The ctid provided in the condition will be used only after a successful * check. * * Args: * Arg1: cond_swi_var64_s pointer that stores the condition to check. * Arg2: cond_swi_var64_s observed value to check for conditionally sleep. * Arg3: mask to apply to the condition to check. * Arg4: interruptible flag for wait. * Arg5: deadline for wait. * * Conditions: The inheritor specified cannot return to user space or exit until another inheritor is specified for the cond or a * wakeup for the cond is called. * * Returns: result of the wait. */ wait_result_t cond_sleep_with_inheritor64_mask(cond_swi_var_t cond, cond_swi_var64_s expected_cond, uint64_t check_mask, wait_interrupt_t interruptible, uint64_t deadline) { bool (^cond_sleep_check)(uint32_t*) = ^(ctid_t *ctid) { cond_swi_var64_s cond_val = {.cond64_data = os_atomic_load((uint64_t*) cond, relaxed)}; bool ret; if ((cond_val.cond64_data & check_mask) == (expected_cond.cond64_data & check_mask)) { ret = true; *ctid = cond_val.cond64_owner; } else { ret = false; } return ret; }; return cond_sleep_with_inheritor_and_turnstile_type(cond, cond_sleep_check, interruptible, deadline, TURNSTILE_SLEEP_INHERITOR); } /* * Name: cond_sleep_with_inheritor32 * * Description: Conditionally sleeps with inheritor, with condition variable of 32bits. * Allows a thread to conditionally sleep while indicating which thread should * inherit the priority push associated with the condition. * The condition should be expressed through a cond_swi_var32_s pointer. * The condition needs to be populated by the caller with the ctid of the * thread that should inherit the push. The remaining bits of the condition * can be used by the caller to implement its own synchronization logic. * A copy of the condition value observed by the caller when it decided to call * this function should be provided to prevent races with matching wakeups. * This function will atomically check the value stored in the condition against * the expected/observed one provided. If the check doesn't pass the thread will not * sleep and the function will return. * The ctid provided in the condition will be used only after a successful * check. * * Args: * Arg1: cond_swi_var32_s pointer that stores the condition to check. * Arg2: cond_swi_var32_s observed value to check for conditionally sleep. * Arg3: interruptible flag for wait. * Arg4: deadline for wait. * * Conditions: The inheritor specified cannot return to user space or exit until another inheritor is specified for the cond or a * wakeup for the cond is called. * * Returns: result of the wait. */ wait_result_t cond_sleep_with_inheritor32(cond_swi_var_t cond, cond_swi_var32_s expected_cond, wait_interrupt_t interruptible, uint64_t deadline) { return cond_sleep_with_inheritor32_mask(cond, expected_cond, ~0u, interruptible, deadline); } /* * Name: cond_sleep_with_inheritor64 * * Description: Conditionally sleeps with inheritor, with condition variable of 64bits. * Allows a thread to conditionally sleep while indicating which thread should * inherit the priority push associated with the condition. * The condition should be expressed through a cond_swi_var64_s pointer. * The condition needs to be populated by the caller with the ctid of the * thread that should inherit the push. The remaining bits of the condition * can be used by the caller to implement its own synchronization logic. * A copy of the condition value observed by the caller when it decided to call * this function should be provided to prevent races with matching wakeups. * This function will atomically check the value stored in the condition against * the expected/observed one provided. If the check doesn't pass the thread will not * sleep and the function will return. * The ctid provided in the condition will be used only after a successful * check. * * Args: * Arg1: cond_swi_var64_s pointer that stores the condition to check. * Arg2: cond_swi_var64_s observed value to check for conditionally sleep. * Arg3: interruptible flag for wait. * Arg4: deadline for wait. * * Conditions: The inheritor specified cannot return to user space or exit until another inheritor is specified for the cond or a * wakeup for the cond is called. * * Returns: result of the wait. */ wait_result_t cond_sleep_with_inheritor64(cond_swi_var_t cond, cond_swi_var64_s expected_cond, wait_interrupt_t interruptible, uint64_t deadline) { return cond_sleep_with_inheritor64_mask(cond, expected_cond, ~0ull, interruptible, deadline); } /* * Name: cond_wakeup_one_with_inheritor * * Description: Wake up one waiter waiting on the condition (if any). * The thread woken up will be the one with the higher sched priority waiting on the condition. * The push for the condition will be transferred from the last inheritor to the woken up thread. * * Args: * Arg1: condition to wake from. * Arg2: wait result to pass to the woken up thread. * Arg3: pointer for storing the thread wokenup. * * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. * * Conditions: The new inheritor wokenup cannot return to user space or exit until another inheritor is specified for the * condition or a wakeup for the event is called. * A reference for the wokenup thread is acquired. * NOTE: this cannot be called from interrupt context. */ kern_return_t cond_wakeup_one_with_inheritor(cond_swi_var_t cond, wait_result_t result, lck_wake_action_t action, thread_t *thread_wokenup) { return wakeup_with_inheritor_and_turnstile((event_t)cond, result, LCK_WAKEUP_ONE, action, thread_wokenup); } /* * Name: cond_wakeup_all_with_inheritor * * Description: Wake up all waiters waiting on the same condition. The old inheritor will lose the push. * * Args: * Arg1: condition to wake from. * Arg2: wait result to pass to the woken up threads. * * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. * * Conditions: NOTE: this cannot be called from interrupt context. */ kern_return_t cond_wakeup_all_with_inheritor(cond_swi_var_t cond, wait_result_t result) { return wakeup_with_inheritor_and_turnstile((event_t)cond, result, LCK_WAKEUP_ALL, 0, NULL); } #pragma mark - gates #define GATE_TYPE 3 #define GATE_ILOCK_BIT 0 #define GATE_WAITERS_BIT 1 #define GATE_ILOCK (1 << GATE_ILOCK_BIT) #define GATE_WAITERS (1 << GATE_WAITERS_BIT) #define gate_ilock(gate) hw_lock_bit((hw_lock_bit_t*)(&(gate)->gt_data), GATE_ILOCK_BIT, LCK_GRP_NULL) #define gate_iunlock(gate) hw_unlock_bit((hw_lock_bit_t*)(&(gate)->gt_data), GATE_ILOCK_BIT) #define gate_has_waiter_bit(state) ((state & GATE_WAITERS) != 0) #define ordered_load_gate(gate) os_atomic_load(&(gate)->gt_data, compiler_acq_rel) #define ordered_store_gate(gate, value) os_atomic_store(&(gate)->gt_data, value, compiler_acq_rel) #define GATE_THREAD_MASK (~(uintptr_t)(GATE_ILOCK | GATE_WAITERS)) #define GATE_STATE_TO_THREAD(state) (thread_t)((state) & GATE_THREAD_MASK) #define GATE_STATE_MASKED(state) (uintptr_t)((state) & GATE_THREAD_MASK) #define GATE_THREAD_TO_STATE(thread) ((uintptr_t)(thread)) #define GATE_DESTROYED GATE_STATE_MASKED(0xdeadbeefdeadbeef) #define GATE_EVENT(gate) ((event_t) gate) #define EVENT_TO_GATE(event) ((gate_t *) event) typedef void (*void_func_void)(void); __abortlike static void gate_verify_tag_panic(gate_t *gate) { panic("Gate used is invalid. gate %p data %lx turnstile %p refs %d flags %x ", gate, gate->gt_data, gate->gt_turnstile, gate->gt_refs, gate->gt_flags); } __abortlike static void gate_verify_destroy_panic(gate_t *gate) { panic("Gate used was destroyed. gate %p data %lx turnstile %p refs %d flags %x", gate, gate->gt_data, gate->gt_turnstile, gate->gt_refs, gate->gt_flags); } static void gate_verify(gate_t *gate) { if (gate->gt_type != GATE_TYPE) { gate_verify_tag_panic(gate); } if (GATE_STATE_MASKED(gate->gt_data) == GATE_DESTROYED) { gate_verify_destroy_panic(gate); } assert(gate->gt_refs > 0); } __abortlike static void gate_already_owned_panic(gate_t *gate, thread_t holder) { panic("Trying to close a gate already closed gate %p holder %p current_thread %p", gate, holder, current_thread()); } static kern_return_t gate_try_close(gate_t *gate) { uintptr_t state; thread_t holder; kern_return_t ret; thread_t thread = current_thread(); gate_verify(gate); if (os_atomic_cmpxchg(&gate->gt_data, 0, GATE_THREAD_TO_STATE(thread), acquire)) { return KERN_SUCCESS; } gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); if (holder == NULL) { assert(gate_has_waiter_bit(state) == FALSE); state = GATE_THREAD_TO_STATE(current_thread()); state |= GATE_ILOCK; ordered_store_gate(gate, state); ret = KERN_SUCCESS; } else { if (holder == current_thread()) { gate_already_owned_panic(gate, holder); } ret = KERN_FAILURE; } gate_iunlock(gate); return ret; } static void gate_close(gate_t* gate) { uintptr_t state; thread_t holder; thread_t thread = current_thread(); gate_verify(gate); if (os_atomic_cmpxchg(&gate->gt_data, 0, GATE_THREAD_TO_STATE(thread), acquire)) { return; } gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); if (holder != NULL) { gate_already_owned_panic(gate, holder); } assert(gate_has_waiter_bit(state) == FALSE); state = GATE_THREAD_TO_STATE(thread); state |= GATE_ILOCK; ordered_store_gate(gate, state); gate_iunlock(gate); } static void gate_open_turnstile(gate_t *gate) { struct turnstile *ts = NULL; ts = turnstile_prepare((uintptr_t)gate, &gate->gt_turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX); waitq_wakeup64_all(&ts->ts_waitq, CAST_EVENT64_T(GATE_EVENT(gate)), THREAD_AWAKENED, WAITQ_UPDATE_INHERITOR); turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD); turnstile_complete((uintptr_t)gate, &gate->gt_turnstile, NULL, TURNSTILE_KERNEL_MUTEX); /* * We can do the cleanup while holding the interlock. * It is ok because: * 1. current_thread is the previous inheritor and it is running * 2. new inheritor is NULL. * => No chain of turnstiles needs to be updated. */ turnstile_cleanup(); } __abortlike static void gate_not_owned_panic(gate_t *gate, thread_t holder, bool open) { if (open) { panic("Trying to open a gate %p owned by %p from current_thread %p", gate, holder, current_thread()); } else { panic("Trying to handoff a gate %p owned by %p from current_thread %p", gate, holder, current_thread()); } } static void gate_open(gate_t *gate) { uintptr_t state; thread_t holder; bool waiters; thread_t thread = current_thread(); gate_verify(gate); if (os_atomic_cmpxchg(&gate->gt_data, GATE_THREAD_TO_STATE(thread), 0, release)) { return; } gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); waiters = gate_has_waiter_bit(state); if (holder != thread) { gate_not_owned_panic(gate, holder, true); } if (waiters) { gate_open_turnstile(gate); } state = GATE_ILOCK; ordered_store_gate(gate, state); gate_iunlock(gate); } static kern_return_t gate_handoff_turnstile(gate_t *gate, int flags, thread_t *thread_woken_up, bool *waiters) { struct turnstile *ts = NULL; kern_return_t ret = KERN_FAILURE; thread_t hp_thread; ts = turnstile_prepare((uintptr_t)gate, &gate->gt_turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX); /* * Wake up the higest priority thread waiting on the gate */ hp_thread = waitq_wakeup64_identify(&ts->ts_waitq, CAST_EVENT64_T(GATE_EVENT(gate)), THREAD_AWAKENED, WAITQ_UPDATE_INHERITOR); if (hp_thread != NULL) { /* * In this case waitq_wakeup64_identify has called turnstile_update_inheritor for us */ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD); *thread_woken_up = hp_thread; *waiters = turnstile_has_waiters(ts); /* * Note: hp_thread is the new holder and the new inheritor. * In case there are no more waiters, it doesn't need to be the inheritor * and it shouldn't be it by the time it finishes the wait, so that its next open or * handoff can go through the fast path. * We could set the inheritor to NULL here, or the new holder itself can set it * on its way back from the sleep. In the latter case there are more chanses that * new waiters will come by, avoiding to do the opearation at all. */ ret = KERN_SUCCESS; } else { /* * waiters can have been woken up by an interrupt and still not * have updated gate->waiters, so we couldn't find them on the waitq. * Update the inheritor to NULL here, so that the current thread can return to userspace * indipendently from when the interrupted waiters will finish the wait. */ if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) { turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE); turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD); } // there are no waiters. ret = KERN_NOT_WAITING; } turnstile_complete((uintptr_t)gate, &gate->gt_turnstile, NULL, TURNSTILE_KERNEL_MUTEX); /* * We can do the cleanup while holding the interlock. * It is ok because: * 1. current_thread is the previous inheritor and it is running * 2. new inheritor is NULL or it is a just wokenup thread that will race acquiring the lock * of the gate before trying to sleep. * => No chain of turnstiles needs to be updated. */ turnstile_cleanup(); return ret; } static kern_return_t gate_handoff(gate_t *gate, int flags) { kern_return_t ret; thread_t new_holder = NULL; uintptr_t state; thread_t holder; bool waiters; thread_t thread = current_thread(); assert(flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS || flags == GATE_HANDOFF_DEFAULT); gate_verify(gate); if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) { if (os_atomic_cmpxchg(&gate->gt_data, GATE_THREAD_TO_STATE(thread), 0, release)) { //gate opened but there were no waiters, so return KERN_NOT_WAITING. return KERN_NOT_WAITING; } } gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); waiters = gate_has_waiter_bit(state); if (holder != current_thread()) { gate_not_owned_panic(gate, holder, false); } if (waiters) { ret = gate_handoff_turnstile(gate, flags, &new_holder, &waiters); if (ret == KERN_SUCCESS) { state = GATE_THREAD_TO_STATE(new_holder); if (waiters) { state |= GATE_WAITERS; } } else { if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) { state = 0; } } } else { if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) { state = 0; } ret = KERN_NOT_WAITING; } state |= GATE_ILOCK; ordered_store_gate(gate, state); gate_iunlock(gate); if (new_holder) { thread_deallocate(new_holder); } return ret; } static void_func_void gate_steal_turnstile(gate_t *gate, thread_t new_inheritor) { struct turnstile *ts = NULL; ts = turnstile_prepare((uintptr_t)gate, &gate->gt_turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX); turnstile_update_inheritor(ts, new_inheritor, (TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_THREAD)); turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD); turnstile_complete((uintptr_t)gate, &gate->gt_turnstile, NULL, TURNSTILE_KERNEL_MUTEX); /* * turnstile_cleanup might need to update the chain of the old holder. * This operation should happen without the turnstile interlock held. */ return turnstile_cleanup; } __abortlike static void gate_not_closed_panic(gate_t *gate, bool wait) { if (wait) { panic("Trying to wait on a not closed gate %p from current_thread %p", gate, current_thread()); } else { panic("Trying to steal a not closed gate %p from current_thread %p", gate, current_thread()); } } static void gate_steal(gate_t *gate) { uintptr_t state; thread_t holder; thread_t thread = current_thread(); bool waiters; void_func_void func_after_interlock_unlock; gate_verify(gate); gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); waiters = gate_has_waiter_bit(state); if (holder == NULL) { gate_not_closed_panic(gate, false); } state = GATE_THREAD_TO_STATE(thread) | GATE_ILOCK; if (waiters) { state |= GATE_WAITERS; ordered_store_gate(gate, state); func_after_interlock_unlock = gate_steal_turnstile(gate, thread); gate_iunlock(gate); func_after_interlock_unlock(); } else { ordered_store_gate(gate, state); gate_iunlock(gate); } } static void_func_void gate_wait_turnstile(gate_t *gate, wait_interrupt_t interruptible, uint64_t deadline, thread_t holder, wait_result_t* wait, bool* waiters) { struct turnstile *ts; uintptr_t state; ts = turnstile_prepare((uintptr_t)gate, &gate->gt_turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX); turnstile_update_inheritor(ts, holder, (TURNSTILE_DELAYED_UPDATE | TURNSTILE_INHERITOR_THREAD)); waitq_assert_wait64(&ts->ts_waitq, CAST_EVENT64_T(GATE_EVENT(gate)), interruptible, deadline); gate_iunlock(gate); turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); *wait = thread_block(THREAD_CONTINUE_NULL); gate_ilock(gate); *waiters = turnstile_has_waiters(ts); if (!*waiters) { /* * We want to enable the fast path as soon as we see that there are no more waiters. * On the fast path the holder will not do any turnstile operations. * Set the inheritor as NULL here. * * NOTE: if it was an open operation that woke this thread up, the inheritor has * already been set to NULL. */ state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); if (holder && ((*wait != THREAD_AWAKENED) || // thread interrupted or timedout holder == current_thread())) { // thread was woken up and it is the new holder turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE); turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); } } turnstile_complete((uintptr_t)gate, &gate->gt_turnstile, NULL, TURNSTILE_KERNEL_MUTEX); /* * turnstile_cleanup might need to update the chain of the old holder. * This operation should happen without the turnstile primitive interlock held. */ return turnstile_cleanup; } static void gate_free_internal(gate_t *gate) { zfree(KT_GATE, gate); } __abortlike static void gate_too_many_refs_panic(gate_t *gate) { panic("Too many refs taken on gate. gate %p data %lx turnstile %p refs %d flags %x", gate, gate->gt_data, gate->gt_turnstile, gate->gt_refs, gate->gt_flags); } static gate_wait_result_t gate_wait(gate_t* gate, wait_interrupt_t interruptible, uint64_t deadline, void (^primitive_unlock)(void), void (^primitive_lock)(void)) { gate_wait_result_t ret; void_func_void func_after_interlock_unlock; wait_result_t wait_result; uintptr_t state; thread_t holder; bool waiters; gate_verify(gate); gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); if (holder == NULL) { gate_not_closed_panic(gate, true); } /* * Get a ref on the gate so it will not * be freed while we are coming back from the sleep. */ if (gate->gt_refs == UINT16_MAX) { gate_too_many_refs_panic(gate); } gate->gt_refs++; state |= GATE_WAITERS; ordered_store_gate(gate, state); /* * Release the primitive lock before any * turnstile operation. Turnstile * does not support a blocking primitive as * interlock. * * In this way, concurrent threads will be * able to acquire the primitive lock * but still will wait for me through the * gate interlock. */ primitive_unlock(); func_after_interlock_unlock = gate_wait_turnstile( gate, interruptible, deadline, holder, &wait_result, &waiters); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); switch (wait_result) { case THREAD_INTERRUPTED: case THREAD_TIMED_OUT: assert(holder != current_thread()); if (waiters) { state |= GATE_WAITERS; } else { state &= ~GATE_WAITERS; } ordered_store_gate(gate, state); if (wait_result == THREAD_INTERRUPTED) { ret = GATE_INTERRUPTED; } else { ret = GATE_TIMED_OUT; } break; default: /* * Note it is possible that even if the gate was handed off to * me, someone called gate_steal() before I woke up. * * As well as it is possible that the gate was opened, but someone * closed it while I was waking up. * * In both cases we return GATE_OPENED, as the gate was opened to me * at one point, it is the caller responsibility to check again if * the gate is open. */ if (holder == current_thread()) { ret = GATE_HANDOFF; } else { ret = GATE_OPENED; } break; } assert(gate->gt_refs > 0); uint32_t ref = --gate->gt_refs; bool to_free = gate->gt_alloc; gate_iunlock(gate); if (GATE_STATE_MASKED(state) == GATE_DESTROYED) { if (to_free == true) { assert(!waiters); if (ref == 0) { gate_free_internal(gate); } ret = GATE_OPENED; } else { gate_verify_destroy_panic(gate); } } /* * turnstile func that needs to be executed without * holding the primitive interlock */ func_after_interlock_unlock(); primitive_lock(); return ret; } static void gate_assert(gate_t *gate, int flags) { uintptr_t state; thread_t holder; gate_verify(gate); gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); switch (flags) { case GATE_ASSERT_CLOSED: assert(holder != NULL); break; case GATE_ASSERT_OPEN: assert(holder == NULL); break; case GATE_ASSERT_HELD: assert(holder == current_thread()); break; default: panic("invalid %s flag %d", __func__, flags); } gate_iunlock(gate); } enum { GT_INIT_DEFAULT = 0, GT_INIT_ALLOC }; static void gate_init(gate_t *gate, uint type) { bzero(gate, sizeof(gate_t)); gate->gt_data = 0; gate->gt_turnstile = NULL; gate->gt_refs = 1; switch (type) { case GT_INIT_ALLOC: gate->gt_alloc = 1; break; default: gate->gt_alloc = 0; break; } gate->gt_type = GATE_TYPE; gate->gt_flags_pad = 0; } static gate_t* gate_alloc_init(void) { gate_t *gate; gate = zalloc_flags(KT_GATE, Z_WAITOK | Z_NOFAIL); gate_init(gate, GT_INIT_ALLOC); return gate; } __abortlike static void gate_destroy_owned_panic(gate_t *gate, thread_t holder) { panic("Trying to destroy a gate owned by %p. Gate %p", holder, gate); } __abortlike static void gate_destroy_waiter_panic(gate_t *gate) { panic("Trying to destroy a gate with waiters. Gate %p data %lx turnstile %p", gate, gate->gt_data, gate->gt_turnstile); } static uint16_t gate_destroy_internal(gate_t *gate) { uintptr_t state; thread_t holder; uint16_t ref; gate_ilock(gate); state = ordered_load_gate(gate); holder = GATE_STATE_TO_THREAD(state); /* * The gate must be open * and all the threads must * have been woken up by this time */ if (holder != NULL) { gate_destroy_owned_panic(gate, holder); } if (gate_has_waiter_bit(state)) { gate_destroy_waiter_panic(gate); } assert(gate->gt_refs > 0); ref = --gate->gt_refs; /* * Mark the gate as destroyed. * The interlock bit still need * to be available to let the * last wokenup threads to clear * the wait. */ state = GATE_DESTROYED; state |= GATE_ILOCK; ordered_store_gate(gate, state); gate_iunlock(gate); return ref; } __abortlike static void gate_destroy_panic(gate_t *gate) { panic("Trying to destroy a gate that was allocated by gate_alloc_init(). gate_free() should be used instead, gate %p thread %p", gate, current_thread()); } static void gate_destroy(gate_t *gate) { gate_verify(gate); if (gate->gt_alloc == 1) { gate_destroy_panic(gate); } gate_destroy_internal(gate); } __abortlike static void gate_free_panic(gate_t *gate) { panic("Trying to free a gate that was not allocated by gate_alloc_init(), gate %p thread %p", gate, current_thread()); } static void gate_free(gate_t *gate) { uint16_t ref; gate_verify(gate); if (gate->gt_alloc == 0) { gate_free_panic(gate); } ref = gate_destroy_internal(gate); /* * Some of the threads waiting on the gate * might still need to run after being woken up. * They will access the gate to cleanup the * state, so we cannot free it. * The last waiter will free the gate in this case. */ if (ref == 0) { gate_free_internal(gate); } } /* * Name: lck_rw_gate_init * * Description: initializes a variable declared with decl_lck_rw_gate_data. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. */ void lck_rw_gate_init(lck_rw_t *lock, gate_t *gate) { (void) lock; gate_init(gate, GT_INIT_DEFAULT); } /* * Name: lck_rw_gate_alloc_init * * Description: allocates and initializes a gate_t. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * * Returns: * gate_t allocated. */ gate_t* lck_rw_gate_alloc_init(lck_rw_t *lock) { (void) lock; return gate_alloc_init(); } /* * Name: lck_rw_gate_destroy * * Description: destroys a variable previously initialized * with lck_rw_gate_init(). * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. */ void lck_rw_gate_destroy(lck_rw_t *lock, gate_t *gate) { (void) lock; gate_destroy(gate); } /* * Name: lck_rw_gate_free * * Description: destroys and tries to free a gate previously allocated * with lck_rw_gate_alloc_init(). * The gate free might be delegated to the last thread returning * from the gate_wait(). * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate obtained with lck_rw_gate_alloc_init(). */ void lck_rw_gate_free(lck_rw_t *lock, gate_t *gate) { (void) lock; gate_free(gate); } /* * Name: lck_rw_gate_try_close * * Description: Tries to close the gate. * In case of success the current thread will be set as * the holder of the gate. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * * Returns: * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder * of the gate. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped * between the calls to lck_rw_gate_try_close() and lck_rw_gate_wait(). * * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate. * lck_rw_gate_wait() should be called instead if the intent is to unconditionally wait on this gate. * The calls to lck_rw_gate_try_close() and lck_rw_gate_wait() should * be done without dropping the lock that is protecting the gate in between. */ int lck_rw_gate_try_close(__assert_only lck_rw_t *lock, gate_t *gate) { LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); return gate_try_close(gate); } /* * Name: lck_rw_gate_close * * Description: Closes the gate. The current thread will be set as * the holder of the gate. Will panic if the gate is already closed. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be open. * */ void lck_rw_gate_close(__assert_only lck_rw_t *lock, gate_t *gate) { LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); return gate_close(gate); } /* * Name: lck_rw_gate_open * * Description: Opens the gate and wakes up possible waiters. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * */ void lck_rw_gate_open(__assert_only lck_rw_t *lock, gate_t *gate) { LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); gate_open(gate); } /* * Name: lck_rw_gate_handoff * * Description: Tries to transfer the ownership of the gate. The waiter with highest sched * priority will be selected as the new holder of the gate, and woken up, * with the gate remaining in the closed state throughout. * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING * will be returned. * GATE_HANDOFF_OPEN_IF_NO_WAITERS flag can be used to specify if the gate should be opened in * case no waiters were found. * * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * Arg3: flags - GATE_HANDOFF_DEFAULT or GATE_HANDOFF_OPEN_IF_NO_WAITERS * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * * Returns: * KERN_SUCCESS in case one of the waiters became the new holder. * KERN_NOT_WAITING in case there were no waiters. * */ kern_return_t lck_rw_gate_handoff(__assert_only lck_rw_t *lock, gate_t *gate, gate_handoff_flags_t flags) { LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); return gate_handoff(gate, flags); } /* * Name: lck_rw_gate_steal * * Description: Set the current ownership of the gate. It sets the current thread as the * new holder of the gate. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * NOTE: the previous holder should not call lck_rw_gate_open() or lck_rw_gate_handoff() * anymore. * * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be closed and the current thread must not already be the holder. * */ void lck_rw_gate_steal(__assert_only lck_rw_t *lock, gate_t *gate) { LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); gate_steal(gate); } /* * Name: lck_rw_gate_wait * * Description: Waits for the current thread to become the holder of the gate or for the * gate to become open. An interruptible mode and deadline can be specified * to return earlier from the wait. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE, LCK_SLEEP_UNLOCK. * Arg3: interruptible flag for wait. * Arg4: deadline * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The gate must be closed. * * Returns: Reason why the thread was woken up. * GATE_HANDOFF - the current thread was handed off the ownership of the gate. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on. * to wake up possible waiters on the gate before returning to userspace. * GATE_OPENED - the gate was opened by the holder. * GATE_TIMED_OUT - the thread was woken up by a timeout. * GATE_INTERRUPTED - the thread was interrupted while sleeping. */ gate_wait_result_t lck_rw_gate_wait(lck_rw_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline) { __block lck_rw_type_t lck_rw_type = LCK_RW_TYPE_EXCLUSIVE; LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return gate_wait(gate, interruptible, deadline, ^{lck_rw_type = lck_rw_done(lock);}, ^{;}); } else if (!(lck_sleep_action & (LCK_SLEEP_SHARED | LCK_SLEEP_EXCLUSIVE))) { return gate_wait(gate, interruptible, deadline, ^{lck_rw_type = lck_rw_done(lock);}, ^{lck_rw_lock(lock, lck_rw_type);}); } else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) { return gate_wait(gate, interruptible, deadline, ^{lck_rw_type = lck_rw_done(lock);}, ^{lck_rw_lock_exclusive(lock);}); } else { return gate_wait(gate, interruptible, deadline, ^{lck_rw_type = lck_rw_done(lock);}, ^{lck_rw_lock_shared(lock);}); } } /* * Name: lck_rw_gate_assert * * Description: asserts that the gate is in the specified state. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * Arg3: flags to specified assert type. * GATE_ASSERT_CLOSED - the gate is currently closed * GATE_ASSERT_OPEN - the gate is currently opened * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder */ void lck_rw_gate_assert(__assert_only lck_rw_t *lock, gate_t *gate, gate_assert_flags_t flags) { LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD); gate_assert(gate, flags); return; } /* * Name: lck_mtx_gate_init * * Description: initializes a variable declared with decl_lck_mtx_gate_data. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. */ void lck_mtx_gate_init(lck_mtx_t *lock, gate_t *gate) { (void) lock; gate_init(gate, GT_INIT_DEFAULT); } /* * Name: lck_mtx_gate_alloc_init * * Description: allocates and initializes a gate_t. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * * Returns: * gate_t allocated. */ gate_t* lck_mtx_gate_alloc_init(lck_mtx_t *lock) { (void) lock; return gate_alloc_init(); } /* * Name: lck_mtx_gate_destroy * * Description: destroys a variable previously initialized * with lck_mtx_gate_init(). * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. */ void lck_mtx_gate_destroy(lck_mtx_t *lock, gate_t *gate) { (void) lock; gate_destroy(gate); } /* * Name: lck_mtx_gate_free * * Description: destroys and tries to free a gate previously allocated * with lck_mtx_gate_alloc_init(). * The gate free might be delegated to the last thread returning * from the gate_wait(). * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate obtained with lck_rw_gate_alloc_init(). */ void lck_mtx_gate_free(lck_mtx_t *lock, gate_t *gate) { (void) lock; gate_free(gate); } /* * Name: lck_mtx_gate_try_close * * Description: Tries to close the gate. * In case of success the current thread will be set as * the holder of the gate. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * * Returns: * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder * of the gate. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped * between the calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait(). * * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate. * lck_mtx_gate_wait() should be called instead if the intent is to unconditionally wait on this gate. * The calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait() should * be done without dropping the lock that is protecting the gate in between. */ int lck_mtx_gate_try_close(__assert_only lck_mtx_t *lock, gate_t *gate) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); return gate_try_close(gate); } /* * Name: lck_mtx_gate_close * * Description: Closes the gate. The current thread will be set as * the holder of the gate. Will panic if the gate is already closed. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be open. * */ void lck_mtx_gate_close(__assert_only lck_mtx_t *lock, gate_t *gate) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); return gate_close(gate); } /* * Name: lck_mtx_gate_open * * Description: Opens of the gate and wakes up possible waiters. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * */ void lck_mtx_gate_open(__assert_only lck_mtx_t *lock, gate_t *gate) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); gate_open(gate); } /* * Name: lck_mtx_gate_handoff * * Description: Tries to transfer the ownership of the gate. The waiter with highest sched * priority will be selected as the new holder of the gate, and woken up, * with the gate remaining in the closed state throughout. * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING * will be returned. * GATE_HANDOFF_OPEN_IF_NO_WAITERS flag can be used to specify if the gate should be opened in * case no waiters were found. * * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * Arg3: flags - GATE_HANDOFF_DEFAULT or GATE_HANDOFF_OPEN_IF_NO_WAITERS * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * * Returns: * KERN_SUCCESS in case one of the waiters became the new holder. * KERN_NOT_WAITING in case there were no waiters. * */ kern_return_t lck_mtx_gate_handoff(__assert_only lck_mtx_t *lock, gate_t *gate, gate_handoff_flags_t flags) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); return gate_handoff(gate, flags); } /* * Name: lck_mtx_gate_steal * * Description: Steals the ownership of the gate. It sets the current thread as the * new holder of the gate. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * NOTE: the previous holder should not call lck_mtx_gate_open() or lck_mtx_gate_handoff() * anymore. * * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be closed and the current thread must not already be the holder. * */ void lck_mtx_gate_steal(__assert_only lck_mtx_t *lock, gate_t *gate) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); gate_steal(gate); } /* * Name: lck_mtx_gate_wait * * Description: Waits for the current thread to become the holder of the gate or for the * gate to become open. An interruptible mode and deadline can be specified * to return earlier from the wait. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS. * Arg3: interruptible flag for wait. * Arg4: deadline * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The gate must be closed. * * Returns: Reason why the thread was woken up. * GATE_HANDOFF - the current thread was handed off the ownership of the gate. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * GATE_OPENED - the gate was opened by the holder. * GATE_TIMED_OUT - the thread was woken up by a timeout. * GATE_INTERRUPTED - the thread was interrupted while sleeping. */ gate_wait_result_t lck_mtx_gate_wait(lck_mtx_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); if (lck_sleep_action & LCK_SLEEP_UNLOCK) { return gate_wait(gate, interruptible, deadline, ^{lck_mtx_unlock(lock);}, ^{;}); } else if (lck_sleep_action & LCK_SLEEP_SPIN) { return gate_wait(gate, interruptible, deadline, ^{lck_mtx_unlock(lock);}, ^{lck_mtx_lock_spin(lock);}); } else if (lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS) { return gate_wait(gate, interruptible, deadline, ^{lck_mtx_unlock(lock);}, ^{lck_mtx_lock_spin_always(lock);}); } else { return gate_wait(gate, interruptible, deadline, ^{lck_mtx_unlock(lock);}, ^{lck_mtx_lock(lock);}); } } /* * Name: lck_mtx_gate_assert * * Description: asserts that the gate is in the specified state. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * Arg3: flags to specified assert type. * GATE_ASSERT_CLOSED - the gate is currently closed * GATE_ASSERT_OPEN - the gate is currently opened * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder */ void lck_mtx_gate_assert(__assert_only lck_mtx_t *lock, gate_t *gate, gate_assert_flags_t flags) { LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED); gate_assert(gate, flags); } #pragma mark - LCK_*_DECLARE support __startup_func void lck_spin_startup_init(struct lck_spin_startup_spec *sp) { lck_spin_init(sp->lck, sp->lck_grp, sp->lck_attr); } __startup_func void lck_mtx_startup_init(struct lck_mtx_startup_spec *sp) { lck_mtx_init(sp->lck, sp->lck_grp, sp->lck_attr); } __startup_func void lck_rw_startup_init(struct lck_rw_startup_spec *sp) { lck_rw_init(sp->lck, sp->lck_grp, sp->lck_attr); } __startup_func void usimple_lock_startup_init(struct usimple_lock_startup_spec *sp) { simple_lock_init(sp->lck, sp->lck_init_arg); } __startup_func void lck_ticket_startup_init(struct lck_ticket_startup_spec *sp) { lck_ticket_init(sp->lck, sp->lck_grp); }