/* * Copyright (c) 2007-2021 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * Mach Operating System Copyright (c) 1991,1990,1989,1988,1987 Carnegie * Mellon University All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright notice * and this permission notice appear in all copies of the software, * derivative works or modified versions, and any portions thereof, and that * both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science Carnegie Mellon University Pittsburgh PA * 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon the * rights to redistribute these changes. */ /* * File: kern/lock.c * Author: Avadis Tevanian, Jr., Michael Wayne Young * Date: 1985 * * Locking primitives implementation */ #define LOCK_PRIVATE 1 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define ANY_LOCK_DEBUG (USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG) // Panic in tests that check lock usage correctness // These are undesirable when in a panic or a debugger is runnning. #define LOCK_CORRECTNESS_PANIC() (kernel_debugger_entry_count == 0) /* Forwards */ extern unsigned int not_in_kdp; MACHINE_TIMEOUT(lock_panic_timeout, "lock-panic", 0xc00000 /* 12.5 m ticks ~= 524ms with 24MHz OSC */, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL); #define NOINLINE __attribute__((noinline)) #define interrupts_disabled(mask) (mask & DAIF_IRQF) KALLOC_TYPE_DEFINE(KT_LCK_SPIN, lck_spin_t, KT_PRIV_ACCT); #pragma GCC visibility push(hidden) /* * atomic exchange API is a low level abstraction of the operations * to atomically read, modify, and write a pointer. This abstraction works * for both Intel and ARMv8.1 compare and exchange atomic instructions as * well as the ARM exclusive instructions. * * atomic_exchange_begin() - begin exchange and retrieve current value * atomic_exchange_complete() - conclude an exchange * atomic_exchange_abort() - cancel an exchange started with atomic_exchange_begin() */ uint32_t load_exclusive32(uint32_t *target, enum memory_order ord) { uint32_t value; if (_os_atomic_mo_has_acquire(ord)) { value = __builtin_arm_ldaex(target); // ldaxr } else { value = __builtin_arm_ldrex(target); // ldxr } return value; } boolean_t store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord) { boolean_t err; if (_os_atomic_mo_has_release(ord)) { err = __builtin_arm_stlex(value, target); // stlxr } else { err = __builtin_arm_strex(value, target); // stxr } return !err; } uint32_t atomic_exchange_begin32(uint32_t *target, uint32_t *previous, enum memory_order ord) { uint32_t val; #if !OS_ATOMIC_USE_LLSC ord = memory_order_relaxed; #endif val = load_exclusive32(target, ord); *previous = val; return val; } boolean_t atomic_exchange_complete32(uint32_t *target, uint32_t previous, uint32_t newval, enum memory_order ord) { #if !OS_ATOMIC_USE_LLSC return __c11_atomic_compare_exchange_strong((_Atomic uint32_t *)target, &previous, newval, ord, memory_order_relaxed); #else (void)previous; // Previous not needed, monitor is held return store_exclusive32(target, newval, ord); #endif } void atomic_exchange_abort(void) { os_atomic_clear_exclusive(); } boolean_t atomic_test_and_set32(uint32_t *target, uint32_t test_mask, uint32_t set_mask, enum memory_order ord, boolean_t wait) { uint32_t value, prev; for (;;) { value = atomic_exchange_begin32(target, &prev, ord); if (value & test_mask) { if (wait) { wait_for_event(); // Wait with monitor held } else { atomic_exchange_abort(); // Clear exclusive monitor } return FALSE; } value |= set_mask; if (atomic_exchange_complete32(target, prev, value, ord)) { return TRUE; } } } #pragma GCC visibility pop #if CONFIG_PV_TICKET __startup_func void lck_init_pv(void) { uint32_t pvtck = 1; PE_parse_boot_argn("pvticket", &pvtck, sizeof(pvtck)); if (pvtck == 0) { return; } has_lock_pv = hvg_is_hcall_available(HVG_HCALL_VCPU_WFK) && hvg_is_hcall_available(HVG_HCALL_VCPU_KICK); } STARTUP(LOCKS, STARTUP_RANK_FIRST, lck_init_pv); #endif #pragma mark lck_spin_t #if LCK_SPIN_IS_TICKET_LOCK lck_spin_t * lck_spin_alloc_init(lck_grp_t *grp, lck_attr_t *attr) { lck_spin_t *lck; lck = zalloc(KT_LCK_SPIN); lck_spin_init(lck, grp, attr); return lck; } void lck_spin_free(lck_spin_t *lck, lck_grp_t *grp) { lck_spin_destroy(lck, grp); zfree(KT_LCK_SPIN, lck); } void lck_spin_init(lck_spin_t *lck, lck_grp_t *grp, __unused lck_attr_t *attr) { lck_ticket_init(lck, grp); } /* * arm_usimple_lock is a lck_spin_t without a group or attributes */ MARK_AS_HIBERNATE_TEXT void inline arm_usimple_lock_init(simple_lock_t lck, __unused unsigned short initial_value) { lck_ticket_init((lck_ticket_t *)lck, LCK_GRP_NULL); } void lck_spin_assert(const lck_spin_t *lock, unsigned int type) { if (type == LCK_ASSERT_OWNED) { lck_ticket_assert_owned(lock); } else if (type == LCK_ASSERT_NOTOWNED) { lck_ticket_assert_not_owned(lock); } else { panic("lck_spin_assert(): invalid arg (%u)", type); } } void lck_spin_lock(lck_spin_t *lock) { lck_ticket_lock(lock, LCK_GRP_NULL); } void lck_spin_lock_nopreempt(lck_spin_t *lock) { lck_ticket_lock_nopreempt(lock, LCK_GRP_NULL); } int lck_spin_try_lock(lck_spin_t *lock) { return lck_ticket_lock_try(lock, LCK_GRP_NULL); } int lck_spin_try_lock_nopreempt(lck_spin_t *lock) { return lck_ticket_lock_try_nopreempt(lock, LCK_GRP_NULL); } void lck_spin_unlock(lck_spin_t *lock) { lck_ticket_unlock(lock); } void lck_spin_destroy(lck_spin_t *lck, lck_grp_t *grp) { lck_ticket_destroy(lck, grp); } /* * those really should be in an alias file instead, * but you can't make that conditional. * * it will be good enough for perf evals for now * * we also can't make aliases for symbols that * are in alias files like lck_spin_init and friends, * so this suffers double jump penalties for kexts * (LTO does the right thing for XNU). */ #define make_alias(a, b) asm(".globl _" #a "\n" ".set _" #a ", _" #b "\n") make_alias(lck_spin_lock_grp, lck_ticket_lock); make_alias(lck_spin_lock_nopreempt_grp, lck_ticket_lock_nopreempt); make_alias(lck_spin_try_lock_grp, lck_ticket_lock_try); make_alias(lck_spin_try_lock_nopreempt_grp, lck_ticket_lock_try_nopreempt); make_alias(lck_spin_unlock_nopreempt, lck_ticket_unlock_nopreempt); make_alias(kdp_lck_spin_is_acquired, kdp_lck_ticket_is_acquired); #undef make_alias #else /* !LCK_SPIN_IS_TICKET_LOCK */ #if DEVELOPMENT || DEBUG __abortlike static void __lck_spin_invalid_panic(lck_spin_t *lck) { const char *how = "Invalid"; if (lck->type == LCK_SPIN_TYPE_DESTROYED || lck->lck_spin_data == LCK_SPIN_TAG_DESTROYED) { how = "Destroyed"; } panic("%s spinlock %p: <0x%016lx 0x%16lx>", how, lck, lck->lck_spin_data, lck->type); } static inline void lck_spin_verify(lck_spin_t *lck) { if (lck->type != LCK_SPIN_TYPE || lck->lck_spin_data == LCK_SPIN_TAG_DESTROYED) { __lck_spin_invalid_panic(lck); } } #else /* DEVELOPMENT || DEBUG */ #define lck_spin_verify(lck) ((void)0) #endif /* DEVELOPMENT || DEBUG */ lck_spin_t * lck_spin_alloc_init(lck_grp_t *grp, lck_attr_t *attr) { lck_spin_t *lck; lck = zalloc(KT_LCK_SPIN); lck_spin_init(lck, grp, attr); return lck; } void lck_spin_free(lck_spin_t *lck, lck_grp_t *grp) { lck_spin_destroy(lck, grp); zfree(KT_LCK_SPIN, lck); } void lck_spin_init(lck_spin_t *lck, lck_grp_t *grp, __unused lck_attr_t *attr) { lck->type = LCK_SPIN_TYPE; hw_lock_init(&lck->hwlock); if (grp) { lck_grp_reference(grp, &grp->lck_grp_spincnt); } } /* * arm_usimple_lock is a lck_spin_t without a group or attributes */ MARK_AS_HIBERNATE_TEXT void inline arm_usimple_lock_init(simple_lock_t lck, __unused unsigned short initial_value) { lck->type = LCK_SPIN_TYPE; hw_lock_init(&lck->hwlock); } void lck_spin_assert(const lck_spin_t *lock, unsigned int type) { thread_t thread, holder; if (lock->type != LCK_SPIN_TYPE) { panic("Invalid spinlock %p", lock); } holder = HW_LOCK_STATE_TO_THREAD(lock->lck_spin_data); thread = current_thread(); if (type == LCK_ASSERT_OWNED) { if (holder == 0) { panic("Lock not owned %p = %p", lock, holder); } if (holder != thread) { panic("Lock not owned by current thread %p = %p", lock, holder); } } else if (type == LCK_ASSERT_NOTOWNED) { if (holder != THREAD_NULL && holder == thread) { panic("Lock owned by current thread %p = %p", lock, holder); } } else { panic("lck_spin_assert(): invalid arg (%u)", type); } } void lck_spin_lock(lck_spin_t *lock) { lck_spin_verify(lock); hw_lock_lock(&lock->hwlock, LCK_GRP_NULL); } void lck_spin_lock_grp(lck_spin_t *lock, lck_grp_t *grp) { #pragma unused(grp) lck_spin_verify(lock); hw_lock_lock(&lock->hwlock, grp); } void lck_spin_lock_nopreempt(lck_spin_t *lock) { lck_spin_verify(lock); hw_lock_lock_nopreempt(&lock->hwlock, LCK_GRP_NULL); } void lck_spin_lock_nopreempt_grp(lck_spin_t *lock, lck_grp_t *grp) { #pragma unused(grp) lck_spin_verify(lock); hw_lock_lock_nopreempt(&lock->hwlock, grp); } int lck_spin_try_lock(lck_spin_t *lock) { lck_spin_verify(lock); return hw_lock_try(&lock->hwlock, LCK_GRP_NULL); } int lck_spin_try_lock_grp(lck_spin_t *lock, lck_grp_t *grp) { #pragma unused(grp) lck_spin_verify(lock); return hw_lock_try(&lock->hwlock, grp); } int lck_spin_try_lock_nopreempt(lck_spin_t *lock) { lck_spin_verify(lock); return hw_lock_try_nopreempt(&lock->hwlock, LCK_GRP_NULL); } int lck_spin_try_lock_nopreempt_grp(lck_spin_t *lock, lck_grp_t *grp) { #pragma unused(grp) lck_spin_verify(lock); return hw_lock_try_nopreempt(&lock->hwlock, grp); } void lck_spin_unlock(lck_spin_t *lock) { lck_spin_verify(lock); hw_lock_unlock(&lock->hwlock); } void lck_spin_unlock_nopreempt(lck_spin_t *lock) { lck_spin_verify(lock); hw_lock_unlock_nopreempt(&lock->hwlock); } void lck_spin_destroy(lck_spin_t *lck, lck_grp_t *grp) { lck_spin_verify(lck); *lck = (lck_spin_t){ .lck_spin_data = LCK_SPIN_TAG_DESTROYED, .type = LCK_SPIN_TYPE_DESTROYED, }; if (grp) { lck_grp_deallocate(grp, &grp->lck_grp_spincnt); } } /* * Routine: kdp_lck_spin_is_acquired * NOT SAFE: To be used only by kernel debugger to avoid deadlock. */ boolean_t kdp_lck_spin_is_acquired(lck_spin_t *lck) { if (not_in_kdp) { panic("panic: spinlock acquired check done outside of kernel debugger"); } return ((lck->lck_spin_data & ~LCK_SPIN_TAG_DESTROYED) != 0) ? TRUE:FALSE; } #endif /* !LCK_SPIN_IS_TICKET_LOCK */ /* * Initialize a usimple_lock. * * No change in preemption state. */ void usimple_lock_init( usimple_lock_t l, unsigned short tag) { simple_lock_init((simple_lock_t) l, tag); } /* * Acquire a usimple_lock. * * Returns with preemption disabled. Note * that the hw_lock routines are responsible for * maintaining preemption state. */ void (usimple_lock)( usimple_lock_t l LCK_GRP_ARG(lck_grp_t *grp)) { simple_lock((simple_lock_t) l, LCK_GRP_PROBEARG(grp)); } /* * Release a usimple_lock. * * Returns with preemption enabled. Note * that the hw_lock routines are responsible for * maintaining preemption state. */ void (usimple_unlock)( usimple_lock_t l) { simple_unlock((simple_lock_t)l); } /* * Conditionally acquire a usimple_lock. * * On success, returns with preemption disabled. * On failure, returns with preemption in the same state * as when first invoked. Note that the hw_lock routines * are responsible for maintaining preemption state. * * XXX No stats are gathered on a miss; I preserved this * behavior from the original assembly-language code, but * doesn't it make sense to log misses? XXX */ unsigned int (usimple_lock_try)( usimple_lock_t l LCK_GRP_ARG(lck_grp_t *grp)) { return simple_lock_try((simple_lock_t) l, grp); }