/* * Copyright (c) 2000-2021 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ * */ /* * File: kern/sync_sema.c * Author: Joseph CaraDonna * * Contains RT distributed semaphore synchronization services. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const uint8_t semaphore_event; #define SEMAPHORE_EVENT CAST_EVENT64_T(&semaphore_event) ZONE_DEFINE_ID(ZONE_ID_SEMAPHORE, "semaphores", struct semaphore, ZC_ZFREE_CLEARMEM); os_refgrp_decl(static, sema_refgrp, "semaphore", NULL); /* Forward declarations */ static inline bool semaphore_active(semaphore_t semaphore) { return semaphore->owner != TASK_NULL; } static __inline__ uint64_t semaphore_deadline( unsigned int sec, clock_res_t nsec) { uint64_t abstime; nanotime_to_absolutetime(sec, nsec, &abstime); clock_absolutetime_interval_to_deadline(abstime, &abstime); return abstime; } /* * Routine: semaphore_create * * Creates a semaphore. * The port representing the semaphore is returned as a parameter. */ kern_return_t semaphore_create( task_t task, semaphore_t *new_semaphore, int policy, int value) { semaphore_t s = SEMAPHORE_NULL; *new_semaphore = SEMAPHORE_NULL; if (task == TASK_NULL || value < 0 || (policy & ~SYNC_POLICY_USER_MASK)) { return KERN_INVALID_ARGUMENT; } s = zalloc_id(ZONE_ID_SEMAPHORE, Z_ZERO | Z_WAITOK | Z_NOFAIL); /* * Associate the new semaphore with the task by adding * the new semaphore to the task's semaphore list. */ task_lock(task); /* Check for race with task_terminate */ if (!task->active) { task_unlock(task); zfree_id(ZONE_ID_SEMAPHORE, s); return KERN_INVALID_TASK; } waitq_init(&s->waitq, WQT_QUEUE, policy | SYNC_POLICY_INIT_LOCKED); /* init everything under both the task and semaphore locks */ os_ref_init_raw(&s->ref_count, &sema_refgrp); s->count = value; s->owner = task; enqueue_head(&task->semaphore_list, &s->task_link); task->semaphores_owned++; semaphore_unlock(s); task_unlock(task); *new_semaphore = s; return KERN_SUCCESS; } /* * Routine: semaphore_destroy_internal * * Disassociate a semaphore from its owning task, mark it inactive, * and set any waiting threads running with THREAD_RESTART. * * Conditions: * task is locked * semaphore is owned by the specified task * if semaphore is locked, interrupts are disabled * Returns: * with semaphore unlocked, interrupts enabled */ static void semaphore_destroy_internal( task_t task, semaphore_t semaphore, bool semaphore_locked) { int old_count; /* unlink semaphore from owning task */ assert(semaphore->owner == task); remqueue(&semaphore->task_link); task->semaphores_owned--; spl_t spl_level = 0; if (semaphore_locked) { spl_level = 1; } else { spl_level = splsched(); semaphore_lock(semaphore); } /* * deactivate semaphore under both locks * and then wake up all waiters. */ semaphore->owner = TASK_NULL; old_count = semaphore->count; semaphore->count = 0; if (old_count < 0) { waitq_wakeup64_all_locked(&semaphore->waitq, SEMAPHORE_EVENT, THREAD_RESTART, waitq_flags_splx(spl_level) | WAITQ_UNLOCK); /* waitq/semaphore is unlocked, splx handled */ assert(ml_get_interrupts_enabled()); } else { assert(circle_queue_empty(&semaphore->waitq.waitq_queue)); semaphore_unlock(semaphore); splx(spl_level); assert(ml_get_interrupts_enabled()); } } /* * Routine: semaphore_free * * Free a semaphore that hit a 0 refcount. * * Conditions: * Nothing is locked. */ __attribute__((noinline)) static void semaphore_free( semaphore_t semaphore) { ipc_port_t port; task_t task; /* * Last ref, clean up the port [if any] * associated with the semaphore, destroy * it (if still active) and then free * the semaphore. */ port = semaphore->port; if (IP_VALID(port)) { assert(!port->ip_srights); ipc_kobject_dealloc_port(port, 0, IKOT_SEMAPHORE); } /* * If the semaphore owned by the current task, * we know the current task can't go away, * so we can take locks in the right order. * * Else we try to take locks in the "wrong" order * but if we fail to, we take a task ref and do it "right". */ task = current_task(); if (semaphore->owner == task) { task_lock(task); if (semaphore->owner == task) { semaphore_destroy_internal(task, semaphore, false); } else { assert(semaphore->owner == TASK_NULL); } task_unlock(task); } else { spl_t spl = splsched(); /* semaphore_destroy_internal will always enable, can't nest */ assert(spl); semaphore_lock(semaphore); task = semaphore->owner; if (task == TASK_NULL) { semaphore_unlock(semaphore); splx(spl); } else if (task_lock_try(task)) { semaphore_destroy_internal(task, semaphore, true); /* semaphore unlocked, interrupts enabled */ task_unlock(task); } else { task_reference(task); semaphore_unlock(semaphore); splx(spl); task_lock(task); if (semaphore->owner == task) { semaphore_destroy_internal(task, semaphore, false); } task_unlock(task); task_deallocate(task); } } waitq_deinit(&semaphore->waitq); zfree_id(ZONE_ID_SEMAPHORE, semaphore); } /* * Routine: semaphore_destroy * * Destroys a semaphore and consume the caller's reference on the * semaphore. */ kern_return_t semaphore_destroy( task_t task, semaphore_t semaphore) { if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } if (task == TASK_NULL) { semaphore_dereference(semaphore); return KERN_INVALID_ARGUMENT; } if (semaphore->owner == task) { task_lock(task); if (semaphore->owner == task) { semaphore_destroy_internal(task, semaphore, false); } task_unlock(task); } semaphore_dereference(semaphore); return KERN_SUCCESS; } /* * Routine: semaphore_destroy_all * * Destroy all the semaphores associated with a given task. */ void semaphore_destroy_all( task_t task) { semaphore_t semaphore; task_lock(task); qe_foreach_element_safe(semaphore, &task->semaphore_list, task_link) { semaphore_destroy_internal(task, semaphore, false); } task_unlock(task); } /* * Routine: semaphore_signal_internal * * Signals the semaphore as direct. * Assumptions: * Semaphore is locked. */ static kern_return_t semaphore_signal_internal( semaphore_t semaphore, thread_t thread, int options) { kern_return_t kr; spl_t spl_level = splsched(); semaphore_lock(semaphore); if (!semaphore_active(semaphore)) { semaphore_unlock(semaphore); splx(spl_level); return KERN_TERMINATED; } if (thread != THREAD_NULL) { if (semaphore->count < 0) { kr = waitq_wakeup64_thread_and_unlock( &semaphore->waitq, SEMAPHORE_EVENT, thread, THREAD_AWAKENED); /* waitq/semaphore is unlocked */ splx(spl_level); } else { kr = KERN_NOT_WAITING; semaphore_unlock(semaphore); splx(spl_level); } return kr; } if (options & SEMAPHORE_SIGNAL_ALL) { int old_count = semaphore->count; kr = KERN_NOT_WAITING; if (old_count < 0) { semaphore->count = 0; /* always reset */ kr = waitq_wakeup64_all_locked(&semaphore->waitq, SEMAPHORE_EVENT, THREAD_AWAKENED, WAITQ_UNLOCK | waitq_flags_splx(spl_level)); /* waitq / semaphore is unlocked, splx handled */ } else { if (options & SEMAPHORE_SIGNAL_PREPOST) { semaphore->count++; } kr = KERN_SUCCESS; semaphore_unlock(semaphore); splx(spl_level); } return kr; } if (semaphore->count < 0) { waitq_wakeup_flags_t flags = WAITQ_KEEP_LOCKED; if (options & SEMAPHORE_THREAD_HANDOFF) { flags |= WAITQ_HANDOFF; } kr = waitq_wakeup64_one_locked(&semaphore->waitq, SEMAPHORE_EVENT, THREAD_AWAKENED, flags); if (kr == KERN_SUCCESS) { semaphore_unlock(semaphore); splx(spl_level); return KERN_SUCCESS; } else { semaphore->count = 0; /* all waiters gone */ } } if (options & SEMAPHORE_SIGNAL_PREPOST) { semaphore->count++; } semaphore_unlock(semaphore); splx(spl_level); return KERN_NOT_WAITING; } /* * Routine: semaphore_signal_thread * * If the specified thread is blocked on the semaphore, it is * woken up. If a NULL thread was supplied, then any one * thread is woken up. Otherwise the caller gets KERN_NOT_WAITING * and the semaphore is unchanged. */ kern_return_t semaphore_signal_thread( semaphore_t semaphore, thread_t thread) { if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } return semaphore_signal_internal(semaphore, thread, SEMAPHORE_OPTION_NONE); } /* * Routine: semaphore_signal_thread_trap * * Trap interface to the semaphore_signal_thread function. */ kern_return_t semaphore_signal_thread_trap( struct semaphore_signal_thread_trap_args *args) { mach_port_name_t sema_name = args->signal_name; mach_port_name_t thread_name = args->thread_name; semaphore_t semaphore; thread_t thread; kern_return_t kr; /* * MACH_PORT_NULL is not an error. It means that we want to * select any one thread that is already waiting, but not to * pre-post the semaphore. */ if (thread_name != MACH_PORT_NULL) { thread = port_name_to_thread(thread_name, PORT_INTRANS_OPTIONS_NONE); if (thread == THREAD_NULL) { return KERN_INVALID_ARGUMENT; } } else { thread = THREAD_NULL; } kr = port_name_to_semaphore(sema_name, &semaphore); if (kr == KERN_SUCCESS) { kr = semaphore_signal_internal(semaphore, thread, SEMAPHORE_OPTION_NONE); semaphore_dereference(semaphore); } if (thread != THREAD_NULL) { thread_deallocate(thread); } return kr; } /* * Routine: semaphore_signal * * Traditional (in-kernel client and MIG interface) semaphore * signal routine. Most users will access the trap version. * * This interface in not defined to return info about whether * this call found a thread waiting or not. The internal * routines (and future external routines) do. We have to * convert those into plain KERN_SUCCESS returns. */ kern_return_t semaphore_signal( semaphore_t semaphore) { kern_return_t kr; if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } kr = semaphore_signal_internal(semaphore, THREAD_NULL, SEMAPHORE_SIGNAL_PREPOST); if (kr == KERN_NOT_WAITING) { return KERN_SUCCESS; } return kr; } /* * Routine: semaphore_signal_trap * * Trap interface to the semaphore_signal function. */ kern_return_t semaphore_signal_trap( struct semaphore_signal_trap_args *args) { mach_port_name_t sema_name = args->signal_name; return semaphore_signal_internal_trap(sema_name); } kern_return_t semaphore_signal_internal_trap(mach_port_name_t sema_name) { semaphore_t semaphore; kern_return_t kr; kr = port_name_to_semaphore(sema_name, &semaphore); if (kr == KERN_SUCCESS) { kr = semaphore_signal_internal(semaphore, THREAD_NULL, SEMAPHORE_SIGNAL_PREPOST); semaphore_dereference(semaphore); if (kr == KERN_NOT_WAITING) { kr = KERN_SUCCESS; } } return kr; } /* * Routine: semaphore_signal_all * * Awakens ALL threads currently blocked on the semaphore. * The semaphore count returns to zero. */ kern_return_t semaphore_signal_all( semaphore_t semaphore) { kern_return_t kr; if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } kr = semaphore_signal_internal(semaphore, THREAD_NULL, SEMAPHORE_SIGNAL_ALL); if (kr == KERN_NOT_WAITING) { return KERN_SUCCESS; } return kr; } /* * Routine: semaphore_signal_all_trap * * Trap interface to the semaphore_signal_all function. */ kern_return_t semaphore_signal_all_trap( struct semaphore_signal_all_trap_args *args) { mach_port_name_t sema_name = args->signal_name; semaphore_t semaphore; kern_return_t kr; kr = port_name_to_semaphore(sema_name, &semaphore); if (kr == KERN_SUCCESS) { kr = semaphore_signal_internal(semaphore, THREAD_NULL, SEMAPHORE_SIGNAL_ALL); semaphore_dereference(semaphore); if (kr == KERN_NOT_WAITING) { kr = KERN_SUCCESS; } } return kr; } /* * Routine: semaphore_convert_wait_result * * Generate the return code after a semaphore wait/block. It * takes the wait result as an input and coverts that to an * appropriate result. */ static kern_return_t semaphore_convert_wait_result(int wait_result) { switch (wait_result) { case THREAD_AWAKENED: return KERN_SUCCESS; case THREAD_TIMED_OUT: return KERN_OPERATION_TIMED_OUT; case THREAD_INTERRUPTED: return KERN_ABORTED; case THREAD_RESTART: return KERN_TERMINATED; default: panic("semaphore_block"); return KERN_FAILURE; } } /* * Routine: semaphore_wait_continue * * Common continuation routine after waiting on a semphore. * It returns directly to user space. */ static void semaphore_wait_continue(void *arg __unused, wait_result_t wr) { thread_t self = current_thread(); semaphore_cont_t caller_cont = self->sth_continuation; assert(self->sth_waitsemaphore != SEMAPHORE_NULL); semaphore_dereference(self->sth_waitsemaphore); if (self->sth_signalsemaphore != SEMAPHORE_NULL) { semaphore_dereference(self->sth_signalsemaphore); } assert(self->handoff_thread == THREAD_NULL); assert(caller_cont != NULL); (*caller_cont)(semaphore_convert_wait_result(wr)); } /* * Routine: semaphore_wait_internal * * Decrements the semaphore count by one. If the count is * negative after the decrement, the calling thread blocks * (possibly at a continuation and/or with a timeout). * * Assumptions: * The reference * A reference is held on the signal semaphore. */ static kern_return_t semaphore_wait_internal( semaphore_t wait_semaphore, semaphore_t signal_semaphore, uint64_t deadline, int option, semaphore_cont_t caller_cont) { int wait_result; spl_t spl_level; kern_return_t kr = KERN_ALREADY_WAITING; thread_t self = current_thread(); thread_t handoff_thread = THREAD_NULL; int semaphore_signal_options = SEMAPHORE_SIGNAL_PREPOST; thread_handoff_option_t handoff_option = THREAD_HANDOFF_NONE; spl_level = splsched(); semaphore_lock(wait_semaphore); if (!semaphore_active(wait_semaphore)) { kr = KERN_TERMINATED; } else if (wait_semaphore->count > 0) { wait_semaphore->count--; kr = KERN_SUCCESS; } else if (option & SEMAPHORE_TIMEOUT_NOBLOCK) { kr = KERN_OPERATION_TIMED_OUT; } else { wait_semaphore->count = -1; /* we don't keep an actual count */ thread_set_pending_block_hint(self, kThreadWaitSemaphore); (void)waitq_assert_wait64_locked( &wait_semaphore->waitq, SEMAPHORE_EVENT, THREAD_ABORTSAFE, TIMEOUT_URGENCY_USER_NORMAL, deadline, TIMEOUT_NO_LEEWAY, self); semaphore_signal_options |= SEMAPHORE_THREAD_HANDOFF; } semaphore_unlock(wait_semaphore); splx(spl_level); /* * wait_semaphore is unlocked so we are free to go ahead and * signal the signal_semaphore (if one was provided). */ if (signal_semaphore != SEMAPHORE_NULL) { kern_return_t signal_kr; /* * lock the signal semaphore reference we got and signal it. * This will NOT block (we cannot block after having asserted * our intention to wait above). */ signal_kr = semaphore_signal_internal(signal_semaphore, THREAD_NULL, semaphore_signal_options); if (signal_kr == KERN_NOT_WAITING) { assert(self->handoff_thread == THREAD_NULL); signal_kr = KERN_SUCCESS; } else if (signal_kr == KERN_TERMINATED) { /* * Uh!Oh! The semaphore we were to signal died. * We have to get ourselves out of the wait in * case we get stuck here forever (it is assumed * that the semaphore we were posting is gating * the decision by someone else to post the * semaphore we are waiting on). People will * discover the other dead semaphore soon enough. * If we got out of the wait cleanly (someone * already posted a wakeup to us) then return that * (most important) result. Otherwise, * return the KERN_TERMINATED status. */ assert(self->handoff_thread == THREAD_NULL); clear_wait(self, THREAD_INTERRUPTED); kr = semaphore_convert_wait_result(self->wait_result); if (kr == KERN_ABORTED) { kr = KERN_TERMINATED; } } } /* * If we had an error, or we didn't really need to wait we can * return now that we have signalled the signal semaphore. */ if (kr != KERN_ALREADY_WAITING) { assert(self->handoff_thread == THREAD_NULL); return kr; } if (self->handoff_thread) { handoff_thread = self->handoff_thread; self->handoff_thread = THREAD_NULL; handoff_option = THREAD_HANDOFF_SETRUN_NEEDED; } /* * Now, we can block. If the caller supplied a continuation * pointer of his own for after the block, block with the * appropriate semaphore continuation. This will gather the * semaphore results, release references on the semaphore(s), * and then call the caller's continuation. */ if (caller_cont) { self->sth_continuation = caller_cont; self->sth_waitsemaphore = wait_semaphore; self->sth_signalsemaphore = signal_semaphore; thread_handoff_parameter(handoff_thread, semaphore_wait_continue, NULL, handoff_option); } else { wait_result = thread_handoff_deallocate(handoff_thread, handoff_option); } assert(self->handoff_thread == THREAD_NULL); return semaphore_convert_wait_result(wait_result); } /* * Routine: semaphore_wait * * Traditional (non-continuation) interface presented to * in-kernel clients to wait on a semaphore. */ kern_return_t semaphore_wait( semaphore_t semaphore) { if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } return semaphore_wait_internal(semaphore, SEMAPHORE_NULL, 0ULL, SEMAPHORE_OPTION_NONE, SEMAPHORE_CONT_NULL); } kern_return_t semaphore_wait_noblock( semaphore_t semaphore) { if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } return semaphore_wait_internal(semaphore, SEMAPHORE_NULL, 0ULL, SEMAPHORE_TIMEOUT_NOBLOCK, SEMAPHORE_CONT_NULL); } kern_return_t semaphore_wait_deadline( semaphore_t semaphore, uint64_t deadline) { if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } return semaphore_wait_internal(semaphore, SEMAPHORE_NULL, deadline, SEMAPHORE_OPTION_NONE, SEMAPHORE_CONT_NULL); } /* * Trap: semaphore_wait_trap * * Trap version of semaphore wait. Called on behalf of user-level * clients. */ kern_return_t semaphore_wait_trap( struct semaphore_wait_trap_args *args) { return semaphore_wait_trap_internal(args->wait_name, thread_syscall_return); } kern_return_t semaphore_wait_trap_internal( mach_port_name_t name, semaphore_cont_t caller_cont) { semaphore_t semaphore; kern_return_t kr; kr = port_name_to_semaphore(name, &semaphore); if (kr == KERN_SUCCESS) { kr = semaphore_wait_internal(semaphore, SEMAPHORE_NULL, 0ULL, SEMAPHORE_OPTION_NONE, caller_cont); semaphore_dereference(semaphore); } return kr; } /* * Routine: semaphore_timedwait * * Traditional (non-continuation) interface presented to * in-kernel clients to wait on a semaphore with a timeout. * * A timeout of {0,0} is considered non-blocking. */ kern_return_t semaphore_timedwait( semaphore_t semaphore, mach_timespec_t wait_time) { int option = SEMAPHORE_OPTION_NONE; uint64_t deadline = 0; if (semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } if (BAD_MACH_TIMESPEC(&wait_time)) { return KERN_INVALID_VALUE; } if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0) { option = SEMAPHORE_TIMEOUT_NOBLOCK; } else { deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec); } return semaphore_wait_internal(semaphore, SEMAPHORE_NULL, deadline, option, SEMAPHORE_CONT_NULL); } /* * Trap: semaphore_timedwait_trap * * Trap version of a semaphore_timedwait. The timeout parameter * is passed in two distinct parts and re-assembled on this side * of the trap interface (to accomodate calling conventions that * pass structures as pointers instead of inline in registers without * having to add a copyin). * * A timeout of {0,0} is considered non-blocking. */ kern_return_t semaphore_timedwait_trap( struct semaphore_timedwait_trap_args *args) { return semaphore_timedwait_trap_internal(args->wait_name, args->sec, args->nsec, thread_syscall_return); } kern_return_t semaphore_timedwait_trap_internal( mach_port_name_t name, unsigned int sec, clock_res_t nsec, semaphore_cont_t caller_cont) { semaphore_t semaphore; mach_timespec_t wait_time; kern_return_t kr; wait_time.tv_sec = sec; wait_time.tv_nsec = nsec; if (BAD_MACH_TIMESPEC(&wait_time)) { return KERN_INVALID_VALUE; } kr = port_name_to_semaphore(name, &semaphore); if (kr == KERN_SUCCESS) { int option = SEMAPHORE_OPTION_NONE; uint64_t deadline = 0; if (sec == 0 && nsec == 0) { option = SEMAPHORE_TIMEOUT_NOBLOCK; } else { deadline = semaphore_deadline(sec, nsec); } kr = semaphore_wait_internal(semaphore, SEMAPHORE_NULL, deadline, option, caller_cont); semaphore_dereference(semaphore); } return kr; } /* * Routine: semaphore_wait_signal * * Atomically register a wait on a semaphore and THEN signal * another. This is the in-kernel entry point that does not * block at a continuation and does not free a signal_semaphore * reference. */ kern_return_t semaphore_wait_signal( semaphore_t wait_semaphore, semaphore_t signal_semaphore) { if (wait_semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } return semaphore_wait_internal(wait_semaphore, signal_semaphore, 0ULL, SEMAPHORE_OPTION_NONE, SEMAPHORE_CONT_NULL); } /* * Trap: semaphore_wait_signal_trap * * Atomically register a wait on a semaphore and THEN signal * another. This is the trap version from user space. */ kern_return_t semaphore_wait_signal_trap( struct semaphore_wait_signal_trap_args *args) { return semaphore_wait_signal_trap_internal(args->wait_name, args->signal_name, thread_syscall_return); } kern_return_t semaphore_wait_signal_trap_internal( mach_port_name_t wait_name, mach_port_name_t signal_name, semaphore_cont_t caller_cont) { semaphore_t wait_semaphore; semaphore_t signal_semaphore; kern_return_t kr; kr = port_name_to_semaphore(signal_name, &signal_semaphore); if (kr == KERN_SUCCESS) { kr = port_name_to_semaphore(wait_name, &wait_semaphore); if (kr == KERN_SUCCESS) { kr = semaphore_wait_internal(wait_semaphore, signal_semaphore, 0ULL, SEMAPHORE_OPTION_NONE, caller_cont); semaphore_dereference(wait_semaphore); } semaphore_dereference(signal_semaphore); } return kr; } /* * Routine: semaphore_timedwait_signal * * Atomically register a wait on a semaphore and THEN signal * another. This is the in-kernel entry point that does not * block at a continuation. * * A timeout of {0,0} is considered non-blocking. */ kern_return_t semaphore_timedwait_signal( semaphore_t wait_semaphore, semaphore_t signal_semaphore, mach_timespec_t wait_time) { int option = SEMAPHORE_OPTION_NONE; uint64_t deadline = 0; if (wait_semaphore == SEMAPHORE_NULL) { return KERN_INVALID_ARGUMENT; } if (BAD_MACH_TIMESPEC(&wait_time)) { return KERN_INVALID_VALUE; } if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0) { option = SEMAPHORE_TIMEOUT_NOBLOCK; } else { deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec); } return semaphore_wait_internal(wait_semaphore, signal_semaphore, deadline, option, SEMAPHORE_CONT_NULL); } /* * Trap: semaphore_timedwait_signal_trap * * Atomically register a timed wait on a semaphore and THEN signal * another. This is the trap version from user space. */ kern_return_t semaphore_timedwait_signal_trap( struct semaphore_timedwait_signal_trap_args *args) { return semaphore_timedwait_signal_trap_internal(args->wait_name, args->signal_name, args->sec, args->nsec, thread_syscall_return); } kern_return_t semaphore_timedwait_signal_trap_internal( mach_port_name_t wait_name, mach_port_name_t signal_name, unsigned int sec, clock_res_t nsec, semaphore_cont_t caller_cont) { semaphore_t wait_semaphore; semaphore_t signal_semaphore; mach_timespec_t wait_time; kern_return_t kr; wait_time.tv_sec = sec; wait_time.tv_nsec = nsec; if (BAD_MACH_TIMESPEC(&wait_time)) { return KERN_INVALID_VALUE; } kr = port_name_to_semaphore(signal_name, &signal_semaphore); if (kr == KERN_SUCCESS) { kr = port_name_to_semaphore(wait_name, &wait_semaphore); if (kr == KERN_SUCCESS) { int option = SEMAPHORE_OPTION_NONE; uint64_t deadline = 0; if (sec == 0 && nsec == 0) { option = SEMAPHORE_TIMEOUT_NOBLOCK; } else { deadline = semaphore_deadline(sec, nsec); } kr = semaphore_wait_internal(wait_semaphore, signal_semaphore, deadline, option, caller_cont); semaphore_dereference(wait_semaphore); } semaphore_dereference(signal_semaphore); } return kr; } /* * Routine: semaphore_reference * * Take out a reference on a semaphore. This keeps the data structure * in existence (but the semaphore may be deactivated). */ void semaphore_reference( semaphore_t semaphore) { zone_id_require(ZONE_ID_SEMAPHORE, sizeof(*semaphore), semaphore); os_ref_retain_raw(&semaphore->ref_count, &sema_refgrp); } /* * Routine: semaphore_dereference * * Release a reference on a semaphore. If this is the last reference, * the semaphore data structure is deallocated. */ void semaphore_dereference( semaphore_t semaphore) { if (semaphore == NULL) { return; } if (os_ref_release_raw(&semaphore->ref_count, &sema_refgrp) == 0) { return semaphore_free(semaphore); } } void kdp_sema_find_owner(struct waitq *waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo) { semaphore_t sem = __container_of(waitq, struct semaphore, waitq); assert(event == SEMAPHORE_EVENT); zone_id_require(ZONE_ID_SEMAPHORE, sizeof(*sem), sem); waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(sem->port); if (sem->owner) { waitinfo->owner = pid_from_task(sem->owner); } } /* * Routine: port_name_to_semaphore * Purpose: * Convert from a port name in the current space to a semaphore. * Produces a semaphore ref, which may be null. * Conditions: * Nothing locked. */ kern_return_t port_name_to_semaphore( mach_port_name_t name, semaphore_t *semaphorep) { ipc_port_t port; kern_return_t kr; if (!MACH_PORT_VALID(name)) { *semaphorep = SEMAPHORE_NULL; return KERN_INVALID_NAME; } kr = ipc_port_translate_send(current_space(), name, &port); if (kr != KERN_SUCCESS) { *semaphorep = SEMAPHORE_NULL; return kr; } /* have the port locked */ *semaphorep = convert_port_to_semaphore(port); if (*semaphorep == SEMAPHORE_NULL) { /* the port is valid, but doesn't denote a semaphore */ kr = KERN_INVALID_CAPABILITY; } else { kr = KERN_SUCCESS; } ip_mq_unlock(port); return kr; } /* * Routine: convert_port_to_semaphore * Purpose: * Convert from a port to a semaphore. * Doesn't consume the port [send-right] ref; * produces a semaphore ref, which may be null. * Conditions: * Caller has a send-right reference to port. * Port may or may not be locked. */ semaphore_t convert_port_to_semaphore(ipc_port_t port) { semaphore_t semaphore = SEMAPHORE_NULL; if (IP_VALID(port)) { semaphore = ipc_kobject_get_stable(port, IKOT_SEMAPHORE); if (semaphore != SEMAPHORE_NULL) { semaphore_reference(semaphore); } } return semaphore; } /* * Routine: convert_semaphore_to_port * Purpose: * Convert a semaphore reference to a send right to a * semaphore port. * * Consumes the semaphore reference. If the semaphore * port currently has no send rights (or doesn't exist * yet), the reference is donated to the port to represent * all extant send rights collectively. */ ipc_port_t convert_semaphore_to_port(semaphore_t semaphore) { if (semaphore == SEMAPHORE_NULL) { return IP_NULL; } /* * make a send right and donate our reference for * semaphore_no_senders if this is the first send right */ if (!ipc_kobject_make_send_lazy_alloc_port(&semaphore->port, semaphore, IKOT_SEMAPHORE, IPC_KOBJECT_ALLOC_NONE)) { semaphore_dereference(semaphore); } return semaphore->port; } /* * Routine: semaphore_no_senders * Purpose: * Called whenever the Mach port system detects no-senders * on the semaphore port. * * When a send-right is first created, a no-senders * notification is armed (and a semaphore reference is donated). * * A no-senders notification will be posted when no one else holds a * send-right (reference) to the semaphore's port. This notification function * will consume the semaphore reference donated to the extant collection of * send-rights. */ static void semaphore_no_senders(ipc_port_t port, __unused mach_port_mscount_t mscount) { semaphore_t semaphore = ipc_kobject_get_stable(port, IKOT_SEMAPHORE); assert(semaphore != SEMAPHORE_NULL); assert(semaphore->port == port); semaphore_dereference(semaphore); } IPC_KOBJECT_DEFINE(IKOT_SEMAPHORE, .iko_op_stable = true, .iko_op_no_senders = semaphore_no_senders);