/* * Copyright (c) 2000-2025 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include "std_safe.h" #include "dt_proxy.h" #include "mock_thread.h" #include "unit_test_utils.h" #include "mock_thread.h" #include "fibers/fibers.h" #include "fibers/mutex.h" #include "fibers/condition.h" #include "fibers/rwlock.h" #include "fibers/random.h" #include "fibers/checker.h" #include // for cpu_data #include #include #include #include #include #include #define UNDEFINED_MOCK \ raw_printf("%s: WIP mock, this should not be called\n", __FUNCTION__); \ print_current_backtrace(); /* * Unit tests that wants to use fibers must redefine this global with a value not 0. * The test executable should not do this directly, instead it should call macro UT_USE_FIBERS in its global scope. * * We use a weak global and not a macro that defines a constructor to avoid initialization code running before such constructor to run * with ut_mocks_use_fibers=0 before that the constructor change its value. * Switching from the pthread mocks to fibers is not supported, we must be consistent from the very beginning. */ int ut_mocks_use_fibers __attribute__((weak)) = 0; /* * Unit tests that wants to use fibers with data race checking must redefine this global with a value not 0. * FIBERS_CHECKER=1 as env var will do the same job too. */ int ut_fibers_use_data_race_checker __attribute__((weak)) = 0; /* * Unit tests can set this variable to force `lck_rw_lock_shared_to_exclusive` to fail. * * RANGELOCKINGTODO rdar://150846598 model when to return FALSE */ bool ut_mocks_lock_upgrade_fail = 0; /* * This constructor is used to set the configuration variables of the fibers using env vars. * The main use case is fuzzing, unit tests should set the variables in the test function or * by calling the correspondig macros (UT_FIBERS_*, see mock_thread.h) in their global scope. */ __attribute__((constructor)) static void initialize_fiber_settings(void) { const char *debug_env = getenv("FIBERS_DEBUG"); if (debug_env != NULL) { fibers_debug = atoi(debug_env); } const char *err_env = getenv("FIBERS_ABORT_ON_ERROR"); if (err_env != NULL) { fibers_abort_on_error = atoi(err_env); } const char *verbose_env = getenv("FIBERS_LOG"); if (verbose_env != NULL) { fibers_log_level = atoi(verbose_env); } const char *prob_env = getenv("FIBERS_MAY_YIELD_PROB"); if (prob_env != NULL) { fibers_may_yield_probability = atoi(prob_env); } const char *checker_env = getenv("FIBERS_CHECK_RACES"); if (checker_env != NULL) { #ifndef __BUILDING_WITH_SANCOV_LOAD_STORES__ raw_printf("==== Fibers data race checker disabled ====\n"); raw_printf("You cannot enable the data race checker if the FIBERS_PREEMPTION=1 flag was to not used as make parameter."); return; #else if (!ut_mocks_use_fibers) { raw_printf("==== Fibers data race checker disabled ====\n"); raw_printf("You cannot enable the data race checker if the test is not using fibers (see UT_USE_FIBERS in the readme)."); return; } ut_fibers_use_data_race_checker = atoi(checker_env); if (ut_fibers_use_data_race_checker) { raw_printf("==== Fibers data race checker enabled ====\n"); } else { raw_printf("==== Fibers data race checker disabled ====\n"); } #endif // __BUILDING_WITH_SANCOV_LOAD_STORES__ } } // --------------- proc and thread ------------------ struct proc; typedef struct proc * proc_t; extern void init_thread_from_template(thread_t thread); extern void ctid_table_init(void); extern void ctid_table_add(thread_t thread); extern void ctid_table_remove(thread_t thread); extern void thread_ro_create(task_t parent_task, thread_t th, thread_ro_t tro_tpl); extern task_t proc_get_task_raw(proc_t proc); extern void task_zone_init(void); extern struct compact_id_table ctid_table; extern lck_grp_t thread_lck_grp; extern size_t proc_struct_size; extern proc_t kernproc; void mock_init_proc(proc_t p, void* (*calloc_call)(size_t, size_t)); // a pointer to this object is kept per thread in thread-local-storage struct mock_thread { struct thread th; fiber_t fiber; struct mock_thread* wq_next; bool interrupts_disabled; }; struct pthread_mock_event_table_entry { event_t ev; pthread_cond_t cond; // the condition variable is owned by the table and is initialized on the first use of the entry bool cond_inited; }; #define PTHREAD_EVENTS_TABLE_SIZE 1000 struct mock_process_state { void *proctask; // buffer for proc and task struct proc *main_proc; struct task *main_task; struct cpu_data cpud; struct mock_thread *main_thread; uint64_t thread_unique_id; uint64_t _faults; uint64_t _pageins; uint64_t _cow_faults; // pthread pthread_key_t tls_thread_key; pthread_mutex_t interrupts_mutex; // if this mutex is locked interrupts are disabled pthread_mutex_t events_mutex; // for all event condition variables struct pthread_mock_event_table_entry events[PTHREAD_EVENTS_TABLE_SIZE]; // !pthread // fibers int interrupts_disabled; // !fibers }; static void mock_destroy_thread(void *th_p) { struct mock_thread *mth = (struct mock_thread *)th_p; // raw_printf("thread_t finished ctid=%u\n", mth->th.ctid); ctid_table_remove(&mth->th); free(mth->th.t_tro); free(mth); } static struct mock_thread * mock_init_new_thread(struct mock_process_state* s) { struct mock_thread *new_mock_thread = calloc(1, sizeof(struct mock_thread)); struct thread *new_thread = &new_mock_thread->th; if (ut_mocks_use_fibers) { new_mock_thread->fiber = fibers_current; fibers_current->extra = new_mock_thread; fibers_current->extra_cleanup_routine = &mock_destroy_thread; } else { pthread_setspecific(s->tls_thread_key, new_mock_thread); } static int mock_init_new_thread_first_call = 1; if (mock_init_new_thread_first_call) { mock_init_new_thread_first_call = 0; compact_id_table_init(&ctid_table); ctid_table_init(); } init_thread_from_template(new_thread); // maybe call thread_create_internal() ? // machine is needed by _enable_preemption_write_count() machine_thread_create(new_thread, s->main_task, true); new_thread->machine.CpuDatap = &s->cpud; new_thread->thread_id = ++s->thread_unique_id; //new_thread->ctid = (uint32_t)new_thread->thread_id; ctid_table_add(new_thread); thread_lock_init(new_thread); wake_lock_init(new_thread); fake_init_lock(&new_thread->mutex); new_thread->t_tro = calloc(1, sizeof(struct thread_ro)); new_thread->t_tro->tro_owner = new_thread; new_thread->t_tro->tro_task = s->main_task; new_thread->t_tro->tro_proc = s->main_proc; // for the main thread this happens before zalloc init so don't do the following which uses zalloc //struct thread_ro tro_tpl = { }; //thread_ro_create(&s->main_task, new_thread, &tro_tpl); new_thread->state = TH_RUN; // raw_printf("thread_t created ctid=%u\n", new_thread->ctid); return new_mock_thread; } void fake_init_task(task_t new_task) { // can't call task_create_internal() since it does zalloc fake_init_lock(&new_task->lock); fake_init_lock(&new_task->task_objq_lock); queue_init(&new_task->task_objq); queue_init(&new_task->threads); new_task->suspend_count = 0; new_task->thread_count = 0; new_task->active_thread_count = 0; new_task->user_stop_count = 0; new_task->legacy_stop_count = 0; new_task->active = TRUE; new_task->halting = FALSE; new_task->priv_flags = 0; new_task->t_flags = 0; new_task->t_procflags = 0; new_task->t_returnwaitflags = 0; new_task->importance = 0; new_task->crashed_thread_id = 0; new_task->watchports = NULL; new_task->t_rr_ranges = NULL; new_task->bank_context = NULL; new_task->pageins = calloc(1, sizeof(uint64_t)); fake_init_lock(&new_task->task_objq_lock); queue_init(&new_task->task_objq); } static void mock_init_threads_state(struct mock_process_state* s) { //task_zone_init(); s->proctask = calloc(1, proc_struct_size + sizeof(struct task)); s->main_proc = (proc_t)s->proctask; s->main_task = proc_get_task_raw(s->main_proc); memset(s->main_proc, 0, proc_struct_size); mock_init_proc(s->main_proc, calloc); kernproc = s->main_proc; // set global variable memset(s->main_task, 0, sizeof(*s->main_task)); fake_init_task(s->main_task); s->_faults = 0; s->main_task->faults = &s->_faults; s->_pageins = 0; s->main_task->pageins = &s->_pageins; s->_cow_faults = 0; s->main_task->cow_faults = &s->_cow_faults; kernel_task = s->main_task; // without this machine_thread_create allocates cpu_data_init(&s->cpud); s->thread_unique_id = 100; if (!ut_mocks_use_fibers) { int ret = pthread_key_create(&s->tls_thread_key, &mock_destroy_thread); if (ret != 0) { raw_printf("failed pthread_key_create"); exit(1); } pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); ret = pthread_mutex_init(&s->interrupts_mutex, &attr); if (ret != 0) { raw_printf("failed pthread_key_create"); exit(1); } pthread_mutexattr_destroy(&attr); ret = pthread_mutex_init(&s->events_mutex, NULL); if (ret != 0) { raw_printf("failed pthread_key_create"); exit(1); } memset(&s->events, 0, sizeof(s->events)); } s->main_thread = mock_init_new_thread(s); } struct mock_process_state * get_proc_state(void) { static struct mock_process_state s; static bool initialized = false; if (!initialized) { // TODO move to fake_kinit.c ? initialized = true; mock_init_threads_state(&s); } return &s; } struct mock_thread * get_mock_thread(void) { struct mock_process_state *s = get_proc_state(); struct mock_thread *mth; if (ut_mocks_use_fibers) { mth = (struct mock_thread *)fibers_current->extra; } else { mth = pthread_getspecific(s->tls_thread_key); } if (mth == NULL) { mth = mock_init_new_thread(s); } return mth; } T_MOCK(thread_t, current_thread_fast, (void)) { return &get_mock_thread()->th; } T_MOCK(uint32_t, kauth_cred_getuid, (void* cred)) { return 0; } // --------------- interrupts disable (spl) --------------------- T_MOCK(boolean_t, ml_get_interrupts_enabled, (void)) { if (ut_mocks_use_fibers) { return get_mock_thread()->interrupts_disabled == 0; } else { pthread_mutex_t *m = &get_proc_state()->interrupts_mutex; int r = pthread_mutex_trylock(m); if (r == 0) { // it's locked, meaning interrupts are disabled pthread_mutex_unlock(m); return false; } PT_QUIET; PT_ASSERT_TRUE(r == EBUSY, "unexpected value in get_interrupts_enabled"); return true; } } // original calls DAIF // interupts disable is mocked by disabling context switches with fiber_t.may_yield_disabled T_MOCK(boolean_t, ml_set_interrupts_enabled, (boolean_t enable)) { if (ut_mocks_use_fibers) { bool prev_interrupts_disabled = get_mock_thread()->interrupts_disabled; FIBERS_LOG(FIBERS_LOG_DEBUG, "ml_set_interrupts_enabled: enable=%d, previous state=%d, may_yield_disabled=%d", enable, !get_mock_thread()->interrupts_disabled, fibers_current->may_yield_disabled); fibers_may_yield_internal_with_reason( (enable ? FIBERS_YIELD_REASON_PREEMPTION_WILL_ENABLE : FIBERS_YIELD_REASON_PREEMPTION_WILL_DISABLE) | FIBERS_YIELD_REASON_ERROR_IF(enable != prev_interrupts_disabled)); // Track the interrupt state per fiber through yield_disabled if (enable && prev_interrupts_disabled) { get_mock_thread()->interrupts_disabled = false; fibers_current->may_yield_disabled--; } else if (!enable && !prev_interrupts_disabled) { get_mock_thread()->interrupts_disabled = true; fibers_current->may_yield_disabled++; } FIBERS_LOG(FIBERS_LOG_DEBUG, "ml_set_interrupts_enabled exit: enable=%d, state=%d, may_yield_disabled=%d", enable, !get_mock_thread()->interrupts_disabled, fibers_current->may_yield_disabled); fibers_may_yield_internal_with_reason( (enable ? FIBERS_YIELD_REASON_PREEMPTION_DID_ENABLE : FIBERS_YIELD_REASON_PREEMPTION_DID_DISABLE) | FIBERS_YIELD_REASON_ERROR_IF(enable != prev_interrupts_disabled)); return !prev_interrupts_disabled; } else { pthread_mutex_t *m = &get_proc_state()->interrupts_mutex; if (enable) { int ret = pthread_mutex_unlock(m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "interrupts pthread_mutex_unlock"); } else { // disable interrupts locks int ret = pthread_mutex_lock(m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "interrupts pthread_mutex_lock"); } } return true; } T_MOCK(boolean_t, ml_set_interrupts_enabled_with_debug, (boolean_t enable, boolean_t __unused debug)) { return MOCK_ml_set_interrupts_enabled(enable); } T_MOCK(void, _disable_preemption, (void)) { if (ut_mocks_use_fibers) { fibers_may_yield_internal_with_reason( FIBERS_YIELD_REASON_PREEMPTION_WILL_DISABLE | FIBERS_YIELD_REASON_ERROR_IF(fibers_current->may_yield_disabled != 0)); fibers_current->may_yield_disabled++; FIBERS_LOG(FIBERS_LOG_DEBUG, "disable_preemption: may_yield_disabled=%d", fibers_current->may_yield_disabled); thread_t thread = MOCK_current_thread_fast(); unsigned int count = thread->machine.preemption_count; os_atomic_store(&thread->machine.preemption_count, count + 1, compiler_acq_rel); fibers_may_yield_internal_with_reason( FIBERS_YIELD_REASON_PREEMPTION_DID_DISABLE | FIBERS_YIELD_REASON_ERROR_IF(fibers_current->may_yield_disabled != 1)); } else { pthread_mutex_t *m = &get_proc_state()->interrupts_mutex; int ret = pthread_mutex_lock(m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "_disable_preemption pthread_mutex_lock"); thread_t thread = MOCK_current_thread_fast(); unsigned int count = thread->machine.preemption_count; os_atomic_store(&thread->machine.preemption_count, count + 1, compiler_acq_rel); } } T_MOCK(void, _disable_preemption_without_measurements, (void)) { MOCK__disable_preemption(); } T_MOCK(void, lock_disable_preemption_for_thread, (thread_t t)) { MOCK__disable_preemption(); } T_MOCK(void, _enable_preemption, (void)) { if (ut_mocks_use_fibers) { fibers_may_yield_internal_with_reason( FIBERS_YIELD_REASON_PREEMPTION_WILL_ENABLE | FIBERS_YIELD_REASON_ERROR_IF(fibers_current->may_yield_disabled != 1)); fibers_current->may_yield_disabled--; FIBERS_LOG(FIBERS_LOG_DEBUG, "enable_preemption: may_yield_disabled=%d", fibers_current->may_yield_disabled); thread_t thread = current_thread(); unsigned int count = thread->machine.preemption_count; os_atomic_store(&thread->machine.preemption_count, count - 1, compiler_acq_rel); fibers_may_yield_internal_with_reason( FIBERS_YIELD_REASON_PREEMPTION_DID_ENABLE | FIBERS_YIELD_REASON_ERROR_IF(fibers_current->may_yield_disabled != 0)); } else { thread_t thread = current_thread(); unsigned int count = thread->machine.preemption_count; os_atomic_store(&thread->machine.preemption_count, count - 1, compiler_acq_rel); pthread_mutex_t *m = &get_proc_state()->interrupts_mutex; int ret = pthread_mutex_unlock(m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "_enable_preemption pthread_mutex_unlock"); } } // --------------- mutex ------------------ struct mock_lck_mtx_t { union { pthread_mutex_t *pt_m; fibers_mutex_t *f_m; }; lck_mtx_state_t lck_mtx; }; static_assert(sizeof(struct mock_lck_mtx_t) == sizeof(lck_mtx_t)); void fake_init_lock(lck_mtx_t * lck) { struct mock_lck_mtx_t* mlck = (struct mock_lck_mtx_t*)lck; if (ut_mocks_use_fibers) { mlck->f_m = calloc(1, sizeof(fibers_mutex_t)); fibers_mutex_init(mlck->f_m); } else { mlck->pt_m = calloc(1, sizeof(pthread_mutex_t)); int ret = pthread_mutex_init(mlck->pt_m, NULL); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "pthread_mutex_init"); } } T_MOCK(void, lck_mtx_init, (lck_mtx_t * lck, lck_grp_t * grp, lck_attr_t * attr)) { fake_init_lock(lck); } T_MOCK(void, lck_mtx_destroy, (lck_mtx_t * lck, lck_grp_t * grp)) { struct mock_lck_mtx_t* mlck = (struct mock_lck_mtx_t*)lck; if (ut_mocks_use_fibers) { fibers_mutex_destroy(mlck->f_m); free(mlck->f_m); mlck->f_m = NULL; } else { int ret = pthread_mutex_destroy(mlck->pt_m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "pthread_mutex_destroy"); free(mlck->pt_m); mlck->pt_m = NULL; } } T_MOCK(void, lck_mtx_lock, (lck_mtx_t * lock)) { uint32_t ctid = MOCK_current_thread_fast()->ctid; struct mock_lck_mtx_t* mlck = (struct mock_lck_mtx_t*)lock; if (ut_mocks_use_fibers) { fibers_mutex_lock(mlck->f_m, true); } else { int ret = pthread_mutex_lock(mlck->pt_m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "pthread_mutex_lock"); } mlck->lck_mtx.owner = ctid; } T_MOCK(void, lck_mtx_lock_spin, (lck_mtx_t * lock)) { uint32_t ctid = MOCK_current_thread_fast()->ctid; struct mock_lck_mtx_t* mlck = (struct mock_lck_mtx_t*)lock; if (ut_mocks_use_fibers) { fibers_mutex_lock(mlck->f_m, false); // do not check for disabled preemption if spinlock } else { int ret = pthread_mutex_lock(mlck->pt_m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "pthread_mutex_lock"); } mlck->lck_mtx.owner = ctid; } T_MOCK(boolean_t, lck_mtx_try_lock, (lck_mtx_t * lock)) { uint32_t ctid = MOCK_current_thread_fast()->ctid; struct mock_lck_mtx_t* mlck = (struct mock_lck_mtx_t*)lock; int ret; if (ut_mocks_use_fibers) { ret = fibers_mutex_try_lock(mlck->f_m); } else { int ret = pthread_mutex_trylock(mlck->pt_m); } if (ret == 0) { mlck->lck_mtx.owner = ctid; return TRUE; } else { return FALSE; } } T_MOCK(void, lck_mtx_unlock, (lck_mtx_t * lock)) { struct mock_lck_mtx_t* mlck = (struct mock_lck_mtx_t*)lock; mlck->lck_mtx.owner = 0; if (ut_mocks_use_fibers) { fibers_mutex_unlock(mlck->f_m); } else { int ret = pthread_mutex_unlock(mlck->pt_m); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "pthread_mutex_unlock"); } } T_MOCK(void, mutex_pause, (uint32_t collisions)) { if (ut_mocks_use_fibers) { // we can't sleep to not break determinism, trigger a ctxswitch instead fibers_yield(); } else { mutex_pause(collisions); } } // --------------- rwlocks ------------------ struct mock_lck_rw_t { fibers_rwlock_t *rw; // lck_rw_word_t lck_rw; // RANGELOCKINGTODO rdar://150846598 uint32_t lck_rw_owner; }; static_assert(sizeof(struct mock_lck_rw_t) == sizeof(lck_rw_t)); static_assert(LCK_RW_ASSERT_SHARED == FIBERS_RWLOCK_ASSERT_SHARED); static_assert(LCK_RW_ASSERT_EXCLUSIVE == FIBERS_RWLOCK_ASSERT_EXCLUSIVE); static_assert(LCK_RW_ASSERT_HELD == FIBERS_RWLOCK_ASSERT_HELD); static_assert(LCK_RW_ASSERT_NOTHELD == FIBERS_RWLOCK_ASSERT_NOTHELD); void fake_init_rwlock(struct mock_lck_rw_t *mlck) { mlck->rw = calloc(1, sizeof(fibers_rwlock_t)); fibers_rwlock_init(mlck->rw); } static boolean_t fake_rw_try_lock(struct mock_lck_rw_t *mlck, lck_rw_type_t lck_rw_type) { int ret; // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_inc(MOCK_current_thread_fast(), (const void*)mlck); if (lck_rw_type == LCK_RW_TYPE_SHARED) { ret = fibers_rwlock_try_rdlock(mlck->rw); } else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) { ret = fibers_rwlock_try_wrlock(mlck->rw); if (ret == 0) { mlck->lck_rw_owner = MOCK_current_thread_fast()->ctid; } } else { PT_FAIL("lck_rw_try_lock: Invalid lock type"); } if (ret != 0) { // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_dec(MOCK_current_thread_fast(), (const void*)mlck); } return ret == 0; } static bool fake_rw_lock_would_yield_exclusive(struct mock_lck_rw_t *mlck, lck_rw_yield_t mode) { fibers_rwlock_assert(mlck->rw, FIBERS_RWLOCK_ASSERT_EXCLUSIVE); bool yield = false; if (mode == LCK_RW_YIELD_ALWAYS) { yield = true; } else { if (mlck->rw->writer_wait_queue.count > 0) { yield = true; } else if (mode == LCK_RW_YIELD_ANY_WAITER) { yield = (mlck->rw->reader_wait_queue.count != 0); } } return yield; } T_MOCK(void, lck_rw_init, ( lck_rw_t * lck, lck_grp_t * grp, lck_attr_t * attr)) { if (!ut_mocks_use_fibers) { lck_rw_init(lck, grp, attr); return; } // RANGELOCKINGTODO rdar://150846598 mock attr, especially lck_rw_can_sleep struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fake_init_rwlock(mlck); } T_MOCK(void, lck_rw_destroy, (lck_rw_t * lck, lck_grp_t * grp)) { if (!ut_mocks_use_fibers) { lck_rw_destroy(lck, grp); return; } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_destroy(mlck->rw); free(mlck->rw); mlck->rw = NULL; } T_MOCK(void, lck_rw_unlock, (lck_rw_t * lck, lck_rw_type_t lck_rw_type)) { if (!ut_mocks_use_fibers) { lck_rw_unlock(lck, lck_rw_type); return; } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; if (mlck->rw->writer_active) { mlck->lck_rw_owner = 0; } fibers_rwlock_unlock(mlck->rw); // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_dec(MOCK_current_thread_fast(), (const void*)mlck); } static void lck_rw_old_mock_unlock_shared(lck_rw_t * lck) { if (!ut_mocks_use_fibers) { lck_rw_unlock_shared(lck); return; } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_rdunlock(mlck->rw); // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_dec(MOCK_current_thread_fast(), (const void*)mlck); } T_MOCK(void, lck_rw_unlock_shared, (lck_rw_t * lck)) { lck_rw_old_mock_unlock_shared(lck); } T_MOCK(void, lck_rw_unlock_exclusive, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { lck_rw_unlock_exclusive(lck); return; } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; mlck->lck_rw_owner = 0; fibers_rwlock_wrunlock(mlck->rw); // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_dec(MOCK_current_thread_fast(), (const void*)mlck); } T_MOCK(void, lck_rw_lock_exclusive, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { lck_rw_lock_exclusive(lck); return; } // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_inc(MOCK_current_thread_fast(), (const void*)lck); struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_wrlock(mlck->rw, true); mlck->lck_rw_owner = MOCK_current_thread_fast()->ctid; } T_MOCK(void, lck_rw_lock_shared, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { lck_rw_lock_shared(lck); return; } // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_inc(MOCK_current_thread_fast(), (const void*)lck); struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_rdlock(mlck->rw, true); } T_MOCK(boolean_t, lck_rw_try_lock, (lck_rw_t * lck, lck_rw_type_t lck_rw_type)) { if (!ut_mocks_use_fibers) { return lck_rw_try_lock(lck, lck_rw_type); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; return fake_rw_try_lock(mlck, lck_rw_type); } T_MOCK(boolean_t, lck_rw_try_lock_exclusive, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { return lck_rw_try_lock_exclusive(lck); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; return fake_rw_try_lock(mlck, LCK_RW_TYPE_EXCLUSIVE); } T_MOCK(boolean_t, lck_rw_try_lock_shared, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { return lck_rw_try_lock_shared(lck); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; return fake_rw_try_lock(mlck, LCK_RW_TYPE_SHARED); } T_MOCK(lck_rw_type_t, lck_rw_done, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { return lck_rw_done(lck); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; mlck->lck_rw_owner = 0; // If there is a writer locking it must be the current fiber or will trigger an assertion in fibers_rwlock_wrunlock lck_rw_type_t ret = mlck->rw->writer_active ? LCK_RW_TYPE_EXCLUSIVE : LCK_RW_TYPE_SHARED; fibers_rwlock_unlock(mlck->rw); // RANGELOCKINGTODO rdar://150846598 handle old lock can_sleep lck_rw_lock_count_dec(MOCK_current_thread_fast(), (const void*)mlck); return ret; } T_MOCK(boolean_t, lck_rw_lock_shared_to_exclusive, (lck_rw_t * lck)) { if (ut_mocks_lock_upgrade_fail) { lck_rw_old_mock_unlock_shared(lck); return false; } if (!ut_mocks_use_fibers) { return lck_rw_lock_shared_to_exclusive(lck); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; return fibers_rwlock_upgrade(mlck->rw); } T_MOCK(void, lck_rw_lock_exclusive_to_shared, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { lck_rw_lock_exclusive_to_shared(lck); return; } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_downgrade(mlck->rw); } T_MOCK(void, lck_rw_assert, ( lck_rw_t * lck, unsigned int type)) { if (!ut_mocks_use_fibers) { lck_rw_assert(lck, type); return; } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_assert(mlck->rw, type); } T_MOCK(bool, lck_rw_lock_would_yield_exclusive, ( lck_rw_t * lck, lck_rw_yield_t mode)) { if (!ut_mocks_use_fibers) { return lck_rw_lock_would_yield_exclusive(lck, mode); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; return fake_rw_lock_would_yield_exclusive(mlck, mode); } T_MOCK(bool, lck_rw_lock_would_yield_shared, (lck_rw_t * lck)) { if (!ut_mocks_use_fibers) { return lck_rw_lock_would_yield_shared(lck); } struct mock_lck_rw_t* mlck = (struct mock_lck_rw_t*)lck; fibers_rwlock_assert(mlck->rw, FIBERS_RWLOCK_ASSERT_SHARED); return mlck->rw->writer_wait_queue.count != 0; } // Note: No need to mock lck_rw_sleep as it uses lck_rw_* API and waitq, we already mock everything the function uses // --------------- waitq ------------------ /* * If the 4 bytes of mock_waitq.mock_magic are not matching MOCK_WAITQ_MAGIC * it means the waitq comes from an unsupported location and was not created with mock_waitq_init(). */ #define MOCK_WAITQ_MAGIC 0xb60d0d8f struct mock_waitq_extra { bool valid; fibers_condition_t cond; fibers_mutex_t mutex; struct mock_thread *waiting_threads; int waiting_thread_count; // Count of waiting threads }; struct mock_waitq { // 24 bytes WAITQ_FLAGS(waitq, waitq_eventmask:_EVENT_MASK_BITS); unsigned int mock_magic; event64_t current_event; // delete when every waiting thread is removed struct mock_waitq_extra *extra; }; static_assert(sizeof(struct waitq) == sizeof(struct mock_waitq)); #define MWQCAST(xnu_wq) ((struct mock_waitq *)(xnu_wq).wq_q) static bool waitq_use_real_impl(waitq_t wq) { return !ut_mocks_use_fibers || waitq_type(wq) != WQT_QUEUE; } int mock_waitq_init(struct mock_waitq *wq) { if (!wq) { return EINVAL; } wq->mock_magic = MOCK_WAITQ_MAGIC; wq->current_event = 0; wq->extra = calloc(sizeof(struct mock_waitq_extra), 1); wq->extra->valid = true; fibers_mutex_init(&wq->extra->mutex); return 0; } int mock_waitq_destroy(struct mock_waitq *wq) { if (!wq) { return EINVAL; } PT_QUIET; PT_ASSERT_TRUE(wq->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); fibers_condition_destroy(&wq->extra->cond); fibers_mutex_destroy(&wq->extra->mutex); free(wq->extra); wq->extra = NULL; return 0; } static inline bool waitq_should_unlock(waitq_wakeup_flags_t flags) { return (flags & (WAITQ_UNLOCK | WAITQ_KEEP_LOCKED)) == WAITQ_UNLOCK; } static inline bool waitq_should_enable_interrupts(waitq_wakeup_flags_t flags) { return (flags & (WAITQ_UNLOCK | WAITQ_KEEP_LOCKED | WAITQ_ENABLE_INTERRUPTS)) == (WAITQ_UNLOCK | WAITQ_ENABLE_INTERRUPTS); } T_MOCK(void, waitq_init, (waitq_t wq, waitq_type_t type, int policy)) { if (!ut_mocks_use_fibers || type == WQT_PORT) { waitq_init(wq, type, policy); return; } *wq.wq_q = (struct waitq){ .waitq_type = type, .waitq_fifo = ((policy & SYNC_POLICY_REVERSED) == 0), }; // RANGELOCKINGTODO rdar://150846598 PT_QUIET; PT_ASSERT_TRUE(type == WQT_QUEUE, "invalid waitq type"); mock_waitq_init(MWQCAST(wq)); if (policy & SYNC_POLICY_INIT_LOCKED) { fibers_mutex_lock(&MWQCAST(wq)->extra->mutex, false); } } T_MOCK(void, waitq_deinit, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { waitq_deinit(wq); return; } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); mock_waitq_destroy(MWQCAST(wq)); } T_MOCK(void, waitq_lock, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { waitq_lock(wq); return; } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); fibers_mutex_lock(&MWQCAST(wq)->extra->mutex, false); } T_MOCK(void, waitq_unlock, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { waitq_unlock(wq); return; } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); fibers_mutex_unlock(&MWQCAST(wq)->extra->mutex); } T_MOCK(bool, waitq_is_valid, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { return waitq_is_valid(wq); } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); return MWQCAST(wq)->extra->valid; } T_MOCK(void, waitq_invalidate, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { return waitq_invalidate(wq); } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); MWQCAST(wq)->extra->valid = false; } T_MOCK(bool, waitq_held, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { return waitq_held(wq); } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); return MWQCAST(wq)->extra->mutex.holder != NULL; } T_MOCK(void, waitq_lock_wait, (waitq_t wq, uint32_t ticket)) { MOCK_waitq_lock(wq); } T_MOCK(bool, waitq_lock_try, (waitq_t wq)) { if (waitq_use_real_impl(wq)) { return waitq_lock_try(wq); } PT_QUIET; PT_ASSERT_TRUE(MWQCAST(wq)->mock_magic == MOCK_WAITQ_MAGIC, "missing mock_waitq magic"); return fibers_mutex_try_lock(&MWQCAST(wq)->extra->mutex) == 0; } // --------------- events ------------------ #define MOCK_WAITQS_NUM 4096 static struct mock_waitq global_mock_waitqs[MOCK_WAITQS_NUM]; static int global_mock_waitqs_inited = 0; static void global_mock_waitqs_init(void) { for (int i = 0; i < MOCK_WAITQS_NUM; ++i) { MOCK_waitq_init((struct waitq*)&global_mock_waitqs[i], WQT_QUEUE, SYNC_POLICY_FIFO); } global_mock_waitqs_inited = 1; } struct mock_waitq* find_mock_waitq(event64_t event) { if (!global_mock_waitqs_inited) { global_mock_waitqs_init(); } for (int i = 0; i < MOCK_WAITQS_NUM; ++i) { if (global_mock_waitqs[i].current_event == event) { return &global_mock_waitqs[i]; } } return NULL; } struct mock_waitq* find_or_alloc_mock_waitq(event64_t event) { if (!global_mock_waitqs_inited) { global_mock_waitqs_init(); } int first_free = -1; for (int i = 0; i < MOCK_WAITQS_NUM; ++i) { if (global_mock_waitqs[i].current_event == event) { return &global_mock_waitqs[i]; } else if (first_free < 0 && global_mock_waitqs[i].current_event == 0) { first_free = i; } } PT_QUIET; PT_ASSERT_TRUE(first_free >= 0, "no more space in global_mock_waitqs"); global_mock_waitqs[first_free].current_event = event; return &global_mock_waitqs[first_free]; } // --------------- waitq mocks ------------------ // pthread mocks struct pthread_mock_event_table_entry* find_pthread_mock_event_entry(struct mock_process_state *s, event_t ev) { for (int i = 0; i < PTHREAD_EVENTS_TABLE_SIZE; ++i) { if (s->events[i].ev == ev) { return &s->events[i]; } } return NULL; } T_MOCK_DYNAMIC(kern_return_t, thread_wakeup_prim, ( event_t event, boolean_t one_thread, wait_result_t result), (event, one_thread, result), { if (ut_mocks_use_fibers) { // fibers is mocking waitq apis, go forward calling the real thread_wakeup_prim return thread_wakeup_prim(event, one_thread, result); } kern_return_t kr = KERN_SUCCESS; struct mock_process_state *s = get_proc_state(); int ret = pthread_mutex_lock(&s->events_mutex); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_wakeup pthread_mutex_lock"); struct pthread_mock_event_table_entry* event_entry = find_pthread_mock_event_entry(s, event); if (event_entry == NULL) { kr = KERN_NOT_WAITING; goto done; } if (one_thread) { ret = pthread_cond_signal(&event_entry->cond); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_wakeup pthread_cond_signal"); } else { ret = pthread_cond_broadcast(&event_entry->cond); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_wakeup pthread_cond_broadcast"); } done: pthread_mutex_unlock(&s->events_mutex); return kr; }); wait_result_t pthread_mock_thread_block_reason( thread_continue_t continuation, void *parameter, ast_t reason) { PT_QUIET; PT_ASSERT_TRUE(continuation == THREAD_CONTINUE_NULL && parameter == NULL && reason == AST_NONE, "thread_block argument"); struct mock_process_state *s = get_proc_state(); int ret = pthread_mutex_lock(&s->events_mutex); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_block pthread_mutex_lock"); // find empty entry in table struct pthread_mock_event_table_entry *event_entry = find_pthread_mock_event_entry(s, 0); PT_QUIET; PT_ASSERT_NOTNULL(event_entry, "empty entry not found"); // register the entry to this event event_entry->ev = (event_t)MOCK_current_thread_fast()->wait_event; // if it doesn't have a condition variable yet, create one if (!event_entry->cond_inited) { ret = pthread_cond_init(&event_entry->cond, NULL); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_block pthread_cond_init"); event_entry->cond_inited = true; } // wait on variable. This releases the mutex, waits and reaquires it before returning ret = pthread_cond_wait(&event_entry->cond, &s->events_mutex); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_block pthread_cond_wait"); // reset the entry so that it can be reused (will be done by all waiters that woke up) event_entry->ev = 0; ret = pthread_mutex_unlock(&s->events_mutex); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "thread_block pthread_mutex_unlock"); return THREAD_AWAKENED; } kern_return_t pthread_mock_clear_wait( thread_t thread, wait_result_t result) { struct mock_process_state *s = get_proc_state(); int ret = pthread_mutex_lock(&s->events_mutex); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "clear_wait pthread_mutex_lock"); struct pthread_mock_event_table_entry *event_entry = find_pthread_mock_event_entry(s, 0); PT_QUIET; PT_ASSERT_NOTNULL(event_entry, "empty entry not found"); event_entry->ev = 0; ret = pthread_mutex_unlock(&s->events_mutex); PT_QUIET; PT_ASSERT_POSIX_ZERO(ret, "clear_wait pthread_mutex_unlock"); return KERN_SUCCESS; } // fibers mocks T_MOCK(struct waitq *, _global_eventq, (event64_t event)) { if (!ut_mocks_use_fibers) { return _global_eventq(event); } struct waitq *ret = (struct waitq *)find_or_alloc_mock_waitq(event); return ret; } T_MOCK(wait_result_t, waitq_assert_wait64_locked, ( waitq_t waitq, event64_t wait_event, wait_interrupt_t interruptible, wait_timeout_urgency_t urgency, uint64_t deadline, uint64_t leeway, thread_t thread)) { if (waitq_use_real_impl(waitq)) { return waitq_assert_wait64_locked(waitq, wait_event, interruptible, urgency, deadline, leeway, thread); } struct mock_waitq *wq = MWQCAST(waitq); if (wq->current_event == 0) { wq->current_event = wait_event; } PT_QUIET; PT_ASSERT_TRUE(wq->current_event == wait_event, "waitq_assert_wait64_locked another event queue"); struct mock_thread * mock_thread = (struct mock_thread*)thread; // !!! ASSUME every thread_t is created from mock_thread mock_thread->wq_next = wq->extra->waiting_threads; wq->extra->waiting_threads = mock_thread; wq->extra->waiting_thread_count++; thread->wait_event = wait_event; // Store waiting event in thread context thread->state |= TH_WAIT; // Set thread state to waiting thread->waitq = waitq; return THREAD_WAITING; // Indicate thread is now waiting, but not blocked yet } T_MOCK(wait_result_t, waitq_assert_wait64, ( struct waitq *waitq, event64_t wait_event, wait_interrupt_t interruptible, uint64_t deadline)) { if (waitq_use_real_impl(waitq)) { return waitq_assert_wait64(waitq, wait_event, interruptible, deadline); } thread_t thread = MOCK_current_thread_fast(); MOCK_waitq_lock(waitq); wait_result_t res = MOCK_waitq_assert_wait64_locked(waitq, wait_event, interruptible, TIMEOUT_URGENCY_SYS_NORMAL, deadline, TIMEOUT_NO_LEEWAY, thread); MOCK_waitq_unlock(waitq); return res; } static void mock_waitq_clear_wait(struct mock_thread * thread, struct mock_waitq *wq) { struct mock_thread ** mock_thread = &wq->extra->waiting_threads; int removed = 0; while (*mock_thread) { if (*mock_thread == thread) { *mock_thread = (*mock_thread)->wq_next; removed = 1; break; } mock_thread = &(*mock_thread)->wq_next; } PT_QUIET; PT_ASSERT_TRUE(removed, "thread_block thread not in wq"); thread->wq_next = NULL; wq->extra->waiting_thread_count--; if (wq->extra->waiting_thread_count == 0) { wq->current_event = 0; // reset current_event } PT_QUIET; PT_ASSERT_TRUE(wq->extra->waiting_thread_count >= 0, "something bad"); } static struct mock_thread * mock_waitq_pop_wait(struct mock_waitq *wq) { if (wq->extra->waiting_thread_count == 0) { return NULL; } struct mock_thread * thread = wq->extra->waiting_threads; wq->extra->waiting_threads = thread->wq_next; thread->wq_next = NULL; wq->extra->waiting_thread_count--; if (wq->extra->waiting_thread_count == 0) { wq->current_event = 0; // reset current_event } PT_QUIET; PT_ASSERT_TRUE(wq->extra->waiting_thread_count >= 0, "something bad"); return thread; } T_MOCK_DYNAMIC(wait_result_t, thread_block_reason, ( thread_continue_t continuation, void *parameter, ast_t reason), ( continuation, parameter, reason), { if (!ut_mocks_use_fibers) { return pthread_mock_thread_block_reason(continuation, parameter, reason); } PT_QUIET; PT_ASSERT_TRUE(continuation == THREAD_CONTINUE_NULL && parameter == NULL && reason == AST_NONE, "thread_block argument"); thread_t thread = current_thread(); PT_QUIET; PT_ASSERT_TRUE(thread->state & TH_WAIT, "thread_block called but thread state is not TH_WAIT"); /* * In case of a window between assert_wait and thread_block * another thread could wake up the current thread after being added to the waitq * but before the block. * In this case, the thread will still be TH_WAIT but without an assigned waitq. * TH_WAKING must be set. */ struct mock_waitq *wq = MWQCAST(thread->waitq); if (wq == NULL) { PT_QUIET; PT_ASSERT_TRUE(thread->state & TH_WAKING, "with waitq == NULL there must be TH_WAKING set"); thread->state &= ~TH_WAKING; goto awake_thread; } fibers_condition_wait(&wq->extra->cond); if (thread->state & TH_WAKING) { thread->state &= ~TH_WAKING; } else { // is this possible? TH_WAKING is always set ATM in the mocks, keep this code to be more robust thread->waitq.wq_q = NULL; mock_waitq_clear_wait((struct mock_thread *)thread, wq); } awake_thread: thread->state &= ~TH_WAIT; thread->state |= TH_RUN; return thread->wait_result; }); T_MOCK(kern_return_t, clear_wait, (thread_t thread, wait_result_t wresult)) { if (!ut_mocks_use_fibers) { return pthread_mock_clear_wait(thread, wresult); } struct mock_waitq *wq = MWQCAST(thread->waitq); PT_QUIET; PT_ASSERT_TRUE(wq != NULL, "thread->waitq is NULL"); thread->state &= ~TH_WAIT; thread->waitq.wq_q = NULL; thread->wait_result = wresult; mock_waitq_clear_wait((struct mock_thread *)thread, wq); return KERN_SUCCESS; } typedef struct { wait_result_t wait_result; } waitq_wakeup_args_t; static void waitq_wakeup_fiber_callback(void *arg, fiber_t target) { waitq_wakeup_args_t *wakeup_args = (waitq_wakeup_args_t*)arg; struct mock_thread *thread = (struct mock_thread *)target->extra; assert(thread); struct mock_waitq *wq = MWQCAST(thread->th.waitq); assert(wq); thread->th.state |= TH_WAKING; thread->th.waitq.wq_q = NULL; thread->th.wait_result = wakeup_args->wait_result; mock_waitq_clear_wait(thread, wq); } // Called from thread_wakeup_nthreads_prim T_MOCK(uint32_t, waitq_wakeup64_nthreads_locked, ( waitq_t waitq, event64_t wake_event, wait_result_t result, waitq_wakeup_flags_t flags, uint32_t nthreads)) { if (waitq_use_real_impl(waitq)) { return waitq_wakeup64_nthreads_locked(waitq, wake_event, result, flags, nthreads); } // RANGELOCKINGTODO rdar://150846598 flags waitq_wakeup_args_t wakeup_args = { .wait_result = result }; struct mock_waitq *wq = MWQCAST(waitq); PT_QUIET; PT_ASSERT_TRUE(wq->current_event == wake_event, "waitq_wakeup64_nthreads current_event is wrong"); // Avoid to trigger a switch in fibers_condition_wakeup_some before a valid state in the waitq fibers_current->may_yield_disabled++; FIBERS_LOG(FIBERS_LOG_DEBUG, "waitq_wakeup64_nthreads_locked nthreads=%u wake_event=%lld", nthreads, wake_event); int count = fibers_condition_wakeup_some(&wq->extra->cond, nthreads, &waitq_wakeup_fiber_callback, &wakeup_args); /* * In case of a window in which a thread is pushed to the waitq but thread_block was still not called * when another thread wakes up the threads in the waitq here. * fibers_condition_wakeup_some will not find these fibers as they are not waiting on the condition, * In this case these fibers must be in FIBER_STOP that means that they are ready to be scheduled, * but we still need to take action here to remove them from the waitq and clear the state. */ while (wq->extra->waiting_thread_count && count < nthreads) { struct mock_thread *thread = mock_waitq_pop_wait(wq); PT_QUIET; PT_ASSERT_TRUE(thread->fiber->state & FIBER_STOP, "leftover fiber in waitq not in FIBER_STOP"); thread->th.state |= TH_WAKING; thread->th.waitq.wq_q = NULL; thread->th.wait_result = result; ++count; } fibers_current->may_yield_disabled--; if (waitq_should_unlock(flags)) { MOCK_waitq_unlock(waitq); } if (waitq_should_enable_interrupts(flags)) { MOCK_ml_set_interrupts_enabled(1); } return (uint32_t)count; } T_MOCK(thread_t, waitq_wakeup64_identify_locked, ( waitq_t waitq, event64_t wake_event, waitq_wakeup_flags_t flags)) { if (waitq_use_real_impl(waitq)) { return waitq_wakeup64_identify_locked(waitq, wake_event, flags); } // RANGELOCKINGTODO rdar://150846598 flags struct mock_waitq *wq = MWQCAST(waitq); PT_QUIET; PT_ASSERT_TRUE(wq->current_event == wake_event, "waitq_wakeup64_identify_locked current_event is wrong"); // RANGELOCKINGTODO rdar://150845975 for fuzzing select random, not the top of the queue struct mock_thread * mock_thread = wq->extra->waiting_threads; if (mock_thread == NULL) { return THREAD_NULL; } // Preemption will be re-enabled when the thread is resumed in `waitq_resume_identify_thread` MOCK__disable_preemption(); mock_thread->th.state |= TH_WAKING; mock_thread->th.waitq.wq_q = NULL; mock_thread->th.wait_result = THREAD_AWAKENED; mock_waitq_clear_wait(mock_thread, wq); FIBERS_LOG(FIBERS_LOG_DEBUG, "waitq_wakeup64_identify_locked identified fiber %d", mock_thread->fiber->id); if (waitq_should_unlock(flags)) { MOCK_waitq_unlock(waitq); } if (waitq_should_enable_interrupts(flags)) { MOCK_ml_set_interrupts_enabled(1); } fibers_may_yield_internal(); return &mock_thread->th; } T_MOCK(void, waitq_resume_identified_thread, ( waitq_t waitq, thread_t thread, wait_result_t result, waitq_wakeup_flags_t flags)) { if (waitq_use_real_impl(waitq)) { return waitq_resume_identified_thread(waitq, thread, result, flags); } // RANGELOCKINGTODO rdar://150846598 other flags struct mock_thread * mock_thread = (struct mock_thread*)thread; // !!! ASSUME every thread_t is created from mock_thread struct mock_waitq *wq = MWQCAST(waitq); bool found = fibers_condition_wakeup_identified(&wq->extra->cond, mock_thread->fiber); if (!found) { /* * In case of a window in which a thread is pushed to the waitq but thread_block was still not called * when the thread is identified by another one and resumed, we pop it from the waitq in waitq_wakeup64_identify_locked * but we will not find it in wq->cond.wait_queue. * In this case it is not needed any action as the fiber must be in FIBER_STOP and can already be scheduled. */ PT_QUIET; PT_ASSERT_TRUE(mock_thread->fiber->state & FIBER_STOP, "waitq_resume_identified_thread fiber not found in condition and not in FIBER_STOP"); } // Paired with the call to `waitq_wakeup64_identify_locked` MOCK__enable_preemption(); fibers_may_yield_internal_with_reason( FIBERS_YIELD_REASON_WAKEUP | FIBERS_YIELD_REASON_ERROR_IF(!found)); } // Allow to cause a context switch from a function that can be called from XNU T_MOCK(void, ut_fibers_ctxswitch, (void)) { if (ut_mocks_use_fibers) { fibers_yield(); } } // Allow to cause a context switch to a specific fiber from a function that can be called from XNU T_MOCK(void, ut_fibers_ctxswitch_to, (int fiber_id)) { if (ut_mocks_use_fibers) { fibers_yield_to(fiber_id); } } // Get the current fiber id from a function that can be called from XNU T_MOCK(int, ut_fibers_current_id, (void)) { if (ut_mocks_use_fibers) { return fibers_current->id; } return -1; } // --------------- preemption ------------------ #ifdef __BUILDING_WITH_SANCOV_LOAD_STORES__ // Optional: uncomment to enable yield at every basic block entry /* * T_MOCK(void, * __sanitizer_cov_trace_pc_guard, (uint32_t * guard)) * { * fibers_may_yield(); * } */ #define IS_ALIGNED(ptr, size) ( (((uintptr_t)(ptr)) & (((uintptr_t)(size)) - 1)) == 0 ) #define IS_ATOMIC(ptr, size) ( (size) <= sizeof(uint64_t) && IS_ALIGNED(ptr, size) ) // These functions can be called from XNU to enter/exit atomic regions in which the data checker is disabled T_MOCK(void, data_race_checker_atomic_begin, (void)) { fibers_checker_atomic_begin(); } T_MOCK(void, data_race_checker_atomic_end, (void)) { fibers_checker_atomic_end(); } /* * Detecting data races on memory operations: * Memory operation functions are used to check for data races using the fibers checkers API, a software implementation of DataCollider. * The idea is to set a watchpoint before context switching and report a data race every time a concurrent access (watchpoint hit) is in between a write or a write in between a load. * To be more robust, we also check that the value pointed the memory operation address before the context switch is still the same after the context switch. * If not, very likely it is a data race. Atomic memory operations should be excluded from this, we use the IS_ATOMIC macro to filter memory loads. * Note: atomic_fetch_add_explicit() et al. on ARM64 are compiled to LDADD et al. that seem to not be supported by __sanitizer_cov_loadX, ok for us we want to skip atomic operations. */ #define SANCOV_LOAD_STORE_DATA_CHECKER(type, size, access_type) do { \ if (fibers_current->may_yield_disabled) { \ return; \ } \ if (fibers_scheduler->fibers_should_yield(fibers_scheduler_context, \ fibers_may_yield_probability, FIBERS_YIELD_REASON_PREEMPTION_TRIGGER)) { \ volatile type before = *addr; \ void *pc = __builtin_return_address(0); \ bool has_wp = check_and_set_watchpoint(pc, (uintptr_t)addr, size, access_type); \ \ fibers_queue_push(&fibers_run_queue, fibers_current); \ fibers_choose_next(FIBER_STOP); \ \ if (has_wp) { \ post_check_and_remove_watchpoint((uintptr_t)addr, size, access_type); \ } \ type after = *addr; \ if (before != after) { \ report_value_race((uintptr_t)addr, size, access_type); \ } \ } \ } while (0) /* * Mock the SanitizerCoverage load/store instrumentation callbacks (original in san_attached.c). * The functions are execute at every memory operations in libxnu and in the test binary, libmocks is excluded. * Functions and files in tools/sanitizers-ignorelist are excluded from instrumentation. */ #define MOCK_SANCOV_LOAD_STORE(type, size) \ __attribute__((optnone)) \ T_MOCK(void, \ __sanitizer_cov_load##size, (type* addr)) \ { \ if (!ut_fibers_use_data_race_checker || IS_ATOMIC(addr, size) || fibers_current->disable_race_checker) { \ fibers_may_yield_with_reason(FIBERS_YIELD_REASON_PREEMPTION_TRIGGER); \ return; \ } \ SANCOV_LOAD_STORE_DATA_CHECKER(type, size, ACCESS_TYPE_LOAD); \ } \ \ __attribute__((optnone)) \ T_MOCK(void, \ __sanitizer_cov_store##size, (type* addr)) \ { /* do not care about atomicity for stores */ \ if (!ut_fibers_use_data_race_checker || fibers_current->disable_race_checker) { \ fibers_may_yield_with_reason(FIBERS_YIELD_REASON_PREEMPTION_TRIGGER); \ return; \ } \ SANCOV_LOAD_STORE_DATA_CHECKER(type, size, ACCESS_TYPE_STORE); \ } MOCK_SANCOV_LOAD_STORE(uint8_t, 1) MOCK_SANCOV_LOAD_STORE(uint16_t, 2) MOCK_SANCOV_LOAD_STORE(uint32_t, 4) MOCK_SANCOV_LOAD_STORE(uint64_t, 8) MOCK_SANCOV_LOAD_STORE(__uint128_t, 16) #endif // __BUILDING_WITH_SANCOV__