Lines Matching refs:stackshot_ctx
368 #define stackshot_ctx (stackshot_contexts[cur_stackshot_ctx_idx]) macro
369 #define stackshot_args (stackshot_ctx.sc_args)
602 os_atomic_dec(&stackshot_ctx.sc_cpus_working, acquire); in stackshot_panic_spin()
604 if (stackshot_ctx.sc_calling_cpuid == cpu_number()) { in stackshot_panic_spin()
605 while (os_atomic_load(&stackshot_ctx.sc_cpus_working, acquire) != 0) { in stackshot_panic_spin()
621 if (__improbable(os_atomic_load(&stackshot_ctx.sc_state, relaxed) == SS_PANICKED)) { in stackshot_panic_guard()
642 o_state = os_atomic_xchg(&stackshot_ctx.sc_state, SS_PANICKED, seq_cst); in stackshot_cpu_signal_panic()
650 os_atomic_dec(&stackshot_ctx.sc_cpus_working, acquire); in stackshot_cpu_signal_panic()
652 while (os_atomic_load(&stackshot_ctx.sc_cpus_working, seq_cst) != 0) { in stackshot_cpu_signal_panic()
667 …if (__improbable(!os_atomic_cmpxchgv(&stackshot_ctx.sc_state, SS_RUNNING, err_state, &cur_state, s… in stackshot_set_error()
669 os_atomic_cmpxchg(&stackshot_ctx.sc_state, SS_SETUP, err_state, seq_cst); in stackshot_set_error()
684 stackshot_state_t state = os_atomic_load(&stackshot_ctx.sc_state, relaxed); in stackshot_status_check()
799 assert(!stackshot_ctx.sc_is_singlethreaded); in stackshot_buffer_alloc()
838 ret = stackshot_buffer_alloc(size, &stackshot_ctx.sc_buffers[my_cluster], &err); in stackshot_best_buffer_alloc()
842 for (size_t buf_idx = 0; buf_idx < stackshot_ctx.sc_num_buffers; buf_idx++) { in stackshot_best_buffer_alloc()
847 ret = stackshot_buffer_alloc(size, &stackshot_ctx.sc_buffers[buf_idx], &err); in stackshot_best_buffer_alloc()
863 return stackshot_buffer_alloc(size, &stackshot_ctx.sc_buffers[0], error); in stackshot_best_buffer_alloc()
879 assert(!stackshot_ctx.sc_is_singlethreaded); in stackshot_buffer_free()
915 if (stackshot_ctx.sc_is_singlethreaded) { in stackshot_alloc_with_size()
948 size_t buf_size = MAX(min_size, stackshot_ctx.sc_min_kcdata_size); in linked_kcdata_init()
985 …stackshot_buffer_free((void*) descriptor->kcd_addr_end, &stackshot_ctx.sc_buffers[buffer], free_si… in stackshot_kcdata_free_unused()
1062 assert(!stackshot_ctx.sc_panic_stackshot); in stackshot_new_linked_kcdata()
1088 assert(!stackshot_ctx.sc_panic_stackshot); in stackshot_finalize_linked_kcdata()
1278 os_atomic_store(&stackshot_ctx.sc_state, SS_INACTIVE, release); in stack_snapshot_from_kernel()
1284 if (error == KERN_SUCCESS && !stackshot_ctx.sc_is_singlethreaded) { in stack_snapshot_from_kernel()
1295 if (!stackshot_ctx.sc_is_singlethreaded) { in stack_snapshot_from_kernel()
1411 queue = &stackshot_ctx.sc_workqueues[queue_idx]; in stackshot_put_workitem()
1623 for (size_t buf_idx = 0; buf_idx < stackshot_ctx.sc_num_buffers; buf_idx++) { in stackshot_push_duration_and_latency()
1624 struct stackshot_buffer *buf = &stackshot_ctx.sc_buffers[buf_idx]; in stackshot_push_duration_and_latency()
1628 stackshot_ctx.sc_latency.buffer_size = stackshot_ctx.sc_args.buffer_size; in stackshot_push_duration_and_latency()
1629 stackshot_ctx.sc_latency.buffer_overhead = buffer_overhead; in stackshot_push_duration_and_latency()
1630 stackshot_ctx.sc_latency.buffer_used = buffer_used; in stackshot_push_duration_and_latency()
1631 stackshot_ctx.sc_latency.buffer_count = stackshot_ctx.sc_num_buffers; in stackshot_push_duration_and_latency()
1636 sizeof(stackshot_ctx.sc_latency), &stackshot_ctx.sc_latency)); in stackshot_push_duration_and_latency()
1641 assert(!stackshot_ctx.sc_panic_stackshot); in stackshot_push_duration_and_latency()
1645 memcpy(duration_p, &stackshot_ctx.sc_duration, sizeof(*duration_p)); in stackshot_push_duration_and_latency()
1649 …ta(kcdata, STACKSHOT_KCTYPE_STACKSHOT_DURATION, sizeof(stackshot_ctx.sc_duration), &stackshot_ctx.… in stackshot_push_duration_and_latency()
1671 if (stackshot_ctx.sc_is_singlethreaded) { in stackshot_alloc_final_kcdata()
1681 stackshot_ctx.sc_finalized_kcdata = kcdata_memory_alloc_init(final_kcdata_buffer, hdr_tag, in stackshot_alloc_final_kcdata()
1684 if (stackshot_ctx.sc_finalized_kcdata == NULL) { in stackshot_alloc_final_kcdata()
1698 if (stackshot_ctx.sc_is_singlethreaded || (stackshot_ctx.sc_finalized_kcdata == NULL)) { in stackshot_free_final_kcdata()
1702 …kmem_free(kernel_map, stackshot_ctx.sc_finalized_kcdata->kcd_addr_begin, stackshot_args.buffer_siz… in stackshot_free_final_kcdata()
1703 kcdata_memory_destroy(stackshot_ctx.sc_finalized_kcdata); in stackshot_free_final_kcdata()
1704 stackshot_ctx.sc_finalized_kcdata = NULL; in stackshot_free_final_kcdata()
1723 assert(!stackshot_ctx.sc_panic_stackshot && !stackshot_ctx.sc_is_singlethreaded); in stackshot_collect_kcdata()
1728 assert(stackshot_ctx.sc_finalized_kcdata != NULL); in stackshot_collect_kcdata()
1734 …kcd_exit_on_error(kcdata_init_compress(stackshot_ctx.sc_finalized_kcdata, hdr_tag, kdp_memcpy, KCD… in stackshot_collect_kcdata()
1738 …kcd_exit_on_error(stackshot_copy_linked_kcdata(stackshot_ctx.sc_finalized_kcdata, stackshot_ctx.sc… in stackshot_collect_kcdata()
1742 os_atomic_store(&stackshot_ctx.sc_workqueues[i].sswq_cur_item, 0, relaxed); in stackshot_collect_kcdata()
1752 struct stackshot_workqueue *queue = &stackshot_ctx.sc_workqueues[i]; in stackshot_collect_kcdata()
1777 …kcd_exit_on_error(stackshot_copy_linked_kcdata(stackshot_ctx.sc_finalized_kcdata, next_item->sswi_… in stackshot_collect_kcdata()
1781 …kcd_exit_on_error(stackshot_copy_linked_kcdata(stackshot_ctx.sc_finalized_kcdata, stackshot_ctx.sc… in stackshot_collect_kcdata()
1804 assert(!stackshot_ctx.sc_panic_stackshot && !stackshot_ctx.sc_is_singlethreaded); in stackshot_finalize_kcdata()
1807 assert(stackshot_ctx.sc_finalized_kcdata != NULL); in stackshot_finalize_kcdata()
1810 kcd_exit_on_error(stackshot_push_duration_and_latency(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_kcdata()
1813 kcd_finalize_compression(stackshot_ctx.sc_finalized_kcdata); in stackshot_finalize_kcdata()
1814 …kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_ctx.sc_finalized_kcdata, stackshot_… in stackshot_finalize_kcdata()
1815 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_kcdata()
1817 …stackshot_ctx.sc_bytes_traced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_ctx.sc_finalized… in stackshot_finalize_kcdata()
1818 …stackshot_ctx.sc_bytes_uncompressed = (uint32_t) kcdata_memory_get_uncompressed_bytes(stackshot_ct… in stackshot_finalize_kcdata()
1820 if (os_atomic_load(&stackshot_ctx.sc_retval, relaxed) == KERN_SUCCESS) { in stackshot_finalize_kcdata()
1822 kcd_exit_on_error(kcdata_finish(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_kcdata()
1825 …memcpy(stackshot_args.buffer, (void*) stackshot_ctx.sc_finalized_kcdata->kcd_addr_begin, stackshot… in stackshot_finalize_kcdata()
1829 …ackshot_args.buffer + ((mach_vm_address_t) stackshot_duration_outer - stackshot_ctx.sc_finalized_k… in stackshot_finalize_kcdata()
1848 assert(stackshot_ctx.sc_is_singlethreaded); in stackshot_finalize_singlethreaded_kcdata()
1850 kcd_exit_on_error(stackshot_push_duration_and_latency(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_singlethreaded_kcdata()
1852 kcd_finalize_compression(stackshot_ctx.sc_finalized_kcdata); in stackshot_finalize_singlethreaded_kcdata()
1853 …kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_ctx.sc_finalized_kcdata, stackshot_… in stackshot_finalize_singlethreaded_kcdata()
1854 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_singlethreaded_kcdata()
1856 …stackshot_ctx.sc_bytes_traced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_ctx.sc_finalized… in stackshot_finalize_singlethreaded_kcdata()
1857 …stackshot_ctx.sc_bytes_uncompressed = (uint32_t) kcdata_memory_get_uncompressed_bytes(stackshot_ct… in stackshot_finalize_singlethreaded_kcdata()
1859 kcd_exit_on_error(kcdata_finish(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_singlethreaded_kcdata()
1861 if (stackshot_ctx.sc_panic_stackshot) { in stackshot_finalize_singlethreaded_kcdata()
1862 *stackshot_args.descriptor = *stackshot_ctx.sc_finalized_kcdata; in stackshot_finalize_singlethreaded_kcdata()
1978 kcdata_add_container_marker(stackshot_ctx.sc_finalized_kcdata, KCDATA_TYPE_CONTAINER_BEGIN, in collect_exclave_threads()
1994 kr = kcdata_add_container_marker(stackshot_ctx.sc_finalized_kcdata, KCDATA_TYPE_CONTAINER_END, in collect_exclave_threads()
2268 …kr = stackshot_exclaves_process_stackshot(result, stackshot_ctx.sc_finalized_kcdata, want_raw_addr… in stackshot_exclaves_process_result()
2546 os_atomic_store(&stackshot_ctx.sc_state, SS_INACTIVE, release); in kern_stack_snapshot_internal()
2556 if (error == KERN_SUCCESS && !stackshot_ctx.sc_is_singlethreaded) { in kern_stack_snapshot_internal()
2570 if (stackshot_ctx.sc_is_singlethreaded) { in kern_stack_snapshot_internal()
2704 stackshot_ctx = (struct stackshot_context) { in kdp_snapshot_preflight_internal()
2720 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_snapshot_preflight_internal()
2723 stackshot_ctx.sc_num_buffers = is_singlethreaded ? 1 : ml_get_cluster_count(); in kdp_snapshot_preflight_internal()
2725 stackshot_ctx.sc_num_buffers = 1; in kdp_snapshot_preflight_internal()
2727 size_t bufsz = args.buffer_size / stackshot_ctx.sc_num_buffers; in kdp_snapshot_preflight_internal()
2728 for (int buf_idx = 0; buf_idx < stackshot_ctx.sc_num_buffers; buf_idx++) { in kdp_snapshot_preflight_internal()
2729 stackshot_ctx.sc_buffers[buf_idx] = (struct stackshot_buffer) { in kdp_snapshot_preflight_internal()
2808 stackshot_ctx = (struct stackshot_context) { 0 }; in stackshot_reset_state()
2820 return os_atomic_load(&stackshot_ctx.sc_state, relaxed) != SS_INACTIVE; in stackshot_active()
2832 return stackshot_ctx.sc_bytes_traced; in kdp_stack_snapshot_bytes_traced()
2838 return stackshot_ctx.sc_bytes_uncompressed; in kdp_stack_snapshot_bytes_uncompressed()
2899 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in stackshot_plh_est_size()
2916 stackshot_ctx.sc_plh = (struct port_label_hash){.plh_size = 0}; /* structure assignment */ in stackshot_plh_reset()
2964 PLH_STAT_OP(os_atomic_inc(&stackshot_ctx.sc_plh.plh_bad, relaxed)); in stackshot_plh_setup()
2974 stackshot_ctx.sc_plh = plh; /* structure assignment */ in stackshot_plh_setup()
3004 uint16_t plh_size = stackshot_ctx.sc_plh.plh_size; in stackshot_plh_resetgen()
3023 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in stackshot_plh_lookup_locked()
3093 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in kdp_stackshot_plh_record_locked()
3149 plh_lock(&stackshot_ctx.sc_plh); in kdp_stackshot_plh_record()
3151 plh_unlock(&stackshot_ctx.sc_plh); in kdp_stackshot_plh_record()
3159 plh_lock(&stackshot_ctx.sc_plh); in stackshot_plh_lookup()
3161 plh_unlock(&stackshot_ctx.sc_plh); in stackshot_plh_lookup()
3170 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in kdp_stackshot_plh_stats()
3921 assert(stackshot_ctx.sc_is_singlethreaded); in kcdata_record_task_snapshot()
4256 …laves_state & TH_EXCLAVES_RPC) && stackshot_exclave_inspect_ctids && !stackshot_ctx.sc_panic_stack… in kcdata_record_thread_snapshot()
4354 .sbc_allow_faulting = stackshot_ctx.sc_enable_faulting, in kcdata_record_thread_snapshot()
4634 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_record_task()
4938 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_record_task()
5071 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_kcdata_format()
5085 stackshot_ctx.sc_enable_faulting = (stackshot_flags & (STACKSHOT_ENABLE_BT_FAULTING)); in kdp_stackshot_kcdata_format()
5098 if (!stackshot_ctx.sc_panic_stackshot && collect_exclaves) { in kdp_stackshot_kcdata_format()
5109 if (stackshot_ctx.sc_bytes_uncompressed) { in kdp_stackshot_kcdata_format()
5110 stackshot_ctx.sc_bytes_uncompressed = 0; in kdp_stackshot_kcdata_format()
5119 if (!stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5121 stackshot_ctx.sc_pretask_kcdata = stackshot_cpu_ctx.scc_kcdata_head; in kdp_stackshot_kcdata_format()
5136 stackshot_ctx.sc_latency.setup_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5168 …stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &stackshot_ctx.sc_microsecs)); in kdp_stackshot_kcdata_format()
5186 if (!stackshot_ctx.sc_is_singlethreaded && (stackshot_flags & STACKSHOT_THREAD_GROUP)) { in kdp_stackshot_kcdata_format()
5214 if (!stackshot_ctx.sc_panic_stackshot && (thread_group_begin_cpu_cycle_count != 0)) { in kdp_stackshot_kcdata_format()
5225 …stackshot_ctx.sc_latency.setup_latency_mt = mach_absolute_time() - stackshot_ctx.sc_latency.setup_… in kdp_stackshot_kcdata_format()
5226 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5227 stackshot_ctx.sc_latency.total_task_iteration_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5229 stackshot_ctx.sc_latency.task_queue_building_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5263 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5280 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5281 …stackshot_ctx.sc_latency.total_task_iteration_latency_mt = mach_absolute_time() - stackshot_ctx.sc… in kdp_stackshot_kcdata_format()
5283 …stackshot_ctx.sc_latency.task_queue_building_latency_mt = mach_absolute_time() - stackshot_ctx.sc_… in kdp_stackshot_kcdata_format()
5288 if (!stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5291 stackshot_ctx.sc_posttask_kcdata = stackshot_cpu_ctx.scc_kcdata_head; in kdp_stackshot_kcdata_format()
5303 if (!stackshot_ctx.sc_panic_stackshot && (stackshot_flags & STACKSHOT_SAVE_JETSAM_COALITIONS)) { in kdp_stackshot_kcdata_format()
5330 if (!stackshot_ctx.sc_panic_stackshot && (coalition_begin_cpu_cycle_count != 0)) { in kdp_stackshot_kcdata_format()
5343 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5344 stackshot_ctx.sc_latency.total_terminated_task_iteration_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5346 stackshot_ctx.sc_latency.terminated_task_queue_building_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5369 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5382 os_atomic_store(&stackshot_ctx.sc_workqueues[i].sswq_populated, true, release); in kdp_stackshot_kcdata_format()
5390 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5391 …stackshot_ctx.sc_latency.total_terminated_task_iteration_latency_mt = mach_absolute_time() - stack… in kdp_stackshot_kcdata_format()
5393 …stackshot_ctx.sc_latency.terminated_task_queue_building_latency_mt = mach_absolute_time() - stacks… in kdp_stackshot_kcdata_format()
5399 stackshot_ctx.sc_latency.latency_version = 2; in kdp_stackshot_kcdata_format()
5400 stackshot_ctx.sc_latency.main_cpu_number = stackshot_ctx.sc_main_cpuid; in kdp_stackshot_kcdata_format()
5401 stackshot_ctx.sc_latency.calling_cpu_number = stackshot_ctx.sc_calling_cpuid; in kdp_stackshot_kcdata_format()
5406 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_kcdata_format()
5417 if (!stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5426 if (stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_kcdata_format()
5539 if (!stackshot_ctx.sc_panic_stackshot) { in stackshot_find_phys()
5552 …_fault_stats.sfs_time_spent_faulting >= stackshot_max_fault_time) && !stackshot_ctx.sc_panic_stack… in stackshot_find_phys()
5614 if (!stackshot_ctx.sc_is_singlethreaded) { in do_stackshot()
5618 stackshot_ctx.sc_workqueues[i] = (struct stackshot_workqueue) { in do_stackshot()
5642 if (stackshot_ctx.sc_main_cpuid == -1) { in do_stackshot()
5643 os_atomic_cmpxchg(&stackshot_ctx.sc_main_cpuid, -1, cpu_number(), acquire); in do_stackshot()
5648 os_atomic_store(&stackshot_ctx.sc_state, SS_RUNNING, release); in do_stackshot()
5651 if (stackshot_ctx.sc_panic_stackshot || (stackshot_ctx.sc_main_cpuid == cpu_number())) { in do_stackshot()
5652 stackshot_ctx.sc_retval = kdp_stackshot_kcdata_format(); in do_stackshot()
5659 stackshot_ctx.sc_latency.cpu_wait_latency_mt = mach_absolute_time(); in do_stackshot()
5662 os_atomic_dec(&stackshot_ctx.sc_cpus_working, seq_cst); in do_stackshot()
5665 while (os_atomic_load(&stackshot_ctx.sc_cpus_working, seq_cst) != 0) { in do_stackshot()
5670 …stackshot_ctx.sc_latency.cpu_wait_latency_mt = mach_absolute_time() - stackshot_ctx.sc_latency.cpu… in do_stackshot()
5675 stackshot_ctx.sc_duration = (struct stackshot_duration_v2) { in do_stackshot()
5697 if (!stackshot_ctx.sc_panic_stackshot && interrupt_masked_debug_mode) { in do_stackshot()
5709 if (stackshot_ctx.sc_retval == KERN_SUCCESS) { in do_stackshot()
5710 stackshot_ctx.sc_retval = stackshot_status_check(); in do_stackshot()
5715 if (!stackshot_ctx.sc_panic_stackshot && stackshot_ctx.sc_retval == KERN_SUCCESS) { in do_stackshot()
5718 if (stackshot_ctx.sc_retval != KERN_SUCCESS && stackshot_exclave_inspect_ctids) { in do_stackshot()
5727 if (stackshot_ctx.sc_is_singlethreaded) { in do_stackshot()
5728 stackshot_ctx.sc_finalized_kcdata = stackshot_kcdata_p; in do_stackshot()
5731 return stackshot_ctx.sc_retval; in do_stackshot()
5785 os_atomic_cmpxchg(&stackshot_ctx.sc_main_cpuid, -1, my_cpu_no, acquire); in stackshot_cpu_preflight()
5788 is_calling_cpu = stackshot_ctx.sc_calling_cpuid == my_cpu_no; in stackshot_cpu_preflight()
5791 …stackshot_cpu_ctx.scc_can_work = is_calling_cpu || (is_recommended && !stackshot_ctx.sc_is_singlet… in stackshot_cpu_preflight()
5794 os_atomic_inc(&stackshot_ctx.sc_cpus_working, relaxed); in stackshot_cpu_preflight()
5856 kcd_exit_on_error(stackshot_cpu_work_on_queue(&stackshot_ctx.sc_workqueues[i - 1])); in stackshot_cpu_do_work()
5861 kcd_exit_on_error(stackshot_cpu_work_on_queue(&stackshot_ctx.sc_workqueues[i])); in stackshot_cpu_do_work()
5901 while (os_atomic_load(&stackshot_ctx.sc_state, acquire) == SS_SETUP) { in stackshot_aux_cpu_entry()
5913 if (stackshot_ctx.sc_main_cpuid == cpu_number()) { in stackshot_aux_cpu_entry()
5914 stackshot_ctx.sc_retval = kdp_stackshot_kcdata_format(); in stackshot_aux_cpu_entry()
5920 os_atomic_dec(&stackshot_ctx.sc_cpus_working, release); in stackshot_aux_cpu_entry()