Lines Matching refs:stackshot_ctx
368 #define stackshot_ctx (stackshot_contexts[cur_stackshot_ctx_idx]) macro
369 #define stackshot_args (stackshot_ctx.sc_args)
602 os_atomic_dec(&stackshot_ctx.sc_cpus_working, acquire); in stackshot_panic_spin()
604 if (stackshot_ctx.sc_calling_cpuid == cpu_number()) { in stackshot_panic_spin()
605 while (os_atomic_load(&stackshot_ctx.sc_cpus_working, acquire) != 0) { in stackshot_panic_spin()
621 if (__improbable(os_atomic_load(&stackshot_ctx.sc_state, relaxed) == SS_PANICKED)) { in stackshot_panic_guard()
642 o_state = os_atomic_xchg(&stackshot_ctx.sc_state, SS_PANICKED, seq_cst); in stackshot_cpu_signal_panic()
650 os_atomic_dec(&stackshot_ctx.sc_cpus_working, acquire); in stackshot_cpu_signal_panic()
652 while (os_atomic_load(&stackshot_ctx.sc_cpus_working, seq_cst) != 0) { in stackshot_cpu_signal_panic()
667 …if (__improbable(!os_atomic_cmpxchgv(&stackshot_ctx.sc_state, SS_RUNNING, err_state, &cur_state, s… in stackshot_set_error()
669 os_atomic_cmpxchg(&stackshot_ctx.sc_state, SS_SETUP, err_state, seq_cst); in stackshot_set_error()
684 stackshot_state_t state = os_atomic_load(&stackshot_ctx.sc_state, relaxed); in stackshot_status_check()
799 assert(!stackshot_ctx.sc_is_singlethreaded); in stackshot_buffer_alloc()
838 ret = stackshot_buffer_alloc(size, &stackshot_ctx.sc_buffers[my_cluster], &err); in stackshot_best_buffer_alloc()
842 for (size_t buf_idx = 0; buf_idx < stackshot_ctx.sc_num_buffers; buf_idx++) { in stackshot_best_buffer_alloc()
847 ret = stackshot_buffer_alloc(size, &stackshot_ctx.sc_buffers[buf_idx], &err); in stackshot_best_buffer_alloc()
863 return stackshot_buffer_alloc(size, &stackshot_ctx.sc_buffers[0], error); in stackshot_best_buffer_alloc()
879 assert(!stackshot_ctx.sc_is_singlethreaded); in stackshot_buffer_free()
915 if (stackshot_ctx.sc_is_singlethreaded) { in stackshot_alloc_with_size()
948 size_t buf_size = MAX(min_size, stackshot_ctx.sc_min_kcdata_size); in linked_kcdata_init()
985 …stackshot_buffer_free((void*) descriptor->kcd_addr_end, &stackshot_ctx.sc_buffers[buffer], free_si… in stackshot_kcdata_free_unused()
1062 assert(!stackshot_ctx.sc_panic_stackshot); in stackshot_new_linked_kcdata()
1088 assert(!stackshot_ctx.sc_panic_stackshot); in stackshot_finalize_linked_kcdata()
1278 os_atomic_store(&stackshot_ctx.sc_state, SS_INACTIVE, release); in stack_snapshot_from_kernel()
1284 if (error == KERN_SUCCESS && !stackshot_ctx.sc_is_singlethreaded) { in stack_snapshot_from_kernel()
1295 if (!stackshot_ctx.sc_is_singlethreaded) { in stack_snapshot_from_kernel()
1402 queue = &stackshot_ctx.sc_workqueues[queue_idx]; in stackshot_put_workitem()
1614 for (size_t buf_idx = 0; buf_idx < stackshot_ctx.sc_num_buffers; buf_idx++) { in stackshot_push_duration_and_latency()
1615 struct stackshot_buffer *buf = &stackshot_ctx.sc_buffers[buf_idx]; in stackshot_push_duration_and_latency()
1619 stackshot_ctx.sc_latency.buffer_size = stackshot_ctx.sc_args.buffer_size; in stackshot_push_duration_and_latency()
1620 stackshot_ctx.sc_latency.buffer_overhead = buffer_overhead; in stackshot_push_duration_and_latency()
1621 stackshot_ctx.sc_latency.buffer_used = buffer_used; in stackshot_push_duration_and_latency()
1622 stackshot_ctx.sc_latency.buffer_count = stackshot_ctx.sc_num_buffers; in stackshot_push_duration_and_latency()
1627 sizeof(stackshot_ctx.sc_latency), &stackshot_ctx.sc_latency)); in stackshot_push_duration_and_latency()
1632 assert(!stackshot_ctx.sc_panic_stackshot); in stackshot_push_duration_and_latency()
1636 memcpy(duration_p, &stackshot_ctx.sc_duration, sizeof(*duration_p)); in stackshot_push_duration_and_latency()
1640 …ta(kcdata, STACKSHOT_KCTYPE_STACKSHOT_DURATION, sizeof(stackshot_ctx.sc_duration), &stackshot_ctx.… in stackshot_push_duration_and_latency()
1662 if (stackshot_ctx.sc_is_singlethreaded) { in stackshot_alloc_final_kcdata()
1672 stackshot_ctx.sc_finalized_kcdata = kcdata_memory_alloc_init(final_kcdata_buffer, hdr_tag, in stackshot_alloc_final_kcdata()
1675 if (stackshot_ctx.sc_finalized_kcdata == NULL) { in stackshot_alloc_final_kcdata()
1689 if (stackshot_ctx.sc_is_singlethreaded || (stackshot_ctx.sc_finalized_kcdata == NULL)) { in stackshot_free_final_kcdata()
1693 …kmem_free(kernel_map, stackshot_ctx.sc_finalized_kcdata->kcd_addr_begin, stackshot_args.buffer_siz… in stackshot_free_final_kcdata()
1694 kcdata_memory_destroy(stackshot_ctx.sc_finalized_kcdata); in stackshot_free_final_kcdata()
1695 stackshot_ctx.sc_finalized_kcdata = NULL; in stackshot_free_final_kcdata()
1714 assert(!stackshot_ctx.sc_panic_stackshot && !stackshot_ctx.sc_is_singlethreaded); in stackshot_collect_kcdata()
1719 assert(stackshot_ctx.sc_finalized_kcdata != NULL); in stackshot_collect_kcdata()
1725 …kcd_exit_on_error(kcdata_init_compress(stackshot_ctx.sc_finalized_kcdata, hdr_tag, kdp_memcpy, KCD… in stackshot_collect_kcdata()
1729 …kcd_exit_on_error(stackshot_copy_linked_kcdata(stackshot_ctx.sc_finalized_kcdata, stackshot_ctx.sc… in stackshot_collect_kcdata()
1733 os_atomic_store(&stackshot_ctx.sc_workqueues[i].sswq_cur_item, 0, relaxed); in stackshot_collect_kcdata()
1743 struct stackshot_workqueue *queue = &stackshot_ctx.sc_workqueues[i]; in stackshot_collect_kcdata()
1768 …kcd_exit_on_error(stackshot_copy_linked_kcdata(stackshot_ctx.sc_finalized_kcdata, next_item->sswi_… in stackshot_collect_kcdata()
1772 …kcd_exit_on_error(stackshot_copy_linked_kcdata(stackshot_ctx.sc_finalized_kcdata, stackshot_ctx.sc… in stackshot_collect_kcdata()
1795 assert(!stackshot_ctx.sc_panic_stackshot && !stackshot_ctx.sc_is_singlethreaded); in stackshot_finalize_kcdata()
1798 assert(stackshot_ctx.sc_finalized_kcdata != NULL); in stackshot_finalize_kcdata()
1801 kcd_exit_on_error(stackshot_push_duration_and_latency(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_kcdata()
1804 kcd_finalize_compression(stackshot_ctx.sc_finalized_kcdata); in stackshot_finalize_kcdata()
1805 …kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_ctx.sc_finalized_kcdata, stackshot_… in stackshot_finalize_kcdata()
1806 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_kcdata()
1808 …stackshot_ctx.sc_bytes_traced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_ctx.sc_finalized… in stackshot_finalize_kcdata()
1809 …stackshot_ctx.sc_bytes_uncompressed = (uint32_t) kcdata_memory_get_uncompressed_bytes(stackshot_ct… in stackshot_finalize_kcdata()
1811 if (os_atomic_load(&stackshot_ctx.sc_retval, relaxed) == KERN_SUCCESS) { in stackshot_finalize_kcdata()
1813 kcd_exit_on_error(kcdata_finish(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_kcdata()
1816 …memcpy(stackshot_args.buffer, (void*) stackshot_ctx.sc_finalized_kcdata->kcd_addr_begin, stackshot… in stackshot_finalize_kcdata()
1820 …ackshot_args.buffer + ((mach_vm_address_t) stackshot_duration_outer - stackshot_ctx.sc_finalized_k… in stackshot_finalize_kcdata()
1839 assert(stackshot_ctx.sc_is_singlethreaded); in stackshot_finalize_singlethreaded_kcdata()
1841 kcd_exit_on_error(stackshot_push_duration_and_latency(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_singlethreaded_kcdata()
1843 kcd_finalize_compression(stackshot_ctx.sc_finalized_kcdata); in stackshot_finalize_singlethreaded_kcdata()
1844 …kcd_exit_on_error(kcdata_add_uint64_with_description(stackshot_ctx.sc_finalized_kcdata, stackshot_… in stackshot_finalize_singlethreaded_kcdata()
1845 kcd_exit_on_error(kcdata_write_buffer_end(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_singlethreaded_kcdata()
1847 …stackshot_ctx.sc_bytes_traced = (uint32_t) kcdata_memory_get_used_bytes(stackshot_ctx.sc_finalized… in stackshot_finalize_singlethreaded_kcdata()
1848 …stackshot_ctx.sc_bytes_uncompressed = (uint32_t) kcdata_memory_get_uncompressed_bytes(stackshot_ct… in stackshot_finalize_singlethreaded_kcdata()
1850 kcd_exit_on_error(kcdata_finish(stackshot_ctx.sc_finalized_kcdata)); in stackshot_finalize_singlethreaded_kcdata()
1852 if (stackshot_ctx.sc_panic_stackshot) { in stackshot_finalize_singlethreaded_kcdata()
1853 *stackshot_args.descriptor = *stackshot_ctx.sc_finalized_kcdata; in stackshot_finalize_singlethreaded_kcdata()
1969 kcdata_add_container_marker(stackshot_ctx.sc_finalized_kcdata, KCDATA_TYPE_CONTAINER_BEGIN, in collect_exclave_threads()
1985 kr = kcdata_add_container_marker(stackshot_ctx.sc_finalized_kcdata, KCDATA_TYPE_CONTAINER_END, in collect_exclave_threads()
2251 …kr = stackshot_exclaves_process_stackshot(result, stackshot_ctx.sc_finalized_kcdata, want_raw_addr… in stackshot_exclaves_process_result()
2529 os_atomic_store(&stackshot_ctx.sc_state, SS_INACTIVE, release); in kern_stack_snapshot_internal()
2539 if (error == KERN_SUCCESS && !stackshot_ctx.sc_is_singlethreaded) { in kern_stack_snapshot_internal()
2553 if (stackshot_ctx.sc_is_singlethreaded) { in kern_stack_snapshot_internal()
2687 stackshot_ctx = (struct stackshot_context) { in kdp_snapshot_preflight_internal()
2703 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_snapshot_preflight_internal()
2706 stackshot_ctx.sc_num_buffers = is_singlethreaded ? 1 : ml_get_cluster_count(); in kdp_snapshot_preflight_internal()
2708 stackshot_ctx.sc_num_buffers = 1; in kdp_snapshot_preflight_internal()
2710 size_t bufsz = args.buffer_size / stackshot_ctx.sc_num_buffers; in kdp_snapshot_preflight_internal()
2711 for (int buf_idx = 0; buf_idx < stackshot_ctx.sc_num_buffers; buf_idx++) { in kdp_snapshot_preflight_internal()
2712 stackshot_ctx.sc_buffers[buf_idx] = (struct stackshot_buffer) { in kdp_snapshot_preflight_internal()
2791 stackshot_ctx = (struct stackshot_context) { 0 }; in stackshot_reset_state()
2803 return os_atomic_load(&stackshot_ctx.sc_state, relaxed) != SS_INACTIVE; in stackshot_active()
2815 return stackshot_ctx.sc_bytes_traced; in kdp_stack_snapshot_bytes_traced()
2821 return stackshot_ctx.sc_bytes_uncompressed; in kdp_stack_snapshot_bytes_uncompressed()
2882 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in stackshot_plh_est_size()
2899 stackshot_ctx.sc_plh = (struct port_label_hash){.plh_size = 0}; /* structure assignment */ in stackshot_plh_reset()
2947 PLH_STAT_OP(os_atomic_inc(&stackshot_ctx.sc_plh.plh_bad, relaxed)); in stackshot_plh_setup()
2957 stackshot_ctx.sc_plh = plh; /* structure assignment */ in stackshot_plh_setup()
2987 uint16_t plh_size = stackshot_ctx.sc_plh.plh_size; in stackshot_plh_resetgen()
3006 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in stackshot_plh_lookup_locked()
3076 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in kdp_stackshot_plh_record_locked()
3132 plh_lock(&stackshot_ctx.sc_plh); in kdp_stackshot_plh_record()
3134 plh_unlock(&stackshot_ctx.sc_plh); in kdp_stackshot_plh_record()
3142 plh_lock(&stackshot_ctx.sc_plh); in stackshot_plh_lookup()
3144 plh_unlock(&stackshot_ctx.sc_plh); in stackshot_plh_lookup()
3153 struct port_label_hash *plh = &stackshot_ctx.sc_plh; in kdp_stackshot_plh_stats()
3898 assert(stackshot_ctx.sc_is_singlethreaded); in kcdata_record_task_snapshot()
4233 …laves_state & TH_EXCLAVES_RPC) && stackshot_exclave_inspect_ctids && !stackshot_ctx.sc_panic_stack… in kcdata_record_thread_snapshot()
4331 .sbc_allow_faulting = stackshot_ctx.sc_enable_faulting, in kcdata_record_thread_snapshot()
4611 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_record_task()
4915 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_record_task()
5048 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_kcdata_format()
5062 stackshot_ctx.sc_enable_faulting = (stackshot_flags & (STACKSHOT_ENABLE_BT_FAULTING)); in kdp_stackshot_kcdata_format()
5075 if (!stackshot_ctx.sc_panic_stackshot && collect_exclaves) { in kdp_stackshot_kcdata_format()
5086 if (stackshot_ctx.sc_bytes_uncompressed) { in kdp_stackshot_kcdata_format()
5087 stackshot_ctx.sc_bytes_uncompressed = 0; in kdp_stackshot_kcdata_format()
5096 if (!stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5098 stackshot_ctx.sc_pretask_kcdata = stackshot_cpu_ctx.scc_kcdata_head; in kdp_stackshot_kcdata_format()
5113 stackshot_ctx.sc_latency.setup_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5145 …stackshot_kcdata_p, KCDATA_TYPE_USECS_SINCE_EPOCH, sizeof(uint64_t), &stackshot_ctx.sc_microsecs)); in kdp_stackshot_kcdata_format()
5163 if (!stackshot_ctx.sc_is_singlethreaded && (stackshot_flags & STACKSHOT_THREAD_GROUP)) { in kdp_stackshot_kcdata_format()
5191 if (!stackshot_ctx.sc_panic_stackshot && (thread_group_begin_cpu_cycle_count != 0)) { in kdp_stackshot_kcdata_format()
5202 …stackshot_ctx.sc_latency.setup_latency_mt = mach_absolute_time() - stackshot_ctx.sc_latency.setup_… in kdp_stackshot_kcdata_format()
5203 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5204 stackshot_ctx.sc_latency.total_task_iteration_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5206 stackshot_ctx.sc_latency.task_queue_building_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5240 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5257 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5258 …stackshot_ctx.sc_latency.total_task_iteration_latency_mt = mach_absolute_time() - stackshot_ctx.sc… in kdp_stackshot_kcdata_format()
5260 …stackshot_ctx.sc_latency.task_queue_building_latency_mt = mach_absolute_time() - stackshot_ctx.sc_… in kdp_stackshot_kcdata_format()
5265 if (!stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5268 stackshot_ctx.sc_posttask_kcdata = stackshot_cpu_ctx.scc_kcdata_head; in kdp_stackshot_kcdata_format()
5280 if (!stackshot_ctx.sc_panic_stackshot && (stackshot_flags & STACKSHOT_SAVE_JETSAM_COALITIONS)) { in kdp_stackshot_kcdata_format()
5307 if (!stackshot_ctx.sc_panic_stackshot && (coalition_begin_cpu_cycle_count != 0)) { in kdp_stackshot_kcdata_format()
5320 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5321 stackshot_ctx.sc_latency.total_terminated_task_iteration_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5323 stackshot_ctx.sc_latency.terminated_task_queue_building_latency_mt = mach_absolute_time(); in kdp_stackshot_kcdata_format()
5346 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5359 os_atomic_store(&stackshot_ctx.sc_workqueues[i].sswq_populated, true, release); in kdp_stackshot_kcdata_format()
5367 if (stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5368 …stackshot_ctx.sc_latency.total_terminated_task_iteration_latency_mt = mach_absolute_time() - stack… in kdp_stackshot_kcdata_format()
5370 …stackshot_ctx.sc_latency.terminated_task_queue_building_latency_mt = mach_absolute_time() - stacks… in kdp_stackshot_kcdata_format()
5376 stackshot_ctx.sc_latency.latency_version = 2; in kdp_stackshot_kcdata_format()
5377 stackshot_ctx.sc_latency.main_cpu_number = stackshot_ctx.sc_main_cpuid; in kdp_stackshot_kcdata_format()
5378 stackshot_ctx.sc_latency.calling_cpu_number = stackshot_ctx.sc_calling_cpuid; in kdp_stackshot_kcdata_format()
5383 if (!stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_kcdata_format()
5394 if (!stackshot_ctx.sc_is_singlethreaded) { in kdp_stackshot_kcdata_format()
5403 if (stackshot_ctx.sc_panic_stackshot) { in kdp_stackshot_kcdata_format()
5516 if (!stackshot_ctx.sc_panic_stackshot) { in stackshot_find_phys()
5529 …_fault_stats.sfs_time_spent_faulting >= stackshot_max_fault_time) && !stackshot_ctx.sc_panic_stack… in stackshot_find_phys()
5591 if (!stackshot_ctx.sc_is_singlethreaded) { in do_stackshot()
5595 stackshot_ctx.sc_workqueues[i] = (struct stackshot_workqueue) { in do_stackshot()
5619 if (stackshot_ctx.sc_main_cpuid == -1) { in do_stackshot()
5620 os_atomic_cmpxchg(&stackshot_ctx.sc_main_cpuid, -1, cpu_number(), acquire); in do_stackshot()
5625 os_atomic_store(&stackshot_ctx.sc_state, SS_RUNNING, release); in do_stackshot()
5628 if (stackshot_ctx.sc_panic_stackshot || (stackshot_ctx.sc_main_cpuid == cpu_number())) { in do_stackshot()
5629 stackshot_ctx.sc_retval = kdp_stackshot_kcdata_format(); in do_stackshot()
5636 stackshot_ctx.sc_latency.cpu_wait_latency_mt = mach_absolute_time(); in do_stackshot()
5639 os_atomic_dec(&stackshot_ctx.sc_cpus_working, seq_cst); in do_stackshot()
5642 while (os_atomic_load(&stackshot_ctx.sc_cpus_working, seq_cst) != 0) { in do_stackshot()
5647 …stackshot_ctx.sc_latency.cpu_wait_latency_mt = mach_absolute_time() - stackshot_ctx.sc_latency.cpu… in do_stackshot()
5652 stackshot_ctx.sc_duration = (struct stackshot_duration_v2) { in do_stackshot()
5674 if (!stackshot_ctx.sc_panic_stackshot && interrupt_masked_debug_mode) { in do_stackshot()
5686 if (stackshot_ctx.sc_retval == KERN_SUCCESS) { in do_stackshot()
5687 stackshot_ctx.sc_retval = stackshot_status_check(); in do_stackshot()
5692 if (!stackshot_ctx.sc_panic_stackshot && stackshot_ctx.sc_retval == KERN_SUCCESS) { in do_stackshot()
5695 if (stackshot_ctx.sc_retval != KERN_SUCCESS && stackshot_exclave_inspect_ctids) { in do_stackshot()
5704 if (stackshot_ctx.sc_is_singlethreaded) { in do_stackshot()
5705 stackshot_ctx.sc_finalized_kcdata = stackshot_kcdata_p; in do_stackshot()
5708 return stackshot_ctx.sc_retval; in do_stackshot()
5762 os_atomic_cmpxchg(&stackshot_ctx.sc_main_cpuid, -1, my_cpu_no, acquire); in stackshot_cpu_preflight()
5765 is_calling_cpu = stackshot_ctx.sc_calling_cpuid == my_cpu_no; in stackshot_cpu_preflight()
5768 …stackshot_cpu_ctx.scc_can_work = is_calling_cpu || (is_recommended && !stackshot_ctx.sc_is_singlet… in stackshot_cpu_preflight()
5771 os_atomic_inc(&stackshot_ctx.sc_cpus_working, relaxed); in stackshot_cpu_preflight()
5833 kcd_exit_on_error(stackshot_cpu_work_on_queue(&stackshot_ctx.sc_workqueues[i - 1])); in stackshot_cpu_do_work()
5838 kcd_exit_on_error(stackshot_cpu_work_on_queue(&stackshot_ctx.sc_workqueues[i])); in stackshot_cpu_do_work()
5878 while (os_atomic_load(&stackshot_ctx.sc_state, acquire) == SS_SETUP) { in stackshot_aux_cpu_entry()
5890 if (stackshot_ctx.sc_main_cpuid == cpu_number()) { in stackshot_aux_cpu_entry()
5891 stackshot_ctx.sc_retval = kdp_stackshot_kcdata_format(); in stackshot_aux_cpu_entry()
5897 os_atomic_dec(&stackshot_ctx.sc_cpus_working, release); in stackshot_aux_cpu_entry()