1 /*
2 * Copyright (c) 2007-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <debug.h>
30 #include <mach_kdp.h>
31 #include <kern/kern_stackshot.h>
32
33 #include <kern/thread.h>
34 #include <machine/pmap.h>
35 #include <device/device_types.h>
36
37 #include <mach/vm_param.h>
38 #include <mach/clock_types.h>
39 #include <mach/machine.h>
40 #include <mach/kmod.h>
41 #include <pexpert/boot.h>
42 #include <pexpert/pexpert.h>
43
44 #include <ptrauth.h>
45
46 #include <kern/ecc.h>
47 #include <kern/misc_protos.h>
48 #include <kern/startup.h>
49 #include <kern/clock.h>
50 #include <kern/debug.h>
51 #include <kern/processor.h>
52 #include <kdp/kdp_core.h>
53 #if ALTERNATE_DEBUGGER
54 #include <arm64/alternate_debugger.h>
55 #endif
56 #include <machine/atomic.h>
57 #include <machine/trap.h>
58 #include <kern/spl.h>
59 #include <pexpert/pexpert.h>
60 #include <kdp/kdp_callout.h>
61 #include <kdp/kdp_dyld.h>
62 #include <kdp/kdp_internal.h>
63 #include <kdp/kdp_common.h>
64 #include <uuid/uuid.h>
65 #include <sys/codesign.h>
66 #include <sys/time.h>
67
68 #if CONFIG_SPTM
69 #include <kern/percpu.h>
70 #include <arm64/sptm/pmap/pmap_data.h>
71 #endif
72
73 #include <IOKit/IOPlatformExpert.h>
74 #include <IOKit/IOKitServer.h>
75
76 #include <mach/vm_prot.h>
77 #include <vm/vm_map_xnu.h>
78 #include <vm/pmap.h>
79 #include <vm/vm_shared_region.h>
80 #include <mach/time_value.h>
81 #include <machine/machparam.h> /* for btop */
82
83 #include <console/video_console.h>
84 #include <console/serial_protos.h>
85 #include <arm/cpu_data.h>
86 #include <arm/cpu_data_internal.h>
87 #include <arm/cpu_internal.h>
88 #include <arm/misc_protos.h>
89 #include <libkern/OSKextLibPrivate.h>
90 #include <vm/vm_kern.h>
91 #include <kern/kern_cdata.h>
92 #include <kern/ledger.h>
93 #include <san/kcov_ksancov.h>
94
95
96 #if DEVELOPMENT || DEBUG
97 #include <kern/ext_paniclog.h>
98 #endif
99
100 #if CONFIG_EXCLAVES
101 #include <kern/exclaves_panic.h>
102 #include <kern/exclaves_inspection.h>
103 #endif
104
105
106 #if MACH_KDP
107 void kdp_trap(unsigned int, struct arm_saved_state *);
108 #endif
109
110 /*
111 * Increment the PANICLOG_VERSION if you change the format of the panic
112 * log in any way.
113 */
114 #define PANICLOG_VERSION 15
115 static struct kcdata_descriptor kc_panic_data;
116
117 extern char iBoot_version[];
118 #if defined(TARGET_OS_OSX) && defined(__arm64__)
119 extern char iBoot_Stage_2_version[];
120 #endif /* defined(TARGET_OS_OSX) && defined(__arm64__) */
121
122 extern volatile uint32_t debug_enabled;
123 extern unsigned int not_in_kdp;
124
125 extern int copyinframe(vm_address_t fp, uint32_t * frame);
126 extern void kdp_callouts(kdp_event_t event);
127
128 #define MAX_PROCNAME_LEN 32
129 /* #include <sys/proc.h> */
130 struct proc;
131 extern int proc_pid(struct proc *p);
132 extern void proc_name_kdp(struct proc *, char *, int);
133
134 /*
135 * Make sure there's enough space to include the relevant bits in the format required
136 * within the space allocated for the panic version string in the panic header.
137 * The format required by OSAnalytics/DumpPanic is 'Product Version (OS Version)'.
138 */
139 #define PANIC_HEADER_VERSION_FMT_STR "%.14s (%.14s)"
140
141 extern const char version[];
142 extern char osversion[];
143 extern char osproductversion[];
144 extern char osreleasetype[];
145
146 #if defined(XNU_TARGET_OS_BRIDGE)
147 extern char macosproductversion[];
148 extern char macosversion[];
149 #endif
150
151 extern uint8_t gPlatformECID[8];
152 extern uint32_t gPlatformMemoryID;
153
154 extern uint64_t last_hwaccess_thread;
155 extern uint8_t last_hwaccess_type; /* 0 : read, 1 : write. */
156 extern uint8_t last_hwaccess_size;
157 extern uint64_t last_hwaccess_paddr;
158
159 /*Choosing the size for gTargetTypeBuffer as 16 and size for gModelTypeBuffer as 32
160 * since the target name and model name typically doesn't exceed this size */
161 extern char gTargetTypeBuffer[16];
162 extern char gModelTypeBuffer[32];
163
164 extern struct timeval gIOLastSleepTime;
165 extern struct timeval gIOLastWakeTime;
166 extern boolean_t is_clock_configured;
167 extern boolean_t kernelcache_uuid_valid;
168 extern uuid_t kernelcache_uuid;
169 extern uuid_string_t bootsessionuuid_string;
170
171 extern uint64_t roots_installed;
172
173 /* Definitions for frame pointers */
174 #define FP_ALIGNMENT_MASK ((uint32_t)(0x3))
175 #define FP_LR_OFFSET ((uint32_t)4)
176 #define FP_LR_OFFSET64 ((uint32_t)8)
177 #define FP_MAX_NUM_TO_EVALUATE (50)
178
179 /* Timeout for all processors responding to debug crosscall */
180 MACHINE_TIMEOUT_ALWAYS_ENABLED(debug_ack_timeout, "debug-ack", 240000, MACHINE_TIMEOUT_UNIT_TIMEBASE);
181
182 /* Forward functions definitions */
183 void panic_display_times(void);
184 void panic_print_symbol_name(vm_address_t search);
185
186
187 /* Global variables */
188 static uint32_t panic_bt_depth;
189 boolean_t PanicInfoSaved = FALSE;
190 boolean_t force_immediate_debug_halt = FALSE;
191 unsigned int debug_ack_timeout_count = 0;
192 _Atomic unsigned int debugger_sync = 0;
193 _Atomic unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */
194 _Atomic unsigned int debug_cpus_spinning = 0; /* Number of signalled CPUs still spinning on mp_kdp_trap (in DebuggerXCall). */
195 unsigned int DebugContextCount = 0;
196 bool trap_is_stackshot = false; /* Whether the trap is for a stackshot */
197
198 #if defined(__arm64__)
199 uint8_t PE_smc_stashed_x86_system_state = 0xFF;
200 uint8_t PE_smc_stashed_x86_power_state = 0xFF;
201 uint8_t PE_smc_stashed_x86_efi_boot_state = 0xFF;
202 uint8_t PE_smc_stashed_x86_shutdown_cause = 0xFF;
203 uint64_t PE_smc_stashed_x86_prev_power_transitions = UINT64_MAX;
204 uint32_t PE_pcie_stashed_link_state = UINT32_MAX;
205 uint64_t PE_nvram_stashed_x86_macos_slide = UINT64_MAX;
206 #endif
207
208
209 static void
do_print_backtrace_internal(pmap_t pmap,vm_offset_t topfp,const char * cur_marker,boolean_t is_64_bit,boolean_t print_kexts_in_backtrace)210 do_print_backtrace_internal(pmap_t pmap, vm_offset_t topfp, const char *cur_marker,
211 boolean_t is_64_bit, boolean_t print_kexts_in_backtrace)
212 {
213 unsigned int i = 0;
214 addr64_t lr = 0;
215 addr64_t fp = topfp;
216 addr64_t fp_for_ppn = 0;
217 ppnum_t ppn = (ppnum_t)NULL;
218 vm_offset_t raddrs[FP_MAX_NUM_TO_EVALUATE] = { 0 };
219 bool dump_kernel_stack = (fp >= VM_MIN_KERNEL_ADDRESS);
220
221 do {
222 if ((fp == 0) || ((fp & FP_ALIGNMENT_MASK) != 0)) {
223 break;
224 }
225
226 if ((!dump_kernel_stack) && (fp >= VM_MIN_KERNEL_ADDRESS)) {
227 break;
228 }
229
230 /*
231 * Check to see if current address will result in a different
232 * ppn than previously computed (to avoid recomputation) via
233 * (addr) ^ fp_for_ppn) >> PAGE_SHIFT)
234 */
235 if ((((fp + FP_LR_OFFSET) ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) {
236 ppn = pmap_find_phys(pmap, fp + FP_LR_OFFSET);
237 fp_for_ppn = fp + (is_64_bit ? FP_LR_OFFSET64 : FP_LR_OFFSET);
238 }
239 if (ppn != (ppnum_t)NULL) {
240 if (is_64_bit) {
241 lr = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET64) & PAGE_MASK));
242 #if defined(HAS_APPLE_PAC)
243 /* return addresses on stack will be signed by arm64e ABI */
244 lr = (addr64_t) ptrauth_strip((void *)lr, ptrauth_key_return_address);
245 #endif
246 } else {
247 lr = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET) & PAGE_MASK));
248 }
249 } else {
250 if (is_64_bit) {
251 paniclog_append_noflush("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64);
252 } else {
253 paniclog_append_noflush("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET));
254 }
255 break;
256 }
257 if (((fp ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) {
258 ppn = pmap_find_phys(pmap, fp);
259 fp_for_ppn = fp;
260 }
261 if (ppn != (ppnum_t)NULL) {
262 if (is_64_bit) {
263 fp = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK));
264 #if defined(HAS_APPLE_PAC)
265 /* frame pointers on stack will be signed by arm64e ABI */
266 fp = (addr64_t) ptrauth_strip((void *)fp, ptrauth_key_frame_pointer);
267 #endif
268 } else {
269 fp = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK));
270 }
271 } else {
272 if (is_64_bit) {
273 paniclog_append_noflush("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp);
274 } else {
275 paniclog_append_noflush("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp);
276 }
277 break;
278 }
279 /*
280 * Counter 'i' may == FP_MAX_NUM_TO_EVALUATE when running one
281 * extra round to check whether we have all frames in order to
282 * indicate (in)complete backtrace below. This happens in a case
283 * where total frame count and FP_MAX_NUM_TO_EVALUATE are equal.
284 * Do not capture anything.
285 */
286 if (i < FP_MAX_NUM_TO_EVALUATE && lr) {
287 if (is_64_bit) {
288 paniclog_append_noflush("%s\t lr: 0x%016llx fp: 0x%016llx\n", cur_marker, lr, fp);
289 } else {
290 paniclog_append_noflush("%s\t lr: 0x%08x fp: 0x%08x\n", cur_marker, (uint32_t)lr, (uint32_t)fp);
291 }
292 raddrs[i] = lr;
293 }
294 } while ((++i <= FP_MAX_NUM_TO_EVALUATE) && (fp != topfp));
295
296 if (i > FP_MAX_NUM_TO_EVALUATE && fp != 0) {
297 paniclog_append_noflush("Backtrace continues...\n");
298 }
299
300 if (print_kexts_in_backtrace && i > 0) {
301 kmod_panic_dump(&raddrs[0], i);
302 }
303 }
304
305 #define SANE_TASK_LIMIT 256
306 #define TOP_RUNNABLE_LIMIT 5
307 #define PANICLOG_UUID_BUF_SIZE 256
308
309 extern void panic_print_vnodes(void);
310
311 static void
panic_display_tpidrs(void)312 panic_display_tpidrs(void)
313 {
314 #if defined(__arm64__)
315 paniclog_append_noflush("TPIDRx_ELy = {1: 0x%016llx 0: 0x%016llx 0ro: 0x%016llx }\n",
316 __builtin_arm_rsr64("TPIDR_EL1"), __builtin_arm_rsr64("TPIDR_EL0"),
317 __builtin_arm_rsr64("TPIDRRO_EL0"));
318 #endif //defined(__arm64__)
319 }
320
321
322
323 static void
panic_display_hung_cpus_help(void)324 panic_display_hung_cpus_help(void)
325 {
326 #if defined(__arm64__)
327 const uint32_t pcsr_offset = 0x90;
328
329 /*
330 * Print some info that might help in cases where nothing
331 * else does
332 */
333 const ml_topology_info_t *info = ml_get_topology_info();
334 if (info) {
335 unsigned i, retry;
336
337 for (i = 0; i < info->num_cpus; i++) {
338 ml_topology_cpu_t *cpu = &info->cpus[i];
339 char cluster_name[16], cluster_letter;
340
341 switch (cpu->cluster_type) {
342 case CLUSTER_TYPE_E:
343 cluster_letter = 'E';
344 break;
345 case CLUSTER_TYPE_P:
346 cluster_letter = 'P';
347 break;
348 default:
349 cluster_letter = '?';
350 }
351 snprintf(cluster_name, sizeof(cluster_name), "%cACC%d", cluster_letter, cpu->cluster_id);
352
353 if (!PE_cpu_power_check_kdp(i)) {
354 paniclog_append_noflush("CORE %u [%s] is offline, skipping\n", i, cluster_name);
355 continue;
356 }
357 if (cpu->cpu_UTTDBG_regs) {
358 volatile uint64_t *pcsr = (volatile uint64_t*)(cpu->cpu_UTTDBG_regs + pcsr_offset);
359 volatile uint32_t *pcsrTrigger = (volatile uint32_t*)pcsr;
360 uint64_t pc = 0;
361
362 // a number of retries are needed till this works
363 for (retry = 1024; retry && !pc; retry--) {
364 //a 32-bit read is required to make a PC sample be produced, else we'll only get a zero
365 (void)*pcsrTrigger;
366 pc = *pcsr;
367 }
368
369 //postprocessing (same as astris does)
370 if (pc >> 48) {
371 pc |= 0xffff000000000000ull;
372 }
373 paniclog_append_noflush("CORE %u [%s] recently retired instr at 0x%016llx\n", i, cluster_name, pc);
374 }
375 }
376 }
377 #endif //defined(__arm64__)
378 }
379
380
381
382 static void
panic_display_pvhs_locked(void)383 panic_display_pvhs_locked(void)
384 {
385 }
386
387 static void
panic_display_pvh_to_lock(void)388 panic_display_pvh_to_lock(void)
389 {
390 }
391
392 static void
panic_display_last_pc_lr(void)393 panic_display_last_pc_lr(void)
394 {
395 #if defined(__arm64__)
396 const int max_cpu = ml_get_max_cpu_number();
397
398 for (int cpu = 0; cpu <= max_cpu; cpu++) {
399 cpu_data_t *current_cpu_datap = cpu_datap(cpu);
400
401 if (current_cpu_datap == NULL) {
402 continue;
403 }
404
405 if (current_cpu_datap == getCpuDatap()) {
406 /**
407 * Skip printing the PC/LR if this is the CPU
408 * that initiated the panic.
409 */
410 paniclog_append_noflush("CORE %u is the one that panicked. Check the full backtrace for details.\n", cpu);
411 continue;
412 }
413
414 paniclog_append_noflush("CORE %u: PC=0x%016llx, LR=0x%016llx, FP=0x%016llx\n", cpu,
415 current_cpu_datap->ipi_pc, (uint64_t)VM_KERNEL_STRIP_PTR(current_cpu_datap->ipi_lr),
416 (uint64_t)VM_KERNEL_STRIP_PTR(current_cpu_datap->ipi_fp));
417 }
418 #endif
419 }
420
421 #if CONFIG_EXCLAVES
422 static void
panic_report_exclaves_stackshot(void)423 panic_report_exclaves_stackshot(void)
424 {
425 if (exclaves_panic_ss_status == EXCLAVES_PANIC_STACKSHOT_FOUND) {
426 paniclog_append_noflush("** Exclaves panic stackshot found\n");
427 } else if (exclaves_panic_ss_status == EXCLAVES_PANIC_STACKSHOT_NOT_FOUND) {
428 paniclog_append_noflush("** Exclaves panic stackshot not found\n");
429 } else if (exclaves_panic_ss_status == EXCLAVES_PANIC_STACKSHOT_DECODE_FAILED) {
430 paniclog_append_noflush("!! Exclaves panic stackshot decode failed !!\n");
431 }
432 }
433 #endif /* CONFIG_EXCLAVES */
434
435 __attribute__((always_inline))
436 static inline void
print_backtrace_internal(thread_t thread,bool filesetKC)437 print_backtrace_internal(thread_t thread, bool filesetKC)
438 {
439 uintptr_t cur_fp = (uintptr_t)__builtin_frame_address(0);
440 const char *nohilite_thread_marker = "\t";
441
442 #if defined(HAS_APPLE_PAC)
443 cur_fp = (addr64_t)ptrauth_strip((void *)cur_fp, ptrauth_key_frame_pointer);
444 #endif
445
446 if (cur_fp < VM_MAX_KERNEL_ADDRESS) {
447 paniclog_append_noflush("Panicked thread: %p, backtrace: 0x%llx, tid: %llu\n",
448 thread, (addr64_t)cur_fp, thread_tid(thread));
449 #if __LP64__
450 do_print_backtrace_internal(kernel_pmap, cur_fp, nohilite_thread_marker, TRUE, filesetKC);
451 #else
452 do_print_backtrace_internal(kernel_pmap, cur_fp, nohilite_thread_marker, FALSE, filesetKC);
453 #endif
454 } else {
455 paniclog_append_noflush("Could not print panicked thread backtrace:"
456 "frame pointer outside kernel vm.\n");
457 }
458 }
459
460 static bool
is_filesetKC(void)461 is_filesetKC(void)
462 {
463 kc_format_t kc_format;
464 bool filesetKC = false;
465
466 __unused bool result = PE_get_primary_kc_format(&kc_format);
467 assert(result == true);
468 filesetKC = kc_format == KCFormatFileset;
469 return filesetKC;
470 }
471
472
473 static void
do_print_all_panic_info(const char * message,uint64_t panic_options,const char * panic_initiator)474 do_print_all_panic_info(const char *message, uint64_t panic_options, const char *panic_initiator)
475 {
476 int logversion = PANICLOG_VERSION;
477 thread_t cur_thread = current_thread();
478 task_t task;
479 struct proc *proc;
480 int print_vnodes = 0;
481
482 /* end_marker_bytes set to 200 for printing END marker + stackshot summary info always */
483 int bytes_traced = 0, bytes_remaining = 0, end_marker_bytes = 200;
484 int bytes_uncompressed = 0;
485 uint64_t bytes_used = 0ULL;
486 int err = 0;
487 char *stackshot_begin_loc = NULL;
488 bool filesetKC = is_filesetKC();
489 uint32_t panic_initiator_len = 0;
490 #if CONFIG_EXT_PANICLOG
491 uint32_t ext_paniclog_bytes = 0;
492 #endif
493
494 if (panic_bt_depth != 0) {
495 return;
496 }
497 panic_bt_depth++;
498
499 /* Truncate panic string to 1200 bytes */
500 paniclog_append_noflush("Debugger message: %.1200s\n", message);
501 if (debug_enabled) {
502 paniclog_append_noflush("Device: %s\n",
503 ('\0' != gTargetTypeBuffer[0]) ? gTargetTypeBuffer : "Not set yet");
504 paniclog_append_noflush("Hardware Model: %s\n",
505 ('\0' != gModelTypeBuffer[0]) ? gModelTypeBuffer:"Not set yet");
506 paniclog_append_noflush("ECID: %02X%02X%02X%02X%02X%02X%02X%02X\n", gPlatformECID[7],
507 gPlatformECID[6], gPlatformECID[5], gPlatformECID[4], gPlatformECID[3],
508 gPlatformECID[2], gPlatformECID[1], gPlatformECID[0]);
509 if (last_hwaccess_thread) {
510 paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread);
511 if (!last_hwaccess_size) {
512 paniclog_append_noflush("AppleHWAccess last access: no access data, this is unexpected.\n");
513 } else {
514 const char *typ = last_hwaccess_type ? "write" : "read";
515 paniclog_append_noflush("AppleHWAccess last access: %s of size %u at address 0x%llx\n", typ, last_hwaccess_size, last_hwaccess_paddr);
516 }
517 }
518 paniclog_append_noflush("Boot args: %s\n", PE_boot_args());
519 }
520 paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID);
521 paniclog_append_noflush("OS release type: %.256s\n",
522 ('\0' != osreleasetype[0]) ? osreleasetype : "Not set yet");
523 paniclog_append_noflush("OS version: %.256s\n",
524 ('\0' != osversion[0]) ? osversion : "Not set yet");
525 #if defined(XNU_TARGET_OS_BRIDGE)
526 paniclog_append_noflush("macOS version: %.256s\n",
527 ('\0' != macosversion[0]) ? macosversion : "Not set");
528 #endif
529 paniclog_append_noflush("Kernel version: %.512s\n", version);
530
531 #if CONFIG_EXCLAVES
532 exclaves_panic_append_info();
533 #endif
534
535 if (kernelcache_uuid_valid) {
536 if (filesetKC) {
537 paniclog_append_noflush("Fileset Kernelcache UUID: ");
538 } else {
539 paniclog_append_noflush("KernelCache UUID: ");
540 }
541 for (size_t index = 0; index < sizeof(uuid_t); index++) {
542 paniclog_append_noflush("%02X", kernelcache_uuid[index]);
543 }
544 paniclog_append_noflush("\n");
545 }
546 panic_display_kernel_uuid();
547
548 if (bootsessionuuid_string[0] != '\0') {
549 paniclog_append_noflush("Boot session UUID: %s\n", bootsessionuuid_string);
550 } else {
551 paniclog_append_noflush("Boot session UUID not yet initialized\n");
552 }
553
554 paniclog_append_noflush("iBoot version: %.128s\n", iBoot_version);
555 #if defined(TARGET_OS_OSX) && defined(__arm64__)
556 paniclog_append_noflush("iBoot Stage 2 version: %.128s\n", iBoot_Stage_2_version);
557 #endif /* defined(TARGET_OS_OSX) && defined(__arm64__) */
558
559 paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES");
560 paniclog_append_noflush("roots installed: %lld\n", roots_installed);
561 #if defined(XNU_TARGET_OS_BRIDGE)
562 paniclog_append_noflush("x86 EFI Boot State: ");
563 if (PE_smc_stashed_x86_efi_boot_state != 0xFF) {
564 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_efi_boot_state);
565 } else {
566 paniclog_append_noflush("not available\n");
567 }
568 paniclog_append_noflush("x86 System State: ");
569 if (PE_smc_stashed_x86_system_state != 0xFF) {
570 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_system_state);
571 } else {
572 paniclog_append_noflush("not available\n");
573 }
574 paniclog_append_noflush("x86 Power State: ");
575 if (PE_smc_stashed_x86_power_state != 0xFF) {
576 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_power_state);
577 } else {
578 paniclog_append_noflush("not available\n");
579 }
580 paniclog_append_noflush("x86 Shutdown Cause: ");
581 if (PE_smc_stashed_x86_shutdown_cause != 0xFF) {
582 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_shutdown_cause);
583 } else {
584 paniclog_append_noflush("not available\n");
585 }
586 paniclog_append_noflush("x86 Previous Power Transitions: ");
587 if (PE_smc_stashed_x86_prev_power_transitions != UINT64_MAX) {
588 paniclog_append_noflush("0x%llx\n", PE_smc_stashed_x86_prev_power_transitions);
589 } else {
590 paniclog_append_noflush("not available\n");
591 }
592 paniclog_append_noflush("PCIeUp link state: ");
593 if (PE_pcie_stashed_link_state != UINT32_MAX) {
594 paniclog_append_noflush("0x%x\n", PE_pcie_stashed_link_state);
595 } else {
596 paniclog_append_noflush("not available\n");
597 }
598 paniclog_append_noflush("macOS kernel slide: ");
599 if (PE_nvram_stashed_x86_macos_slide != UINT64_MAX) {
600 paniclog_append_noflush("%#llx\n", PE_nvram_stashed_x86_macos_slide);
601 } else {
602 paniclog_append_noflush("not available\n");
603 }
604 #endif
605 if (panic_data_buffers != NULL) {
606 paniclog_append_noflush("%s data: ", panic_data_buffers->producer_name);
607 uint8_t *panic_buffer_data = (uint8_t *) panic_data_buffers->buf;
608 for (int i = 0; i < panic_data_buffers->len; i++) {
609 paniclog_append_noflush("%02X", panic_buffer_data[i]);
610 }
611 paniclog_append_noflush("\n");
612 }
613 paniclog_append_noflush("Paniclog version: %d\n", logversion);
614
615 panic_display_kernel_aslr();
616 panic_display_times();
617 panic_display_zalloc();
618 panic_display_hung_cpus_help();
619 panic_display_tpidrs();
620
621
622 panic_display_pvhs_locked();
623 panic_display_pvh_to_lock();
624 panic_display_last_pc_lr();
625 #if CONFIG_ECC_LOGGING
626 panic_display_ecc_errors();
627 #endif /* CONFIG_ECC_LOGGING */
628 panic_display_compressor_stats();
629
630 #if DEVELOPMENT || DEBUG
631 if (cs_debug_unsigned_exec_failures != 0 || cs_debug_unsigned_mmap_failures != 0) {
632 paniclog_append_noflush("Unsigned code exec failures: %u\n", cs_debug_unsigned_exec_failures);
633 paniclog_append_noflush("Unsigned code mmap failures: %u\n", cs_debug_unsigned_mmap_failures);
634 }
635 #endif
636
637 // Highlight threads that used high amounts of CPU in the panic log if requested (historically requested for watchdog panics)
638 if (panic_options & DEBUGGER_OPTION_PRINT_CPU_USAGE_PANICLOG) {
639 thread_t top_runnable[5] = {0};
640 thread_t thread;
641 int total_cpu_usage = 0;
642
643 print_vnodes = 1;
644
645
646 for (thread = (thread_t)queue_first(&threads);
647 PANIC_VALIDATE_PTR(thread) && !queue_end(&threads, (queue_entry_t)thread);
648 thread = (thread_t)queue_next(&thread->threads)) {
649 total_cpu_usage += thread->cpu_usage;
650
651 // Look for the 5 runnable threads with highest priority
652 if (thread->state & TH_RUN) {
653 int k;
654 thread_t comparison_thread = thread;
655
656 for (k = 0; k < TOP_RUNNABLE_LIMIT; k++) {
657 if (top_runnable[k] == 0) {
658 top_runnable[k] = comparison_thread;
659 break;
660 } else if (comparison_thread->sched_pri > top_runnable[k]->sched_pri) {
661 thread_t temp = top_runnable[k];
662 top_runnable[k] = comparison_thread;
663 comparison_thread = temp;
664 } // if comparison thread has higher priority than previously saved thread
665 } // loop through highest priority runnable threads
666 } // Check if thread is runnable
667 } // Loop through all threads
668
669 // Print the relevant info for each thread identified
670 paniclog_append_noflush("Total cpu_usage: %d\n", total_cpu_usage);
671 paniclog_append_noflush("Thread task pri cpu_usage\n");
672
673 for (int i = 0; i < TOP_RUNNABLE_LIMIT; i++) {
674 if (top_runnable[i] &&
675 panic_get_thread_proc_task(top_runnable[i], &task, &proc) && proc) {
676 char name[MAX_PROCNAME_LEN + 1];
677 proc_name_kdp(proc, name, sizeof(name));
678 paniclog_append_noflush("%p %s %d %d\n",
679 top_runnable[i], name, top_runnable[i]->sched_pri, top_runnable[i]->cpu_usage);
680 }
681 } // Loop through highest priority runnable threads
682 paniclog_append_noflush("\n");
683 }
684
685 // print current task info
686 if (panic_get_thread_proc_task(cur_thread, &task, &proc)) {
687 if (PANIC_VALIDATE_PTR(task->map) &&
688 PANIC_VALIDATE_PTR(task->map->pmap)) {
689 ledger_amount_t resident = 0;
690 if (task != kernel_task) {
691 ledger_get_balance(task->ledger, task_ledgers.phys_mem, &resident);
692 resident >>= VM_MAP_PAGE_SHIFT(task->map);
693 }
694 paniclog_append_noflush("Panicked task %p: %lld pages, %d threads: ",
695 task, resident, task->thread_count);
696 } else {
697 paniclog_append_noflush("Panicked task %p: %d threads: ",
698 task, task->thread_count);
699 }
700
701 if (proc) {
702 char name[MAX_PROCNAME_LEN + 1];
703 proc_name_kdp(proc, name, sizeof(name));
704 paniclog_append_noflush("pid %d: %s", proc_pid(proc), name);
705 } else {
706 paniclog_append_noflush("unknown task");
707 }
708
709 paniclog_append_noflush("\n");
710 }
711
712 print_backtrace_internal(cur_thread, filesetKC);
713
714 paniclog_append_noflush("\n");
715 dump_cpu_event_log(&paniclog_append_noflush);
716
717 paniclog_append_noflush("\n");
718 if (filesetKC) {
719 kext_dump_panic_lists(&paniclog_append_noflush);
720 paniclog_append_noflush("\n");
721 }
722 panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_panic_log_offset;
723 /* set the os version data in the panic header in the format 'Product Version (OS Version)' (only if they have been set) */
724 if ((osversion[0] != '\0') && (osproductversion[0] != '\0')) {
725 snprintf((char *)&panic_info->eph_os_version, sizeof(panic_info->eph_os_version), PANIC_HEADER_VERSION_FMT_STR,
726 osproductversion, osversion);
727 }
728 #if defined(XNU_TARGET_OS_BRIDGE)
729 if ((macosversion[0] != '\0') && (macosproductversion[0] != '\0')) {
730 snprintf((char *)&panic_info->eph_macos_version, sizeof(panic_info->eph_macos_version), PANIC_HEADER_VERSION_FMT_STR,
731 macosproductversion, macosversion);
732 }
733 #endif
734 if (bootsessionuuid_string[0] != '\0') {
735 memcpy(panic_info->eph_bootsessionuuid_string, bootsessionuuid_string,
736 sizeof(panic_info->eph_bootsessionuuid_string));
737 }
738 panic_info->eph_roots_installed = roots_installed;
739
740 if (panic_initiator != NULL) {
741 bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)debug_buf_ptr - (uintptr_t)debug_buf_base);
742 // If panic_initiator isn't null, safely copy up to MAX_PANIC_INITIATOR_SIZE
743 panic_initiator_len = strnlen(panic_initiator, MAX_PANIC_INITIATOR_SIZE);
744 // Calculate the bytes to write, accounting for remaining buffer space, and ensuring the lowest size we can have is 0
745 panic_initiator_len = MAX(0, MIN(panic_initiator_len, bytes_remaining));
746 panic_info->eph_panic_initiator_offset = (panic_initiator_len != 0) ? PE_get_offset_into_panic_region(debug_buf_ptr) : 0;
747 panic_info->eph_panic_initiator_len = panic_initiator_len;
748 memcpy(debug_buf_ptr, panic_initiator, panic_initiator_len);
749 debug_buf_ptr += panic_initiator_len;
750 }
751
752 if (debug_ack_timeout_count) {
753 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_DEBUGGERSYNC;
754 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
755 paniclog_append_noflush("!! debugger synchronization failed, no stackshot !!\n");
756 } else if (panic_stackshot_active()) {
757 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_NESTED;
758 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
759 paniclog_append_noflush("!! panicked during stackshot, skipping panic stackshot !!\n");
760 } else {
761 /* Align the stackshot buffer to an 8-byte address (especially important for armv7k devices) */
762 debug_buf_ptr += (8 - ((uintptr_t)debug_buf_ptr % 8));
763 stackshot_begin_loc = debug_buf_ptr;
764
765 bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)stackshot_begin_loc - (uintptr_t)debug_buf_base);
766 err = kcdata_memory_static_init(&kc_panic_data, (mach_vm_address_t)debug_buf_ptr,
767 KCDATA_BUFFER_BEGIN_COMPRESSED, bytes_remaining - end_marker_bytes,
768 KCFLAG_USE_MEMCOPY);
769 if (err == KERN_SUCCESS) {
770 uint64_t stackshot_flags = (STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_LOADINFO | STACKSHOT_KCDATA_FORMAT |
771 STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_FROM_PANIC | STACKSHOT_DO_COMPRESS |
772 STACKSHOT_DISABLE_LATENCY_INFO | STACKSHOT_NO_IO_STATS | STACKSHOT_THREAD_WAITINFO | STACKSHOT_GET_DQ |
773 STACKSHOT_COLLECT_SHAREDCACHE_LAYOUT);
774
775 err = kcdata_init_compress(&kc_panic_data, KCDATA_BUFFER_BEGIN_STACKSHOT, kdp_memcpy, KCDCT_ZLIB);
776 if (err != KERN_SUCCESS) {
777 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COMPRESS_FAILED;
778 stackshot_flags &= ~STACKSHOT_DO_COMPRESS;
779 }
780 if (filesetKC) {
781 stackshot_flags |= STACKSHOT_SAVE_KEXT_LOADINFO;
782 }
783
784 kdp_snapshot_preflight(-1, stackshot_begin_loc, bytes_remaining - end_marker_bytes,
785 stackshot_flags, &kc_panic_data, 0, 0);
786 err = do_panic_stackshot(NULL);
787 bytes_traced = kdp_stack_snapshot_bytes_traced();
788 if (bytes_traced > 0 && !err) {
789 debug_buf_ptr += bytes_traced;
790 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_SUCCEEDED;
791 panic_info->eph_stackshot_offset = PE_get_offset_into_panic_region(stackshot_begin_loc);
792 panic_info->eph_stackshot_len = bytes_traced;
793
794 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
795 #if CONFIG_EXCLAVES
796 panic_report_exclaves_stackshot();
797 #endif /* CONFIG_EXCLAVES */
798 if (stackshot_flags & STACKSHOT_DO_COMPRESS) {
799 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_DATA_COMPRESSED;
800 bytes_uncompressed = kdp_stack_snapshot_bytes_uncompressed();
801 paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d (Uncompressed %d) **\n", bytes_traced, bytes_uncompressed);
802 } else {
803 paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d **\n", bytes_traced);
804 }
805 } else {
806 bytes_used = kcdata_memory_get_used_bytes(&kc_panic_data);
807 #if CONFIG_EXCLAVES
808 panic_report_exclaves_stackshot();
809 #endif /* CONFIG_EXCLAVES */
810 if (bytes_used > 0) {
811 /* Zero out the stackshot data */
812 bzero(stackshot_begin_loc, bytes_used);
813 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_INCOMPLETE;
814
815 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
816 paniclog_append_noflush("\n** Stackshot Incomplete ** Bytes Filled %llu, err %d **\n", bytes_used, err);
817 } else {
818 bzero(stackshot_begin_loc, bytes_used);
819 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR;
820
821 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
822 paniclog_append_noflush("\n!! Stackshot Failed !! Bytes Traced %d, err %d\n", bytes_traced, err);
823 }
824 }
825 } else {
826 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR;
827 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
828 paniclog_append_noflush("\n!! Stackshot Failed !!\nkcdata_memory_static_init returned %d", err);
829 }
830 }
831
832 ksancov_on_panic_log();
833
834 #if CONFIG_EXT_PANICLOG
835 // Write ext paniclog at the end of the paniclog region.
836 ext_paniclog_bytes = ext_paniclog_write_panicdata();
837 panic_info->eph_ext_paniclog_offset = (ext_paniclog_bytes != 0) ?
838 PE_get_offset_into_panic_region((debug_buf_base + debug_buf_size) - ext_paniclog_bytes) :
839 0;
840 panic_info->eph_ext_paniclog_len = ext_paniclog_bytes;
841 #endif
842
843 assert(panic_info->eph_other_log_offset != 0);
844
845 if (print_vnodes != 0) {
846 panic_print_vnodes();
847 }
848
849 panic_bt_depth--;
850 }
851
852 /*
853 * Entry to print_all_panic_info is serialized by the debugger lock
854 */
855 static void
print_all_panic_info(const char * message,uint64_t panic_options,const char * panic_initiator)856 print_all_panic_info(const char *message, uint64_t panic_options, const char *panic_initiator)
857 {
858 unsigned int initial_not_in_kdp = not_in_kdp;
859
860 cpu_data_t * cpu_data_ptr = getCpuDatap();
861
862 assert(cpu_data_ptr->PAB_active == FALSE);
863 cpu_data_ptr->PAB_active = TRUE;
864
865 /*
866 * Because print all backtraces uses the pmap routines, it needs to
867 * avoid taking pmap locks. Right now, this is conditionalized on
868 * not_in_kdp.
869 */
870 not_in_kdp = 0;
871 do_print_all_panic_info(message, panic_options, panic_initiator);
872
873 not_in_kdp = initial_not_in_kdp;
874
875 cpu_data_ptr->PAB_active = FALSE;
876 }
877
878 void
print_curr_backtrace(void)879 print_curr_backtrace(void)
880 {
881 print_backtrace_internal(current_thread(), is_filesetKC());
882 }
883
884 void
panic_display_times()885 panic_display_times()
886 {
887 if (kdp_clock_is_locked()) {
888 paniclog_append_noflush("Warning: clock is locked. Can't get time\n");
889 return;
890 }
891
892 extern lck_ticket_t clock_lock;
893 extern lck_grp_t clock_lock_grp;
894
895 if ((is_clock_configured) && (lck_ticket_lock_try(&clock_lock, &clock_lock_grp))) {
896 clock_sec_t secs, boot_secs;
897 clock_usec_t usecs, boot_usecs;
898
899 lck_ticket_unlock(&clock_lock);
900
901 clock_get_calendar_microtime(&secs, &usecs);
902 clock_get_boottime_microtime(&boot_secs, &boot_usecs);
903
904 paniclog_append_noflush("mach_absolute_time: 0x%llx\n", mach_absolute_time());
905 paniclog_append_noflush("Epoch Time: sec usec\n");
906 paniclog_append_noflush(" Boot : 0x%08x 0x%08x\n", (unsigned int)boot_secs, (unsigned int)boot_usecs);
907 paniclog_append_noflush(" Sleep : 0x%08x 0x%08x\n", (unsigned int)gIOLastSleepTime.tv_sec, (unsigned int)gIOLastSleepTime.tv_usec);
908 paniclog_append_noflush(" Wake : 0x%08x 0x%08x\n", (unsigned int)gIOLastWakeTime.tv_sec, (unsigned int)gIOLastWakeTime.tv_usec);
909 paniclog_append_noflush(" Calendar: 0x%08x 0x%08x\n\n", (unsigned int)secs, (unsigned int)usecs);
910 }
911 }
912
913 void
panic_print_symbol_name(vm_address_t search)914 panic_print_symbol_name(vm_address_t search)
915 {
916 #pragma unused(search)
917 // empty stub. Really only used on x86_64.
918 return;
919 }
920
921 void
SavePanicInfo(const char * message,__unused void * panic_data,uint64_t panic_options,const char * panic_initiator)922 SavePanicInfo(
923 const char *message, __unused void *panic_data, uint64_t panic_options, const char* panic_initiator)
924 {
925 /*
926 * This should be initialized by the time we get here, but
927 * if it is not, asserting about it will be of no use (it will
928 * come right back to here), so just loop right here and now.
929 * This prevents early-boot panics from becoming recursive and
930 * thus makes them easier to debug. If you attached to a device
931 * and see your PC here, look down a few frames to see your
932 * early-boot panic there.
933 */
934 while (!panic_info || panic_info->eph_panic_log_offset == 0) {
935 // rdar://87170225 (PanicHardening: audit panic code for naked spinloops)
936 // rdar://88094367 (Add test hooks for panic at different stages in XNU)
937 ;
938 }
939
940 if (panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
941 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_BUTTON_RESET_PANIC;
942 }
943
944 if (panic_options & DEBUGGER_OPTION_COMPANION_PROC_INITIATED_PANIC) {
945 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COMPANION_PROC_INITIATED_PANIC;
946 }
947
948 if (panic_options & DEBUGGER_OPTION_INTEGRATED_COPROC_INITIATED_PANIC) {
949 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_INTEGRATED_COPROC_INITIATED_PANIC;
950 }
951
952 if (panic_options & DEBUGGER_OPTION_USERSPACE_INITIATED_PANIC) {
953 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_USERSPACE_INITIATED_PANIC;
954 }
955
956 #if defined(XNU_TARGET_OS_BRIDGE)
957 panic_info->eph_x86_power_state = PE_smc_stashed_x86_power_state;
958 panic_info->eph_x86_efi_boot_state = PE_smc_stashed_x86_efi_boot_state;
959 panic_info->eph_x86_system_state = PE_smc_stashed_x86_system_state;
960 #endif
961
962 /*
963 * On newer targets, panic data is stored directly into the iBoot panic region.
964 * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the
965 * panic CRC so that iBoot can hopefully find *something* useful in the panic region.
966 */
967 if (PanicInfoSaved && (debug_buf_base >= (char*)gPanicBase) && (debug_buf_base < (char*)gPanicBase + gPanicSize)) {
968 unsigned int pi_size = (unsigned int)(debug_buf_ptr - gPanicBase);
969 PE_update_panic_crc((unsigned char*)gPanicBase, &pi_size);
970 PE_sync_panic_buffers(); // extra precaution; panic path likely isn't reliable if we're here
971 }
972
973 if (PanicInfoSaved || (debug_buf_size == 0)) {
974 return;
975 }
976
977 PanicInfoSaved = TRUE;
978
979
980 print_all_panic_info(message, panic_options, panic_initiator);
981
982 assert(panic_info->eph_panic_log_len != 0);
983 panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset;
984
985 PEHaltRestart(kPEPanicSync);
986
987 /*
988 * Notifies registered IOPlatformPanicAction callbacks
989 * (which includes one to disable the memcache) and flushes
990 * the buffer contents from the cache
991 */
992 paniclog_flush();
993 }
994
995 void
paniclog_flush()996 paniclog_flush()
997 {
998 unsigned int panicbuf_length = 0;
999
1000 panicbuf_length = (unsigned int)(debug_buf_ptr - gPanicBase);
1001 if (!debug_buf_ptr || !panicbuf_length) {
1002 return;
1003 }
1004
1005 /*
1006 * Updates the log length of the last part of the panic log.
1007 */
1008 panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset;
1009
1010 /*
1011 * Updates the metadata at the beginning of the panic buffer,
1012 * updates the CRC.
1013 */
1014 PE_update_panic_crc((unsigned char *)gPanicBase, &panicbuf_length);
1015
1016 /*
1017 * This is currently unused by platform KEXTs on embedded but is
1018 * kept for compatibility with the published IOKit interfaces.
1019 */
1020 PESavePanicInfo((unsigned char *)gPanicBase, panicbuf_length);
1021
1022 PE_sync_panic_buffers();
1023 }
1024
1025 #if CONFIG_SPTM
1026 /*
1027 * Patch thread state to appear as if a debugger stop IPI occurred, when a thread
1028 * is parked in SPTM panic loop. This allows stackshot to proceed as usual.
1029 */
1030 static void
DebuggerPatchupThreadState(int cpu,xnu_saved_registers_t * regp)1031 DebuggerPatchupThreadState(
1032 int cpu, xnu_saved_registers_t *regp)
1033 {
1034 cpu_data_t *target_cpu_datap;
1035 arm_saved_state_t *statep;
1036 vm_offset_t kstackptr;
1037
1038 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1039 statep = target_cpu_datap->cpu_active_thread->machine.kpcb;
1040 kstackptr = (vm_offset_t)target_cpu_datap->cpu_active_thread->machine.kstackptr;
1041
1042 target_cpu_datap->ipi_pc = regp->pc;
1043 target_cpu_datap->ipi_lr = regp->lr;
1044 target_cpu_datap->ipi_fp = regp->fp;
1045
1046 if (statep != NULL) {
1047 statep->ss_64.fp = regp->fp;
1048 statep->ss_64.lr = regp->lr;
1049 statep->ss_64.sp = regp->sp;
1050 statep->ss_64.pc = regp->pc;
1051 } else if ((void *)kstackptr != NULL) {
1052 arm_kernel_saved_state_t *kstatep = (arm_kernel_saved_state_t *)kstackptr;
1053 kstatep->fp = regp->fp;
1054 kstatep->lr = regp->lr;
1055 kstatep->sp = regp->sp;
1056 }
1057 }
1058 #endif
1059
1060 /*
1061 * @function DebuggerXCallEnter
1062 *
1063 * @abstract IPI other cores so this core can run in a single-threaded context.
1064 *
1065 * @discussion This function should be called with the debugger lock held. It
1066 * signals the other cores to go into a busy loop so this core can run in a
1067 * single-threaded context and inspect kernel memory.
1068 *
1069 * @param proceed_on_sync_failure If true, then go ahead and try to debug even
1070 * if we can't synch with the other cores. This is inherently unsafe and should
1071 * only be used if the kernel is going down in flames anyway.
1072 *
1073 * @param is_stackshot If true, this is a stackshot request.
1074 *
1075 * @result returns KERN_OPERATION_TIMED_OUT if synchronization times out and
1076 * proceed_on_sync_failure is false.
1077 */
1078 kern_return_t
DebuggerXCallEnter(boolean_t proceed_on_sync_failure,bool is_stackshot)1079 DebuggerXCallEnter(
1080 boolean_t proceed_on_sync_failure, bool is_stackshot)
1081 {
1082 uint64_t max_mabs_time, current_mabs_time;
1083 int cpu;
1084 int timeout_cpu = -1;
1085 int max_cpu;
1086 unsigned int sync_pending;
1087 cpu_data_t *target_cpu_datap;
1088 cpu_data_t *cpu_data_ptr = getCpuDatap();
1089
1090 /* Check for nested debugger entry. */
1091 cpu_data_ptr->debugger_active++;
1092 if (cpu_data_ptr->debugger_active != 1) {
1093 return KERN_SUCCESS;
1094 }
1095
1096 /*
1097 * If debugger_sync is not 0, someone responded excessively late to the last
1098 * debug request (we zero the sync variable in the return function). Zero it
1099 * again here. This should prevent us from getting out of sync (heh) and
1100 * timing out on every entry to the debugger if we timeout once.
1101 */
1102
1103 os_atomic_store(&debugger_sync, 0, relaxed);
1104 os_atomic_store(&mp_kdp_trap, 1, relaxed);
1105 os_atomic_store(&debug_cpus_spinning, 0, relaxed);
1106 trap_is_stackshot = is_stackshot;
1107
1108
1109 /*
1110 * Try to signal all CPUs (except ourselves, of course). Use debugger_sync to
1111 * synchronize with every CPU that we appeared to signal successfully (cpu_signal
1112 * is not synchronous).
1113 */
1114 max_cpu = ml_get_max_cpu_number();
1115
1116 boolean_t immediate_halt = FALSE;
1117 if (proceed_on_sync_failure && force_immediate_debug_halt) {
1118 immediate_halt = TRUE;
1119 }
1120
1121 if (!immediate_halt) {
1122 for (cpu = 0; cpu <= max_cpu; cpu++) {
1123 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1124
1125 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1126 continue;
1127 }
1128
1129 kern_return_t ret = cpu_signal(target_cpu_datap, SIGPdebug, (void *)NULL, NULL);
1130 if (ret == KERN_SUCCESS) {
1131 os_atomic_inc(&debugger_sync, relaxed);
1132 os_atomic_inc(&debug_cpus_spinning, relaxed);
1133 } else {
1134 kprintf("%s: cpu_signal failed. cpu=%d ret=%d proceed=%d\n", __func__, cpu, ret, proceed_on_sync_failure);
1135 }
1136 }
1137
1138 max_mabs_time = os_atomic_load(&debug_ack_timeout, relaxed);
1139
1140 if (max_mabs_time > 0) {
1141 current_mabs_time = mach_absolute_time();
1142 max_mabs_time += current_mabs_time;
1143 assert(max_mabs_time > current_mabs_time);
1144 }
1145
1146 /*
1147 * Wait for DEBUG_ACK_TIMEOUT ns for a response from everyone we IPI'd. If we
1148 * timeout, that is simply too bad; we don't have a true NMI, and one CPU may be
1149 * uninterruptibly spinning on someone else. The best we can hope for is that
1150 * all other CPUs have either responded or are spinning in a context that is
1151 * debugger safe.
1152 */
1153 do {
1154 current_mabs_time = mach_absolute_time();
1155 sync_pending = os_atomic_load(&debugger_sync, acquire);
1156 } while ((sync_pending != 0) && (max_mabs_time == 0 || current_mabs_time < max_mabs_time));
1157 }
1158
1159 if (!immediate_halt && max_mabs_time > 0 && current_mabs_time >= max_mabs_time) {
1160 /*
1161 * We timed out trying to IPI the other CPUs. Skip counting any CPUs that
1162 * are offline; then we must account for the remainder, either counting
1163 * them as halted, or trying to dbgwrap them to get them to halt in the
1164 * case where the system is going down and we are running a dev fused
1165 * device.
1166 */
1167 __builtin_arm_dmb(DMB_ISH);
1168 for (cpu = 0; cpu <= max_cpu; cpu++) {
1169 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1170
1171 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1172 continue;
1173 }
1174 if (!(target_cpu_datap->cpu_signal & SIGPdebug)) {
1175 continue;
1176 }
1177 if (processor_array[cpu]->state <= PROCESSOR_PENDING_OFFLINE) {
1178 int dbg_sync_count;
1179
1180 /*
1181 * This is a processor that was successfully sent a SIGPdebug signal
1182 * but which hasn't acknowledged it because it went offline with
1183 * interrupts disabled before the IPI was delivered, so count it
1184 * as halted here.
1185 */
1186 dbg_sync_count = os_atomic_dec(&debugger_sync, relaxed);
1187 kprintf("%s>found CPU %d offline, debugger_sync=%d\n", __FUNCTION__, cpu, dbg_sync_count);
1188 continue;
1189 }
1190 kprintf("%s>Debugger synch pending on cpu %d\n", __FUNCTION__, cpu);
1191 timeout_cpu = cpu;
1192 #if CONFIG_SPTM
1193 if (proceed_on_sync_failure) {
1194 /*
1195 * If a core is spinning in the SPTM panic loop, consider it
1196 * as sync'd, and try to patch up the thread state from the
1197 * SPTM callee saved registers.
1198 */
1199 bool sptm_panic_loop = false;
1200 vm_offset_t base = other_percpu_base(cpu);
1201 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET_WITH_BASE(base, pmap_sptm_percpu);
1202 uint64_t sptm_cpuid = sptm_pcpu->sptm_cpu_id;
1203
1204 if (sptm_get_cpu_state(sptm_cpuid, CPUSTATE_PANIC_SPIN, &sptm_panic_loop)
1205 == SPTM_SUCCESS && sptm_panic_loop) {
1206 xnu_saved_registers_t regs;
1207
1208 if (sptm_copy_callee_saved_state(sptm_cpuid, ®s)
1209 == LIBSPTM_SUCCESS) {
1210 DebuggerPatchupThreadState(cpu, ®s);
1211 }
1212
1213 kprintf("%s>found CPU %d in SPTM\n", __FUNCTION__, cpu);
1214 os_atomic_dec(&debugger_sync, relaxed);
1215 }
1216 }
1217 #endif
1218 }
1219
1220 if (debugger_sync == 0) {
1221 return KERN_SUCCESS;
1222 } else if (!proceed_on_sync_failure) {
1223 panic("%s>Debugger synch pending on cpu %d\n",
1224 __FUNCTION__, timeout_cpu);
1225 }
1226 }
1227 if (immediate_halt || (max_mabs_time > 0 && current_mabs_time >= max_mabs_time)) {
1228 if (immediate_halt) {
1229 __builtin_arm_dmb(DMB_ISH);
1230 }
1231 for (cpu = 0; cpu <= max_cpu; cpu++) {
1232 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1233
1234 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1235 continue;
1236 }
1237 paniclog_append_noflush("Attempting to forcibly halt cpu %d\n", cpu);
1238 dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu(cpu, 0);
1239 if (halt_status < 0) {
1240 paniclog_append_noflush("cpu %d failed to halt with error %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1241 } else {
1242 if (halt_status > 0) {
1243 paniclog_append_noflush("cpu %d halted with warning %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1244 }
1245 target_cpu_datap->halt_status = CPU_HALTED;
1246 }
1247 }
1248 for (cpu = 0; cpu <= max_cpu; cpu++) {
1249 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1250
1251 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1252 continue;
1253 }
1254 dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu_with_state(cpu,
1255 NSEC_PER_SEC, &target_cpu_datap->halt_state);
1256 if ((halt_status < 0) || (halt_status == DBGWRAP_WARN_CPU_OFFLINE)) {
1257 paniclog_append_noflush("Unable to obtain state for cpu %d with status %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1258 debug_ack_timeout_count++;
1259 } else {
1260 paniclog_append_noflush("cpu %d successfully halted\n", cpu);
1261 target_cpu_datap->halt_status = CPU_HALTED_WITH_STATE;
1262 }
1263 }
1264 if (immediate_halt) {
1265 paniclog_append_noflush("Immediate halt requested on all cores\n");
1266 } else {
1267 paniclog_append_noflush("Debugger synchronization timed out; timeout %llu nanoseconds\n",
1268 os_atomic_load(&debug_ack_timeout, relaxed));
1269 }
1270 }
1271 return KERN_SUCCESS;
1272 }
1273
1274 /*
1275 * @function DebuggerXCallReturn
1276 *
1277 * @abstract Resume normal multicore operation after DebuggerXCallEnter()
1278 *
1279 * @discussion This function should be called with debugger lock held.
1280 */
1281 void
DebuggerXCallReturn(void)1282 DebuggerXCallReturn(
1283 void)
1284 {
1285 cpu_data_t *cpu_data_ptr = getCpuDatap();
1286 uint64_t max_mabs_time, current_mabs_time;
1287
1288 cpu_data_ptr->debugger_active--;
1289 if (cpu_data_ptr->debugger_active != 0) {
1290 return;
1291 }
1292
1293 os_atomic_store(&mp_kdp_trap, 0, release);
1294 os_atomic_store(&debugger_sync, 0, relaxed);
1295
1296 max_mabs_time = os_atomic_load(&debug_ack_timeout, relaxed);
1297
1298 if (max_mabs_time > 0) {
1299 current_mabs_time = mach_absolute_time();
1300 max_mabs_time += current_mabs_time;
1301 assert(max_mabs_time > current_mabs_time);
1302 }
1303
1304 /*
1305 * Wait for other CPUs to stop spinning on mp_kdp_trap (see DebuggerXCall).
1306 * It's possible for one or more CPUs to not decrement debug_cpus_spinning,
1307 * since they may be stuck somewhere else with interrupts disabled.
1308 * Wait for DEBUG_ACK_TIMEOUT ns for a response and move on if we don't get it.
1309 *
1310 * Note that the same is done in DebuggerXCallEnter, when we wait for other
1311 * CPUS to update debugger_sync. If we time out, let's hope for all CPUs to be
1312 * spinning in a debugger-safe context
1313 */
1314 while ((os_atomic_load_exclusive(&debug_cpus_spinning, acquire) != 0) &&
1315 (max_mabs_time == 0 || current_mabs_time < max_mabs_time)) {
1316 __builtin_arm_wfe();
1317 current_mabs_time = mach_absolute_time();
1318 }
1319 os_atomic_clear_exclusive();
1320
1321 // checking debug_ack_timeout != 0 is a workaround for rdar://124242354
1322 if (current_mabs_time >= max_mabs_time && os_atomic_load(&debug_ack_timeout, relaxed) != 0) {
1323 panic("Resuming from debugger synchronization failed: waited %llu nanoseconds\n", os_atomic_load(&debug_ack_timeout, relaxed));
1324 }
1325 }
1326
1327 extern void wait_while_mp_kdp_trap(bool check_SIGPdebug);
1328 /*
1329 * Spin while mp_kdp_trap is set.
1330 *
1331 * processor_offline() calls this with check_SIGPdebug=true
1332 * to break out of the spin loop if the cpu has SIGPdebug
1333 * pending.
1334 */
1335 void
wait_while_mp_kdp_trap(bool check_SIGPdebug)1336 wait_while_mp_kdp_trap(bool check_SIGPdebug)
1337 {
1338 bool found_mp_kdp_trap = false;
1339 bool found_SIGPdebug = false;
1340
1341 while (os_atomic_load_exclusive(&mp_kdp_trap, acquire) != 0) {
1342 found_mp_kdp_trap = true;
1343 if (check_SIGPdebug && cpu_has_SIGPdebug_pending()) {
1344 found_SIGPdebug = true;
1345 break;
1346 }
1347 __builtin_arm_wfe();
1348 }
1349 os_atomic_clear_exclusive();
1350
1351 if (check_SIGPdebug && found_mp_kdp_trap) {
1352 kprintf("%s>found_mp_kdp_trap=true found_SIGPdebug=%s\n", __FUNCTION__, found_SIGPdebug ? "true" : "false");
1353 }
1354 }
1355
1356 void
DebuggerXCall(void * ctx)1357 DebuggerXCall(
1358 void *ctx)
1359 {
1360 boolean_t save_context = FALSE;
1361 vm_offset_t kstackptr = 0;
1362 arm_saved_state_t *regs = (arm_saved_state_t *) ctx;
1363
1364 if (regs != NULL) {
1365 #if defined(__arm64__)
1366 current_cpu_datap()->ipi_pc = (uint64_t)get_saved_state_pc(regs);
1367 current_cpu_datap()->ipi_lr = (uint64_t)get_saved_state_lr(regs);
1368 current_cpu_datap()->ipi_fp = (uint64_t)get_saved_state_fp(regs);
1369 save_context = PSR64_IS_KERNEL(get_saved_state_cpsr(regs));
1370 #endif
1371 }
1372
1373 kstackptr = (vm_offset_t)current_thread()->machine.kstackptr;
1374
1375 #if defined(__arm64__)
1376 arm_kernel_saved_state_t *state = (arm_kernel_saved_state_t *)kstackptr;
1377
1378 if (save_context) {
1379 /* Save the interrupted context before acknowledging the signal */
1380 current_thread()->machine.kpcb = regs;
1381 } else if (regs) {
1382 /* zero old state so machine_trace_thread knows not to backtrace it */
1383 state->fp = 0;
1384 state->pc_was_in_userspace = true;
1385 state->lr = 0;
1386 state->sp = 0;
1387 state->ssbs = 0;
1388 state->uao = 0;
1389 state->dit = 0;
1390 #if HAS_MTE
1391 state->tco = 0;
1392 #endif
1393 }
1394 #endif
1395
1396 /*
1397 * When running in serial mode, the core capturing the dump may hold interrupts disabled
1398 * for a time longer than the timeout. That path includes logic to reset the timestamp
1399 * so that we do not eventually trigger the interrupt timeout assert().
1400 *
1401 * Here we check whether other cores have already gone over the timeout at this point
1402 * before spinning, so we at least cover the IPI reception path. After spinning, however,
1403 * we reset the timestamp so as to avoid hitting the interrupt timeout assert().
1404 */
1405 if ((serialmode & SERIALMODE_OUTPUT) || trap_is_stackshot) {
1406 ml_interrupt_masked_debug_end();
1407 }
1408
1409 /*
1410 * Before we decrement debugger sync, do stackshot preflight work (if applicable).
1411 * Namely, we want to signal that we're available to do stackshot work, and we need to
1412 * signal so before the stackshot-calling CPU starts work.
1413 */
1414
1415 if (trap_is_stackshot) {
1416 stackshot_cpu_preflight();
1417 }
1418
1419 os_atomic_dec(&debugger_sync, release);
1420
1421 /* If we trapped because we're doing a stackshot, do our work first. */
1422 if (trap_is_stackshot) {
1423 stackshot_aux_cpu_entry();
1424 }
1425
1426
1427 wait_while_mp_kdp_trap(false);
1428
1429 /**
1430 * Alert the triggering CPU that this CPU is done spinning. The CPU that
1431 * signalled all of the other CPUs will wait (in DebuggerXCallReturn) for
1432 * all of the CPUs to exit the above loop before continuing.
1433 */
1434 os_atomic_dec(&debug_cpus_spinning, release);
1435
1436 #if SCHED_HYGIENE_DEBUG
1437 /*
1438 * We also abandon the measurement for preemption disable
1439 * timeouts, if any. Normally, time in interrupt handlers would be
1440 * subtracted from preemption disable time, and this will happen
1441 * up to this point here, but since we here "end" the interrupt
1442 * handler prematurely (from the point of view of interrupt masked
1443 * debugging), the time spinning would otherwise still be
1444 * attributed to preemption disable time, and potentially trigger
1445 * an event, which could be a panic.
1446 */
1447 abandon_preemption_disable_measurement();
1448
1449 if ((serialmode & SERIALMODE_OUTPUT) || trap_is_stackshot) {
1450 ml_interrupt_masked_debug_start((void *)current_thread()->machine.int_handler_addr, current_thread()->machine.int_type);
1451 }
1452 #endif /* SCHED_HYGIENE_DEBUG */
1453
1454 #if defined(__arm64__)
1455 current_thread()->machine.kpcb = NULL;
1456 #endif /* defined(__arm64__) */
1457
1458 /* Any cleanup for our pushed context should go here */
1459 }
1460
1461 void
DebuggerCall(unsigned int reason,void * ctx)1462 DebuggerCall(
1463 unsigned int reason,
1464 void *ctx)
1465 {
1466 #if !MACH_KDP
1467 #pragma unused(reason,ctx)
1468 #endif /* !MACH_KDP */
1469
1470 #if ALTERNATE_DEBUGGER
1471 alternate_debugger_enter();
1472 #endif
1473
1474 #if MACH_KDP
1475 kdp_trap(reason, (struct arm_saved_state *)ctx);
1476 #else
1477 /* TODO: decide what to do if no debugger config */
1478 #endif
1479 }
1480
1481 boolean_t
bootloader_valid_page(ppnum_t ppn)1482 bootloader_valid_page(ppnum_t ppn)
1483 {
1484 return pmap_bootloader_page(ppn);
1485 }
1486