1 /*
2 * Copyright (c) 2007-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <debug.h>
30 #include <mach_kdp.h>
31 #include <kern/kern_stackshot.h>
32
33 #include <kern/thread.h>
34 #include <machine/pmap.h>
35 #include <device/device_types.h>
36
37 #include <mach/vm_param.h>
38 #include <mach/clock_types.h>
39 #include <mach/machine.h>
40 #include <mach/kmod.h>
41 #include <pexpert/boot.h>
42 #include <pexpert/pexpert.h>
43
44 #include <ptrauth.h>
45
46 #include <kern/misc_protos.h>
47 #include <kern/startup.h>
48 #include <kern/clock.h>
49 #include <kern/debug.h>
50 #include <kern/processor.h>
51 #include <kdp/kdp_core.h>
52 #if ALTERNATE_DEBUGGER
53 #include <arm64/alternate_debugger.h>
54 #endif
55 #include <machine/atomic.h>
56 #include <machine/trap.h>
57 #include <kern/spl.h>
58 #include <pexpert/pexpert.h>
59 #include <kdp/kdp_callout.h>
60 #include <kdp/kdp_dyld.h>
61 #include <kdp/kdp_internal.h>
62 #include <kdp/kdp_common.h>
63 #include <uuid/uuid.h>
64 #include <sys/codesign.h>
65 #include <sys/time.h>
66
67 #if CONFIG_SPTM
68 #include <kern/percpu.h>
69 #include <arm64/sptm/pmap/pmap_data.h>
70 #endif
71
72 #include <IOKit/IOPlatformExpert.h>
73 #include <IOKit/IOKitServer.h>
74
75 #include <mach/vm_prot.h>
76 #include <vm/vm_map_xnu.h>
77 #include <vm/pmap.h>
78 #include <vm/vm_shared_region.h>
79 #include <mach/time_value.h>
80 #include <machine/machparam.h> /* for btop */
81
82 #include <console/video_console.h>
83 #include <console/serial_protos.h>
84 #include <arm/cpu_data.h>
85 #include <arm/cpu_data_internal.h>
86 #include <arm/cpu_internal.h>
87 #include <arm/misc_protos.h>
88 #include <libkern/OSKextLibPrivate.h>
89 #include <vm/vm_kern.h>
90 #include <kern/kern_cdata.h>
91 #include <kern/ledger.h>
92
93
94 #if DEVELOPMENT || DEBUG
95 #include <kern/ext_paniclog.h>
96 #endif
97
98 #if CONFIG_EXCLAVES
99 #include <kern/exclaves_panic.h>
100 #include <kern/exclaves_inspection.h>
101 #endif
102
103 #if MACH_KDP
104 void kdp_trap(unsigned int, struct arm_saved_state *);
105 #endif
106
107 /*
108 * Increment the PANICLOG_VERSION if you change the format of the panic
109 * log in any way.
110 */
111 #define PANICLOG_VERSION 14
112 static struct kcdata_descriptor kc_panic_data;
113
114 extern char iBoot_version[];
115 #if defined(TARGET_OS_OSX) && defined(__arm64__)
116 extern char iBoot_Stage_2_version[];
117 #endif /* defined(TARGET_OS_OSX) && defined(__arm64__) */
118
119 extern volatile uint32_t debug_enabled;
120 extern unsigned int not_in_kdp;
121
122 extern int copyinframe(vm_address_t fp, uint32_t * frame);
123 extern void kdp_callouts(kdp_event_t event);
124
125 /* #include <sys/proc.h> */
126 #define MAXCOMLEN 16
127 struct proc;
128 extern int proc_pid(struct proc *p);
129 extern void proc_name_kdp(struct proc *, char *, int);
130
131 /*
132 * Make sure there's enough space to include the relevant bits in the format required
133 * within the space allocated for the panic version string in the panic header.
134 * The format required by OSAnalytics/DumpPanic is 'Product Version (OS Version)'.
135 */
136 #define PANIC_HEADER_VERSION_FMT_STR "%.14s (%.14s)"
137
138 extern const char version[];
139 extern char osversion[];
140 extern char osproductversion[];
141 extern char osreleasetype[];
142
143 #if defined(XNU_TARGET_OS_BRIDGE)
144 extern char macosproductversion[];
145 extern char macosversion[];
146 #endif
147
148 extern uint8_t gPlatformECID[8];
149 extern uint32_t gPlatformMemoryID;
150
151 extern uint64_t last_hwaccess_thread;
152 extern uint8_t last_hwaccess_type; /* 0 : read, 1 : write. */
153 extern uint8_t last_hwaccess_size;
154 extern uint64_t last_hwaccess_paddr;
155
156 /*Choosing the size for gTargetTypeBuffer as 16 and size for gModelTypeBuffer as 32
157 * since the target name and model name typically doesn't exceed this size */
158 extern char gTargetTypeBuffer[16];
159 extern char gModelTypeBuffer[32];
160
161 extern struct timeval gIOLastSleepTime;
162 extern struct timeval gIOLastWakeTime;
163 extern boolean_t is_clock_configured;
164 extern boolean_t kernelcache_uuid_valid;
165 extern uuid_t kernelcache_uuid;
166 extern uuid_string_t bootsessionuuid_string;
167
168 extern uint64_t roots_installed;
169
170 /* Definitions for frame pointers */
171 #define FP_ALIGNMENT_MASK ((uint32_t)(0x3))
172 #define FP_LR_OFFSET ((uint32_t)4)
173 #define FP_LR_OFFSET64 ((uint32_t)8)
174 #define FP_MAX_NUM_TO_EVALUATE (50)
175
176 /* Timeout for all processors responding to debug crosscall */
177 MACHINE_TIMEOUT(debug_ack_timeout, "debug-ack", 240000, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
178
179 /* Forward functions definitions */
180 void panic_display_times(void);
181 void panic_print_symbol_name(vm_address_t search);
182
183
184 /* Global variables */
185 static uint32_t panic_bt_depth;
186 boolean_t PanicInfoSaved = FALSE;
187 boolean_t force_immediate_debug_halt = FALSE;
188 unsigned int debug_ack_timeout_count = 0;
189 volatile unsigned int debugger_sync = 0;
190 volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */
191 volatile unsigned int debug_cpus_spinning = 0; /* Number of signalled CPUs still spinning on mp_kdp_trap (in DebuggerXCall). */
192 unsigned int DebugContextCount = 0;
193 bool trap_is_stackshot = false; /* Whether the trap is for a stackshot */
194
195 #if defined(__arm64__)
196 uint8_t PE_smc_stashed_x86_system_state = 0xFF;
197 uint8_t PE_smc_stashed_x86_power_state = 0xFF;
198 uint8_t PE_smc_stashed_x86_efi_boot_state = 0xFF;
199 uint8_t PE_smc_stashed_x86_shutdown_cause = 0xFF;
200 uint64_t PE_smc_stashed_x86_prev_power_transitions = UINT64_MAX;
201 uint32_t PE_pcie_stashed_link_state = UINT32_MAX;
202 uint64_t PE_nvram_stashed_x86_macos_slide = UINT64_MAX;
203 #endif
204
205
206 /*
207 * Backtrace a single frame.
208 */
209 static void
print_one_backtrace(pmap_t pmap,vm_offset_t topfp,const char * cur_marker,boolean_t is_64_bit,boolean_t print_kexts_in_backtrace)210 print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker,
211 boolean_t is_64_bit, boolean_t print_kexts_in_backtrace)
212 {
213 unsigned int i = 0;
214 addr64_t lr = 0;
215 addr64_t fp = topfp;
216 addr64_t fp_for_ppn = 0;
217 ppnum_t ppn = (ppnum_t)NULL;
218 vm_offset_t raddrs[FP_MAX_NUM_TO_EVALUATE] = { 0 };
219 bool dump_kernel_stack = (fp >= VM_MIN_KERNEL_ADDRESS);
220
221 #if defined(HAS_APPLE_PAC)
222 fp = (addr64_t)ptrauth_strip((void *)fp, ptrauth_key_frame_pointer);
223 #endif
224 do {
225 if ((fp == 0) || ((fp & FP_ALIGNMENT_MASK) != 0)) {
226 break;
227 }
228 if (dump_kernel_stack && ((fp < VM_MIN_KERNEL_ADDRESS) || (fp > VM_MAX_KERNEL_ADDRESS))) {
229 break;
230 }
231 if ((!dump_kernel_stack) && (fp >= VM_MIN_KERNEL_ADDRESS)) {
232 break;
233 }
234
235 /*
236 * Check to see if current address will result in a different
237 * ppn than previously computed (to avoid recomputation) via
238 * (addr) ^ fp_for_ppn) >> PAGE_SHIFT)
239 */
240 if ((((fp + FP_LR_OFFSET) ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) {
241 ppn = pmap_find_phys(pmap, fp + FP_LR_OFFSET);
242 fp_for_ppn = fp + (is_64_bit ? FP_LR_OFFSET64 : FP_LR_OFFSET);
243 }
244 if (ppn != (ppnum_t)NULL) {
245 if (is_64_bit) {
246 lr = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET64) & PAGE_MASK));
247 #if defined(HAS_APPLE_PAC)
248 /* return addresses on stack will be signed by arm64e ABI */
249 lr = (addr64_t) ptrauth_strip((void *)lr, ptrauth_key_return_address);
250 #endif
251 } else {
252 lr = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET) & PAGE_MASK));
253 }
254 } else {
255 if (is_64_bit) {
256 paniclog_append_noflush("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64);
257 } else {
258 paniclog_append_noflush("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET));
259 }
260 break;
261 }
262 if (((fp ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) {
263 ppn = pmap_find_phys(pmap, fp);
264 fp_for_ppn = fp;
265 }
266 if (ppn != (ppnum_t)NULL) {
267 if (is_64_bit) {
268 fp = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK));
269 #if defined(HAS_APPLE_PAC)
270 /* frame pointers on stack will be signed by arm64e ABI */
271 fp = (addr64_t) ptrauth_strip((void *)fp, ptrauth_key_frame_pointer);
272 #endif
273 } else {
274 fp = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK));
275 }
276 } else {
277 if (is_64_bit) {
278 paniclog_append_noflush("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp);
279 } else {
280 paniclog_append_noflush("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp);
281 }
282 break;
283 }
284 /*
285 * Counter 'i' may == FP_MAX_NUM_TO_EVALUATE when running one
286 * extra round to check whether we have all frames in order to
287 * indicate (in)complete backtrace below. This happens in a case
288 * where total frame count and FP_MAX_NUM_TO_EVALUATE are equal.
289 * Do not capture anything.
290 */
291 if (i < FP_MAX_NUM_TO_EVALUATE && lr) {
292 if (is_64_bit) {
293 paniclog_append_noflush("%s\t lr: 0x%016llx fp: 0x%016llx\n", cur_marker, lr, fp);
294 } else {
295 paniclog_append_noflush("%s\t lr: 0x%08x fp: 0x%08x\n", cur_marker, (uint32_t)lr, (uint32_t)fp);
296 }
297 raddrs[i] = lr;
298 }
299 } while ((++i <= FP_MAX_NUM_TO_EVALUATE) && (fp != topfp));
300
301 if (i > FP_MAX_NUM_TO_EVALUATE && fp != 0) {
302 paniclog_append_noflush("Backtrace continues...\n");
303 }
304
305 if (print_kexts_in_backtrace && i > 0) {
306 kmod_panic_dump(&raddrs[0], i);
307 }
308 }
309
310 #define SANE_TASK_LIMIT 256
311 #define TOP_RUNNABLE_LIMIT 5
312 #define PANICLOG_UUID_BUF_SIZE 256
313
314 extern void panic_print_vnodes(void);
315
316 static void
panic_display_tpidrs(void)317 panic_display_tpidrs(void)
318 {
319 #if defined(__arm64__)
320 paniclog_append_noflush("TPIDRx_ELy = {1: 0x%016llx 0: 0x%016llx 0ro: 0x%016llx }\n",
321 __builtin_arm_rsr64("TPIDR_EL1"), __builtin_arm_rsr64("TPIDR_EL0"),
322 __builtin_arm_rsr64("TPIDRRO_EL0"));
323 #endif //defined(__arm64__)
324 }
325
326
327 static void
panic_display_hung_cpus_help(void)328 panic_display_hung_cpus_help(void)
329 {
330 #if defined(__arm64__)
331 const uint32_t pcsr_offset = 0x90;
332
333 /*
334 * Print some info that might help in cases where nothing
335 * else does
336 */
337 const ml_topology_info_t *info = ml_get_topology_info();
338 if (info) {
339 unsigned i, retry;
340
341 for (i = 0; i < info->num_cpus; i++) {
342 if (!PE_cpu_power_check_kdp(i)) {
343 paniclog_append_noflush("CORE %u is offline, skipping\n", i);
344 continue;
345 }
346 if (info->cpus[i].cpu_UTTDBG_regs) {
347 volatile uint64_t *pcsr = (volatile uint64_t*)(info->cpus[i].cpu_UTTDBG_regs + pcsr_offset);
348 volatile uint32_t *pcsrTrigger = (volatile uint32_t*)pcsr;
349 uint64_t pc = 0;
350
351 // a number of retries are needed till this works
352 for (retry = 1024; retry && !pc; retry--) {
353 //a 32-bit read is required to make a PC sample be produced, else we'll only get a zero
354 (void)*pcsrTrigger;
355 pc = *pcsr;
356 }
357
358 //postprocessing (same as astris does)
359 if (pc >> 48) {
360 pc |= 0xffff000000000000ull;
361 }
362 paniclog_append_noflush("CORE %u recently retired instr at 0x%016llx\n", i, pc);
363 }
364 }
365 }
366 #endif //defined(__arm64__)
367 }
368
369
370
371 static void
panic_display_pvhs_locked(void)372 panic_display_pvhs_locked(void)
373 {
374 }
375
376 static void
panic_display_pvh_to_lock(void)377 panic_display_pvh_to_lock(void)
378 {
379 }
380
381 static void
panic_display_last_pc_lr(void)382 panic_display_last_pc_lr(void)
383 {
384 #if defined(__arm64__)
385 const int max_cpu = ml_get_max_cpu_number();
386
387 for (int cpu = 0; cpu <= max_cpu; cpu++) {
388 cpu_data_t *current_cpu_datap = cpu_datap(cpu);
389
390 if (current_cpu_datap == NULL) {
391 continue;
392 }
393
394 if (current_cpu_datap == getCpuDatap()) {
395 /**
396 * Skip printing the PC/LR if this is the CPU
397 * that initiated the panic.
398 */
399 paniclog_append_noflush("CORE %u is the one that panicked. Check the full backtrace for details.\n", cpu);
400 continue;
401 }
402
403 paniclog_append_noflush("CORE %u: PC=0x%016llx, LR=0x%016llx, FP=0x%016llx\n", cpu,
404 current_cpu_datap->ipi_pc, (uint64_t)VM_KERNEL_STRIP_PTR(current_cpu_datap->ipi_lr),
405 (uint64_t)VM_KERNEL_STRIP_PTR(current_cpu_datap->ipi_fp));
406 }
407 #endif
408 }
409
410 #if CONFIG_EXCLAVES
411 static void
panic_report_exclaves_stackshot(void)412 panic_report_exclaves_stackshot(void)
413 {
414 if (exclaves_panic_ss_status == EXCLAVES_PANIC_STACKSHOT_FOUND) {
415 paniclog_append_noflush("** Exclaves panic stackshot found\n");
416 } else if (exclaves_panic_ss_status == EXCLAVES_PANIC_STACKSHOT_NOT_FOUND) {
417 paniclog_append_noflush("** Exclaves panic stackshot not found\n");
418 } else if (exclaves_panic_ss_status == EXCLAVES_PANIC_STACKSHOT_DECODE_FAILED) {
419 paniclog_append_noflush("!! Exclaves panic stackshot decode failed !!\n");
420 }
421 }
422 #endif /* CONFIG_EXCLAVES */
423
424 static void
do_print_all_backtraces(const char * message,uint64_t panic_options,const char * panic_initiator)425 do_print_all_backtraces(const char *message, uint64_t panic_options, const char *panic_initiator)
426 {
427 int logversion = PANICLOG_VERSION;
428 thread_t cur_thread = current_thread();
429 uintptr_t cur_fp;
430 task_t task;
431 struct proc *proc;
432 int print_vnodes = 0;
433 const char *nohilite_thread_marker = "\t";
434
435 /* end_marker_bytes set to 200 for printing END marker + stackshot summary info always */
436 int bytes_traced = 0, bytes_remaining = 0, end_marker_bytes = 200;
437 int bytes_uncompressed = 0;
438 uint64_t bytes_used = 0ULL;
439 int err = 0;
440 char *stackshot_begin_loc = NULL;
441 kc_format_t kc_format;
442 bool filesetKC = false;
443 uint32_t panic_initiator_len = 0;
444 #if CONFIG_EXT_PANICLOG
445 uint32_t ext_paniclog_bytes = 0;
446 #endif
447
448 #if defined(__arm64__)
449 __asm__ volatile ("add %0, xzr, fp":"=r"(cur_fp));
450 #else
451 #error Unknown architecture.
452 #endif
453 if (panic_bt_depth != 0) {
454 return;
455 }
456 panic_bt_depth++;
457
458 __unused bool result = PE_get_primary_kc_format(&kc_format);
459 assert(result == true);
460 filesetKC = kc_format == KCFormatFileset;
461
462 /* Truncate panic string to 1200 bytes */
463 paniclog_append_noflush("Debugger message: %.1200s\n", message);
464 if (debug_enabled) {
465 paniclog_append_noflush("Device: %s\n",
466 ('\0' != gTargetTypeBuffer[0]) ? gTargetTypeBuffer : "Not set yet");
467 paniclog_append_noflush("Hardware Model: %s\n",
468 ('\0' != gModelTypeBuffer[0]) ? gModelTypeBuffer:"Not set yet");
469 paniclog_append_noflush("ECID: %02X%02X%02X%02X%02X%02X%02X%02X\n", gPlatformECID[7],
470 gPlatformECID[6], gPlatformECID[5], gPlatformECID[4], gPlatformECID[3],
471 gPlatformECID[2], gPlatformECID[1], gPlatformECID[0]);
472 if (last_hwaccess_thread) {
473 paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread);
474 if (!last_hwaccess_size) {
475 paniclog_append_noflush("AppleHWAccess last access: no access data, this is unexpected.\n");
476 } else {
477 const char *typ = last_hwaccess_type ? "write" : "read";
478 paniclog_append_noflush("AppleHWAccess last access: %s of size %u at address 0x%llx\n", typ, last_hwaccess_size, last_hwaccess_paddr);
479 }
480 }
481 paniclog_append_noflush("Boot args: %s\n", PE_boot_args());
482 }
483 paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID);
484 paniclog_append_noflush("OS release type: %.256s\n",
485 ('\0' != osreleasetype[0]) ? osreleasetype : "Not set yet");
486 paniclog_append_noflush("OS version: %.256s\n",
487 ('\0' != osversion[0]) ? osversion : "Not set yet");
488 #if defined(XNU_TARGET_OS_BRIDGE)
489 paniclog_append_noflush("macOS version: %.256s\n",
490 ('\0' != macosversion[0]) ? macosversion : "Not set");
491 #endif
492 paniclog_append_noflush("Kernel version: %.512s\n", version);
493
494 #if CONFIG_EXCLAVES
495 exclaves_panic_append_info();
496 #endif
497
498 if (kernelcache_uuid_valid) {
499 if (filesetKC) {
500 paniclog_append_noflush("Fileset Kernelcache UUID: ");
501 } else {
502 paniclog_append_noflush("KernelCache UUID: ");
503 }
504 for (size_t index = 0; index < sizeof(uuid_t); index++) {
505 paniclog_append_noflush("%02X", kernelcache_uuid[index]);
506 }
507 paniclog_append_noflush("\n");
508 }
509 panic_display_kernel_uuid();
510
511 if (bootsessionuuid_string[0] != '\0') {
512 paniclog_append_noflush("Boot session UUID: %s\n", bootsessionuuid_string);
513 } else {
514 paniclog_append_noflush("Boot session UUID not yet initialized\n");
515 }
516
517 paniclog_append_noflush("iBoot version: %.128s\n", iBoot_version);
518 #if defined(TARGET_OS_OSX) && defined(__arm64__)
519 paniclog_append_noflush("iBoot Stage 2 version: %.128s\n", iBoot_Stage_2_version);
520 #endif /* defined(TARGET_OS_OSX) && defined(__arm64__) */
521
522 paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES");
523 paniclog_append_noflush("roots installed: %lld\n", roots_installed);
524 #if defined(XNU_TARGET_OS_BRIDGE)
525 paniclog_append_noflush("x86 EFI Boot State: ");
526 if (PE_smc_stashed_x86_efi_boot_state != 0xFF) {
527 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_efi_boot_state);
528 } else {
529 paniclog_append_noflush("not available\n");
530 }
531 paniclog_append_noflush("x86 System State: ");
532 if (PE_smc_stashed_x86_system_state != 0xFF) {
533 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_system_state);
534 } else {
535 paniclog_append_noflush("not available\n");
536 }
537 paniclog_append_noflush("x86 Power State: ");
538 if (PE_smc_stashed_x86_power_state != 0xFF) {
539 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_power_state);
540 } else {
541 paniclog_append_noflush("not available\n");
542 }
543 paniclog_append_noflush("x86 Shutdown Cause: ");
544 if (PE_smc_stashed_x86_shutdown_cause != 0xFF) {
545 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_shutdown_cause);
546 } else {
547 paniclog_append_noflush("not available\n");
548 }
549 paniclog_append_noflush("x86 Previous Power Transitions: ");
550 if (PE_smc_stashed_x86_prev_power_transitions != UINT64_MAX) {
551 paniclog_append_noflush("0x%llx\n", PE_smc_stashed_x86_prev_power_transitions);
552 } else {
553 paniclog_append_noflush("not available\n");
554 }
555 paniclog_append_noflush("PCIeUp link state: ");
556 if (PE_pcie_stashed_link_state != UINT32_MAX) {
557 paniclog_append_noflush("0x%x\n", PE_pcie_stashed_link_state);
558 } else {
559 paniclog_append_noflush("not available\n");
560 }
561 paniclog_append_noflush("macOS kernel slide: ");
562 if (PE_nvram_stashed_x86_macos_slide != UINT64_MAX) {
563 paniclog_append_noflush("%#llx\n", PE_nvram_stashed_x86_macos_slide);
564 } else {
565 paniclog_append_noflush("not available\n");
566 }
567 #endif
568 if (panic_data_buffers != NULL) {
569 paniclog_append_noflush("%s data: ", panic_data_buffers->producer_name);
570 uint8_t *panic_buffer_data = (uint8_t *) panic_data_buffers->buf;
571 for (int i = 0; i < panic_data_buffers->len; i++) {
572 paniclog_append_noflush("%02X", panic_buffer_data[i]);
573 }
574 paniclog_append_noflush("\n");
575 }
576 paniclog_append_noflush("Paniclog version: %d\n", logversion);
577
578 panic_display_kernel_aslr();
579 panic_display_times();
580 panic_display_zalloc();
581 panic_display_hung_cpus_help();
582 panic_display_tpidrs();
583 panic_display_pvhs_locked();
584 panic_display_pvh_to_lock();
585 panic_display_last_pc_lr();
586 #if CONFIG_ECC_LOGGING
587 panic_display_ecc_errors();
588 #endif /* CONFIG_ECC_LOGGING */
589 panic_display_compressor_stats();
590
591 #if DEVELOPMENT || DEBUG
592 if (cs_debug_unsigned_exec_failures != 0 || cs_debug_unsigned_mmap_failures != 0) {
593 paniclog_append_noflush("Unsigned code exec failures: %u\n", cs_debug_unsigned_exec_failures);
594 paniclog_append_noflush("Unsigned code mmap failures: %u\n", cs_debug_unsigned_mmap_failures);
595 }
596 #endif
597
598 // Highlight threads that used high amounts of CPU in the panic log if requested (historically requested for watchdog panics)
599 if (panic_options & DEBUGGER_OPTION_PRINT_CPU_USAGE_PANICLOG) {
600 thread_t top_runnable[5] = {0};
601 thread_t thread;
602 int total_cpu_usage = 0;
603
604 print_vnodes = 1;
605
606
607 for (thread = (thread_t)queue_first(&threads);
608 PANIC_VALIDATE_PTR(thread) && !queue_end(&threads, (queue_entry_t)thread);
609 thread = (thread_t)queue_next(&thread->threads)) {
610 total_cpu_usage += thread->cpu_usage;
611
612 // Look for the 5 runnable threads with highest priority
613 if (thread->state & TH_RUN) {
614 int k;
615 thread_t comparison_thread = thread;
616
617 for (k = 0; k < TOP_RUNNABLE_LIMIT; k++) {
618 if (top_runnable[k] == 0) {
619 top_runnable[k] = comparison_thread;
620 break;
621 } else if (comparison_thread->sched_pri > top_runnable[k]->sched_pri) {
622 thread_t temp = top_runnable[k];
623 top_runnable[k] = comparison_thread;
624 comparison_thread = temp;
625 } // if comparison thread has higher priority than previously saved thread
626 } // loop through highest priority runnable threads
627 } // Check if thread is runnable
628 } // Loop through all threads
629
630 // Print the relevant info for each thread identified
631 paniclog_append_noflush("Total cpu_usage: %d\n", total_cpu_usage);
632 paniclog_append_noflush("Thread task pri cpu_usage\n");
633
634 for (int i = 0; i < TOP_RUNNABLE_LIMIT; i++) {
635 if (top_runnable[i] &&
636 panic_get_thread_proc_task(top_runnable[i], &task, &proc) && proc) {
637 char name[MAXCOMLEN + 1];
638 proc_name_kdp(proc, name, sizeof(name));
639 paniclog_append_noflush("%p %s %d %d\n",
640 top_runnable[i], name, top_runnable[i]->sched_pri, top_runnable[i]->cpu_usage);
641 }
642 } // Loop through highest priority runnable threads
643 paniclog_append_noflush("\n");
644 }
645
646 // print current task info
647 if (panic_get_thread_proc_task(cur_thread, &task, &proc)) {
648 if (PANIC_VALIDATE_PTR(task->map) &&
649 PANIC_VALIDATE_PTR(task->map->pmap)) {
650 ledger_amount_t resident = 0;
651 if (task != kernel_task) {
652 ledger_get_balance(task->ledger, task_ledgers.phys_mem, &resident);
653 resident >>= VM_MAP_PAGE_SHIFT(task->map);
654 }
655 paniclog_append_noflush("Panicked task %p: %lld pages, %d threads: ",
656 task, resident, task->thread_count);
657 } else {
658 paniclog_append_noflush("Panicked task %p: %d threads: ",
659 task, task->thread_count);
660 }
661
662 if (proc) {
663 char name[MAXCOMLEN + 1];
664 proc_name_kdp(proc, name, sizeof(name));
665 paniclog_append_noflush("pid %d: %s", proc_pid(proc), name);
666 } else {
667 paniclog_append_noflush("unknown task");
668 }
669
670 paniclog_append_noflush("\n");
671 }
672
673 if (cur_fp < VM_MAX_KERNEL_ADDRESS) {
674 paniclog_append_noflush("Panicked thread: %p, backtrace: 0x%llx, tid: %llu\n",
675 cur_thread, (addr64_t)cur_fp, thread_tid(cur_thread));
676 #if __LP64__
677 print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, TRUE, filesetKC);
678 #else
679 print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, FALSE, filesetKC);
680 #endif
681 } else {
682 paniclog_append_noflush("Could not print panicked thread backtrace:"
683 "frame pointer outside kernel vm.\n");
684 }
685
686 paniclog_append_noflush("\n");
687 if (filesetKC) {
688 kext_dump_panic_lists(&paniclog_append_noflush);
689 paniclog_append_noflush("\n");
690 }
691 panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_panic_log_offset;
692 /* set the os version data in the panic header in the format 'Product Version (OS Version)' (only if they have been set) */
693 if ((osversion[0] != '\0') && (osproductversion[0] != '\0')) {
694 snprintf((char *)&panic_info->eph_os_version, sizeof(panic_info->eph_os_version), PANIC_HEADER_VERSION_FMT_STR,
695 osproductversion, osversion);
696 }
697 #if defined(XNU_TARGET_OS_BRIDGE)
698 if ((macosversion[0] != '\0') && (macosproductversion[0] != '\0')) {
699 snprintf((char *)&panic_info->eph_macos_version, sizeof(panic_info->eph_macos_version), PANIC_HEADER_VERSION_FMT_STR,
700 macosproductversion, macosversion);
701 }
702 #endif
703 if (bootsessionuuid_string[0] != '\0') {
704 memcpy(panic_info->eph_bootsessionuuid_string, bootsessionuuid_string,
705 sizeof(panic_info->eph_bootsessionuuid_string));
706 }
707 panic_info->eph_roots_installed = roots_installed;
708
709 if (panic_initiator != NULL) {
710 bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)debug_buf_ptr - (uintptr_t)debug_buf_base);
711 // If panic_initiator isn't null, safely copy up to MAX_PANIC_INITIATOR_SIZE
712 panic_initiator_len = strnlen(panic_initiator, MAX_PANIC_INITIATOR_SIZE);
713 // Calculate the bytes to write, accounting for remaining buffer space, and ensuring the lowest size we can have is 0
714 panic_initiator_len = MAX(0, MIN(panic_initiator_len, bytes_remaining));
715 panic_info->eph_panic_initiator_offset = (panic_initiator_len != 0) ? PE_get_offset_into_panic_region(debug_buf_ptr) : 0;
716 panic_info->eph_panic_initiator_len = panic_initiator_len;
717 memcpy(debug_buf_ptr, panic_initiator, panic_initiator_len);
718 debug_buf_ptr += panic_initiator_len;
719 }
720
721 if (debug_ack_timeout_count) {
722 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_DEBUGGERSYNC;
723 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
724 paniclog_append_noflush("!! debugger synchronization failed, no stackshot !!\n");
725 } else if (panic_stackshot_active()) {
726 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_NESTED;
727 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
728 paniclog_append_noflush("!! panicked during stackshot, skipping panic stackshot !!\n");
729 } else {
730 /* Align the stackshot buffer to an 8-byte address (especially important for armv7k devices) */
731 debug_buf_ptr += (8 - ((uintptr_t)debug_buf_ptr % 8));
732 stackshot_begin_loc = debug_buf_ptr;
733
734 bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)stackshot_begin_loc - (uintptr_t)debug_buf_base);
735 err = kcdata_memory_static_init(&kc_panic_data, (mach_vm_address_t)debug_buf_ptr,
736 KCDATA_BUFFER_BEGIN_COMPRESSED, bytes_remaining - end_marker_bytes,
737 KCFLAG_USE_MEMCOPY);
738 if (err == KERN_SUCCESS) {
739 uint64_t stackshot_flags = (STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_LOADINFO | STACKSHOT_KCDATA_FORMAT |
740 STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_FROM_PANIC | STACKSHOT_DO_COMPRESS |
741 STACKSHOT_DISABLE_LATENCY_INFO | STACKSHOT_NO_IO_STATS | STACKSHOT_THREAD_WAITINFO | STACKSHOT_GET_DQ |
742 STACKSHOT_COLLECT_SHAREDCACHE_LAYOUT);
743
744 err = kcdata_init_compress(&kc_panic_data, KCDATA_BUFFER_BEGIN_STACKSHOT, kdp_memcpy, KCDCT_ZLIB);
745 if (err != KERN_SUCCESS) {
746 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COMPRESS_FAILED;
747 stackshot_flags &= ~STACKSHOT_DO_COMPRESS;
748 }
749 if (filesetKC) {
750 stackshot_flags |= STACKSHOT_SAVE_KEXT_LOADINFO;
751 }
752
753 kdp_snapshot_preflight(-1, stackshot_begin_loc, bytes_remaining - end_marker_bytes,
754 stackshot_flags, &kc_panic_data, 0, 0);
755 err = do_panic_stackshot(NULL);
756 bytes_traced = kdp_stack_snapshot_bytes_traced();
757 if (bytes_traced > 0 && !err) {
758 debug_buf_ptr += bytes_traced;
759 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_SUCCEEDED;
760 panic_info->eph_stackshot_offset = PE_get_offset_into_panic_region(stackshot_begin_loc);
761 panic_info->eph_stackshot_len = bytes_traced;
762
763 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
764 #if CONFIG_EXCLAVES
765 panic_report_exclaves_stackshot();
766 #endif /* CONFIG_EXCLAVES */
767 if (stackshot_flags & STACKSHOT_DO_COMPRESS) {
768 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_DATA_COMPRESSED;
769 bytes_uncompressed = kdp_stack_snapshot_bytes_uncompressed();
770 paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d (Uncompressed %d) **\n", bytes_traced, bytes_uncompressed);
771 } else {
772 paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d **\n", bytes_traced);
773 }
774 } else {
775 bytes_used = kcdata_memory_get_used_bytes(&kc_panic_data);
776 #if CONFIG_EXCLAVES
777 panic_report_exclaves_stackshot();
778 #endif /* CONFIG_EXCLAVES */
779 if (bytes_used > 0) {
780 /* Zero out the stackshot data */
781 bzero(stackshot_begin_loc, bytes_used);
782 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_INCOMPLETE;
783
784 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
785 paniclog_append_noflush("\n** Stackshot Incomplete ** Bytes Filled %llu, err %d **\n", bytes_used, err);
786 } else {
787 bzero(stackshot_begin_loc, bytes_used);
788 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR;
789
790 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
791 paniclog_append_noflush("\n!! Stackshot Failed !! Bytes Traced %d, err %d\n", bytes_traced, err);
792 }
793 }
794 } else {
795 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR;
796 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
797 paniclog_append_noflush("\n!! Stackshot Failed !!\nkcdata_memory_static_init returned %d", err);
798 }
799 }
800
801 #if CONFIG_EXT_PANICLOG
802 // Write ext paniclog at the end of the paniclog region.
803 ext_paniclog_bytes = ext_paniclog_write_panicdata();
804 panic_info->eph_ext_paniclog_offset = (ext_paniclog_bytes != 0) ?
805 PE_get_offset_into_panic_region((debug_buf_base + debug_buf_size) - ext_paniclog_bytes) :
806 0;
807 panic_info->eph_ext_paniclog_len = ext_paniclog_bytes;
808 #endif
809
810 assert(panic_info->eph_other_log_offset != 0);
811
812 if (print_vnodes != 0) {
813 panic_print_vnodes();
814 }
815
816 panic_bt_depth--;
817 }
818
819 /*
820 * Entry to print_all_backtraces is serialized by the debugger lock
821 */
822 static void
print_all_backtraces(const char * message,uint64_t panic_options,const char * panic_initiator)823 print_all_backtraces(const char *message, uint64_t panic_options, const char *panic_initiator)
824 {
825 unsigned int initial_not_in_kdp = not_in_kdp;
826
827 cpu_data_t * cpu_data_ptr = getCpuDatap();
828
829 assert(cpu_data_ptr->PAB_active == FALSE);
830 cpu_data_ptr->PAB_active = TRUE;
831
832 /*
833 * Because print all backtraces uses the pmap routines, it needs to
834 * avoid taking pmap locks. Right now, this is conditionalized on
835 * not_in_kdp.
836 */
837 not_in_kdp = 0;
838 do_print_all_backtraces(message, panic_options, panic_initiator);
839
840 not_in_kdp = initial_not_in_kdp;
841
842 cpu_data_ptr->PAB_active = FALSE;
843 }
844
845 void
panic_display_times()846 panic_display_times()
847 {
848 if (kdp_clock_is_locked()) {
849 paniclog_append_noflush("Warning: clock is locked. Can't get time\n");
850 return;
851 }
852
853 extern lck_ticket_t clock_lock;
854 extern lck_grp_t clock_lock_grp;
855
856 if ((is_clock_configured) && (lck_ticket_lock_try(&clock_lock, &clock_lock_grp))) {
857 clock_sec_t secs, boot_secs;
858 clock_usec_t usecs, boot_usecs;
859
860 lck_ticket_unlock(&clock_lock);
861
862 clock_get_calendar_microtime(&secs, &usecs);
863 clock_get_boottime_microtime(&boot_secs, &boot_usecs);
864
865 paniclog_append_noflush("mach_absolute_time: 0x%llx\n", mach_absolute_time());
866 paniclog_append_noflush("Epoch Time: sec usec\n");
867 paniclog_append_noflush(" Boot : 0x%08x 0x%08x\n", (unsigned int)boot_secs, (unsigned int)boot_usecs);
868 paniclog_append_noflush(" Sleep : 0x%08x 0x%08x\n", (unsigned int)gIOLastSleepTime.tv_sec, (unsigned int)gIOLastSleepTime.tv_usec);
869 paniclog_append_noflush(" Wake : 0x%08x 0x%08x\n", (unsigned int)gIOLastWakeTime.tv_sec, (unsigned int)gIOLastWakeTime.tv_usec);
870 paniclog_append_noflush(" Calendar: 0x%08x 0x%08x\n\n", (unsigned int)secs, (unsigned int)usecs);
871 }
872 }
873
874 void
panic_print_symbol_name(vm_address_t search)875 panic_print_symbol_name(vm_address_t search)
876 {
877 #pragma unused(search)
878 // empty stub. Really only used on x86_64.
879 return;
880 }
881
882 void
SavePanicInfo(const char * message,__unused void * panic_data,uint64_t panic_options,const char * panic_initiator)883 SavePanicInfo(
884 const char *message, __unused void *panic_data, uint64_t panic_options, const char* panic_initiator)
885 {
886 /*
887 * This should be initialized by the time we get here, but
888 * if it is not, asserting about it will be of no use (it will
889 * come right back to here), so just loop right here and now.
890 * This prevents early-boot panics from becoming recursive and
891 * thus makes them easier to debug. If you attached to a device
892 * and see your PC here, look down a few frames to see your
893 * early-boot panic there.
894 */
895 while (!panic_info || panic_info->eph_panic_log_offset == 0) {
896 // rdar://87170225 (PanicHardening: audit panic code for naked spinloops)
897 // rdar://88094367 (Add test hooks for panic at different stages in XNU)
898 ;
899 }
900
901 if (panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
902 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_BUTTON_RESET_PANIC;
903 }
904
905 if (panic_options & DEBUGGER_OPTION_COMPANION_PROC_INITIATED_PANIC) {
906 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COMPANION_PROC_INITIATED_PANIC;
907 }
908
909 if (panic_options & DEBUGGER_OPTION_INTEGRATED_COPROC_INITIATED_PANIC) {
910 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_INTEGRATED_COPROC_INITIATED_PANIC;
911 }
912
913 if (panic_options & DEBUGGER_OPTION_USERSPACE_INITIATED_PANIC) {
914 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_USERSPACE_INITIATED_PANIC;
915 }
916
917 #if defined(XNU_TARGET_OS_BRIDGE)
918 panic_info->eph_x86_power_state = PE_smc_stashed_x86_power_state;
919 panic_info->eph_x86_efi_boot_state = PE_smc_stashed_x86_efi_boot_state;
920 panic_info->eph_x86_system_state = PE_smc_stashed_x86_system_state;
921 #endif
922
923 /*
924 * On newer targets, panic data is stored directly into the iBoot panic region.
925 * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the
926 * panic CRC so that iBoot can hopefully find *something* useful in the panic region.
927 */
928 if (PanicInfoSaved && (debug_buf_base >= (char*)gPanicBase) && (debug_buf_base < (char*)gPanicBase + gPanicSize)) {
929 unsigned int pi_size = (unsigned int)(debug_buf_ptr - gPanicBase);
930 PE_update_panic_crc((unsigned char*)gPanicBase, &pi_size);
931 PE_sync_panic_buffers(); // extra precaution; panic path likely isn't reliable if we're here
932 }
933
934 if (PanicInfoSaved || (debug_buf_size == 0)) {
935 return;
936 }
937
938 PanicInfoSaved = TRUE;
939
940
941 print_all_backtraces(message, panic_options, panic_initiator);
942
943 assert(panic_info->eph_panic_log_len != 0);
944 panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset;
945
946 PEHaltRestart(kPEPanicSync);
947
948 /*
949 * Notifies registered IOPlatformPanicAction callbacks
950 * (which includes one to disable the memcache) and flushes
951 * the buffer contents from the cache
952 */
953 paniclog_flush();
954 }
955
956 void
paniclog_flush()957 paniclog_flush()
958 {
959 unsigned int panicbuf_length = 0;
960
961 panicbuf_length = (unsigned int)(debug_buf_ptr - gPanicBase);
962 if (!debug_buf_ptr || !panicbuf_length) {
963 return;
964 }
965
966 /*
967 * Updates the log length of the last part of the panic log.
968 */
969 panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset;
970
971 /*
972 * Updates the metadata at the beginning of the panic buffer,
973 * updates the CRC.
974 */
975 PE_update_panic_crc((unsigned char *)gPanicBase, &panicbuf_length);
976
977 /*
978 * This is currently unused by platform KEXTs on embedded but is
979 * kept for compatibility with the published IOKit interfaces.
980 */
981 PESavePanicInfo((unsigned char *)gPanicBase, panicbuf_length);
982
983 PE_sync_panic_buffers();
984 }
985
986 #if CONFIG_SPTM
987 /*
988 * Patch thread state to appear as if a debugger stop IPI occurred, when a thread
989 * is parked in SPTM panic loop. This allows stackshot to proceed as usual.
990 */
991 static void
DebuggerPatchupThreadState(int cpu,xnu_saved_registers_t * regp)992 DebuggerPatchupThreadState(
993 int cpu, xnu_saved_registers_t *regp)
994 {
995 cpu_data_t *target_cpu_datap;
996 arm_saved_state_t *statep;
997 vm_offset_t kstackptr;
998
999 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1000 statep = target_cpu_datap->cpu_active_thread->machine.kpcb;
1001 kstackptr = (vm_offset_t)target_cpu_datap->cpu_active_thread->machine.kstackptr;
1002
1003 target_cpu_datap->ipi_pc = regp->pc;
1004 target_cpu_datap->ipi_lr = regp->lr;
1005 target_cpu_datap->ipi_fp = regp->fp;
1006
1007 if (statep != NULL) {
1008 statep->ss_64.fp = regp->fp;
1009 statep->ss_64.lr = regp->lr;
1010 statep->ss_64.sp = regp->sp;
1011 statep->ss_64.pc = regp->pc;
1012 } else if ((void *)kstackptr != NULL) {
1013 arm_kernel_saved_state_t *kstatep = (arm_kernel_saved_state_t *)kstackptr;
1014 kstatep->fp = regp->fp;
1015 kstatep->lr = regp->lr;
1016 kstatep->sp = regp->sp;
1017 }
1018 }
1019 #endif
1020
1021 /*
1022 * @function DebuggerXCallEnter
1023 *
1024 * @abstract IPI other cores so this core can run in a single-threaded context.
1025 *
1026 * @discussion This function should be called with the debugger lock held. It
1027 * signals the other cores to go into a busy loop so this core can run in a
1028 * single-threaded context and inspect kernel memory.
1029 *
1030 * @param proceed_on_sync_failure If true, then go ahead and try to debug even
1031 * if we can't synch with the other cores. This is inherently unsafe and should
1032 * only be used if the kernel is going down in flames anyway.
1033 *
1034 * @param is_stackshot If true, this is a stackshot request.
1035 *
1036 * @result returns KERN_OPERATION_TIMED_OUT if synchronization times out and
1037 * proceed_on_sync_failure is false.
1038 */
1039 kern_return_t
DebuggerXCallEnter(boolean_t proceed_on_sync_failure,bool is_stackshot)1040 DebuggerXCallEnter(
1041 boolean_t proceed_on_sync_failure, bool is_stackshot)
1042 {
1043 uint64_t max_mabs_time, current_mabs_time;
1044 int cpu;
1045 int timeout_cpu = -1;
1046 int max_cpu;
1047 unsigned int sync_pending;
1048 cpu_data_t *target_cpu_datap;
1049 cpu_data_t *cpu_data_ptr = getCpuDatap();
1050
1051 /* Check for nested debugger entry. */
1052 cpu_data_ptr->debugger_active++;
1053 if (cpu_data_ptr->debugger_active != 1) {
1054 return KERN_SUCCESS;
1055 }
1056
1057 /*
1058 * If debugger_sync is not 0, someone responded excessively late to the last
1059 * debug request (we zero the sync variable in the return function). Zero it
1060 * again here. This should prevent us from getting out of sync (heh) and
1061 * timing out on every entry to the debugger if we timeout once.
1062 */
1063
1064 debugger_sync = 0;
1065 mp_kdp_trap = 1;
1066 debug_cpus_spinning = 0;
1067 trap_is_stackshot = is_stackshot;
1068
1069
1070 /*
1071 * Try to signal all CPUs (except ourselves, of course). Use debugger_sync to
1072 * synchronize with every CPU that we appeared to signal successfully (cpu_signal
1073 * is not synchronous).
1074 */
1075 max_cpu = ml_get_max_cpu_number();
1076
1077 boolean_t immediate_halt = FALSE;
1078 if (proceed_on_sync_failure && force_immediate_debug_halt) {
1079 immediate_halt = TRUE;
1080 }
1081
1082 if (!immediate_halt) {
1083 for (cpu = 0; cpu <= max_cpu; cpu++) {
1084 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1085
1086 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1087 continue;
1088 }
1089
1090 kern_return_t ret = cpu_signal(target_cpu_datap, SIGPdebug, (void *)NULL, NULL);
1091 if (ret == KERN_SUCCESS) {
1092 os_atomic_inc(&debugger_sync, relaxed);
1093 os_atomic_inc(&debug_cpus_spinning, relaxed);
1094 } else if (proceed_on_sync_failure) {
1095 kprintf("cpu_signal failed in DebuggerXCallEnter\n");
1096 }
1097 }
1098
1099 max_mabs_time = os_atomic_load(&debug_ack_timeout, relaxed);
1100
1101 if (max_mabs_time > 0) {
1102 current_mabs_time = mach_absolute_time();
1103 max_mabs_time += current_mabs_time;
1104 assert(max_mabs_time > current_mabs_time);
1105 }
1106
1107 /*
1108 * Wait for DEBUG_ACK_TIMEOUT ns for a response from everyone we IPI'd. If we
1109 * timeout, that is simply too bad; we don't have a true NMI, and one CPU may be
1110 * uninterruptibly spinning on someone else. The best we can hope for is that
1111 * all other CPUs have either responded or are spinning in a context that is
1112 * debugger safe.
1113 */
1114 do {
1115 current_mabs_time = mach_absolute_time();
1116 sync_pending = os_atomic_load(&debugger_sync, acquire);
1117 } while ((sync_pending != 0) && (max_mabs_time == 0 || current_mabs_time < max_mabs_time));
1118 }
1119
1120 if (!immediate_halt && max_mabs_time > 0 && current_mabs_time >= max_mabs_time) {
1121 /*
1122 * We timed out trying to IPI the other CPUs. Skip counting any CPUs that
1123 * are offline; then we must account for the remainder, either counting
1124 * them as halted, or trying to dbgwrap them to get them to halt in the
1125 * case where the system is going down and we are running a dev fused
1126 * device.
1127 */
1128 __builtin_arm_dmb(DMB_ISH);
1129 for (cpu = 0; cpu <= max_cpu; cpu++) {
1130 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1131
1132 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1133 continue;
1134 }
1135 if (!(target_cpu_datap->cpu_signal & SIGPdebug)) {
1136 continue;
1137 }
1138 if (processor_array[cpu]->state <= PROCESSOR_PENDING_OFFLINE) {
1139 /*
1140 * This is a processor that was successfully sent a SIGPdebug signal
1141 * but which hasn't acknowledged it because it went offline with
1142 * interrupts disabled before the IPI was delivered, so count it
1143 * as halted here.
1144 */
1145 os_atomic_dec(&debugger_sync, relaxed);
1146 kprintf("%s>found CPU %d offline, debugger_sync=%d\n", __FUNCTION__, cpu, debugger_sync);
1147 continue;
1148 }
1149 timeout_cpu = cpu;
1150 #if CONFIG_SPTM
1151 if (proceed_on_sync_failure) {
1152 /*
1153 * If a core is spinning in the SPTM panic loop, consider it
1154 * as sync'd, and try to patch up the thread state from the
1155 * SPTM callee saved registers.
1156 */
1157 bool sptm_panic_loop = false;
1158 vm_offset_t base = other_percpu_base(cpu);
1159 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET_WITH_BASE(base, pmap_sptm_percpu);
1160 uint64_t sptm_cpuid = sptm_pcpu->sptm_cpu_id;
1161
1162 if (sptm_get_cpu_state(sptm_cpuid, CPUSTATE_PANIC_SPIN, &sptm_panic_loop)
1163 == SPTM_SUCCESS) {
1164 xnu_saved_registers_t regs;
1165
1166 if (sptm_copy_callee_saved_state(sptm_cpuid, ®s)
1167 == LIBSPTM_SUCCESS) {
1168 DebuggerPatchupThreadState(cpu, ®s);
1169 }
1170
1171 kprintf("%s>found CPU %d in SPTM\n", __FUNCTION__, cpu);
1172 os_atomic_dec(&debugger_sync, relaxed);
1173 }
1174 }
1175 #endif
1176 }
1177
1178 if (debugger_sync == 0) {
1179 return KERN_SUCCESS;
1180 } else if (!proceed_on_sync_failure) {
1181 panic("%s>Debugger synch pending on cpu %d\n",
1182 __FUNCTION__, timeout_cpu);
1183 }
1184 }
1185 if (immediate_halt || (max_mabs_time > 0 && current_mabs_time >= max_mabs_time)) {
1186 if (immediate_halt) {
1187 __builtin_arm_dmb(DMB_ISH);
1188 }
1189 for (cpu = 0; cpu <= max_cpu; cpu++) {
1190 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1191
1192 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1193 continue;
1194 }
1195 paniclog_append_noflush("Attempting to forcibly halt cpu %d\n", cpu);
1196 dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu(cpu, 0);
1197 if (halt_status < 0) {
1198 paniclog_append_noflush("cpu %d failed to halt with error %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1199 } else {
1200 if (halt_status > 0) {
1201 paniclog_append_noflush("cpu %d halted with warning %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1202 }
1203 target_cpu_datap->halt_status = CPU_HALTED;
1204 }
1205 }
1206 for (cpu = 0; cpu <= max_cpu; cpu++) {
1207 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1208
1209 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1210 continue;
1211 }
1212 dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu_with_state(cpu,
1213 NSEC_PER_SEC, &target_cpu_datap->halt_state);
1214 if ((halt_status < 0) || (halt_status == DBGWRAP_WARN_CPU_OFFLINE)) {
1215 paniclog_append_noflush("Unable to obtain state for cpu %d with status %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1216 debug_ack_timeout_count++;
1217 } else {
1218 paniclog_append_noflush("cpu %d successfully halted\n", cpu);
1219 target_cpu_datap->halt_status = CPU_HALTED_WITH_STATE;
1220 }
1221 }
1222 if (immediate_halt) {
1223 paniclog_append_noflush("Immediate halt requested on all cores\n");
1224 } else {
1225 paniclog_append_noflush("Debugger synchronization timed out; timeout %llu nanoseconds\n",
1226 os_atomic_load(&debug_ack_timeout, relaxed));
1227 }
1228 }
1229 return KERN_SUCCESS;
1230 }
1231
1232 /*
1233 * @function DebuggerXCallReturn
1234 *
1235 * @abstract Resume normal multicore operation after DebuggerXCallEnter()
1236 *
1237 * @discussion This function should be called with debugger lock held.
1238 */
1239 void
DebuggerXCallReturn(void)1240 DebuggerXCallReturn(
1241 void)
1242 {
1243 cpu_data_t *cpu_data_ptr = getCpuDatap();
1244 uint64_t max_mabs_time, current_mabs_time;
1245
1246 cpu_data_ptr->debugger_active--;
1247 if (cpu_data_ptr->debugger_active != 0) {
1248 return;
1249 }
1250
1251 mp_kdp_trap = 0;
1252 debugger_sync = 0;
1253
1254 max_mabs_time = os_atomic_load(&debug_ack_timeout, relaxed);
1255
1256 if (max_mabs_time > 0) {
1257 current_mabs_time = mach_absolute_time();
1258 max_mabs_time += current_mabs_time;
1259 assert(max_mabs_time > current_mabs_time);
1260 }
1261
1262 /*
1263 * Wait for other CPUs to stop spinning on mp_kdp_trap (see DebuggerXCall).
1264 * It's possible for one or more CPUs to not decrement debug_cpus_spinning,
1265 * since they may be stuck somewhere else with interrupts disabled.
1266 * Wait for DEBUG_ACK_TIMEOUT ns for a response and move on if we don't get it.
1267 *
1268 * Note that the same is done in DebuggerXCallEnter, when we wait for other
1269 * CPUS to update debugger_sync. If we time out, let's hope for all CPUs to be
1270 * spinning in a debugger-safe context
1271 */
1272 while ((os_atomic_load_exclusive(&debug_cpus_spinning, acquire) != 0) &&
1273 (max_mabs_time == 0 || current_mabs_time < max_mabs_time)) {
1274 __builtin_arm_wfe();
1275 current_mabs_time = mach_absolute_time();
1276 }
1277 os_atomic_clear_exclusive();
1278
1279 // checking debug_ack_timeout != 0 is a workaround for rdar://124242354
1280 if (current_mabs_time >= max_mabs_time && os_atomic_load(&debug_ack_timeout, relaxed) != 0) {
1281 panic("Resuming from debugger synchronization failed: waited %llu nanoseconds\n", os_atomic_load(&debug_ack_timeout, relaxed));
1282 }
1283 }
1284
1285 extern void wait_while_mp_kdp_trap(bool check_SIGPdebug);
1286 /*
1287 * Spin while mp_kdp_trap is set.
1288 *
1289 * processor_offline() calls this with check_SIGPdebug=true
1290 * to break out of the spin loop if the cpu has SIGPdebug
1291 * pending.
1292 */
1293 void
wait_while_mp_kdp_trap(bool check_SIGPdebug)1294 wait_while_mp_kdp_trap(bool check_SIGPdebug)
1295 {
1296 bool found_mp_kdp_trap = false;
1297 bool found_SIGPdebug = false;
1298
1299 while (os_atomic_load_exclusive(&mp_kdp_trap, relaxed) != 0) {
1300 found_mp_kdp_trap = true;
1301 if (check_SIGPdebug && cpu_has_SIGPdebug_pending()) {
1302 found_SIGPdebug = true;
1303 break;
1304 }
1305 __builtin_arm_wfe();
1306 }
1307 os_atomic_clear_exclusive();
1308
1309 if (check_SIGPdebug && found_mp_kdp_trap) {
1310 kprintf("%s>found_mp_kdp_trap=true found_SIGPdebug=%s\n", __FUNCTION__, found_SIGPdebug ? "true" : "false");
1311 }
1312 }
1313
1314 void
DebuggerXCall(void * ctx)1315 DebuggerXCall(
1316 void *ctx)
1317 {
1318 boolean_t save_context = FALSE;
1319 vm_offset_t kstackptr = 0;
1320 arm_saved_state_t *regs = (arm_saved_state_t *) ctx;
1321
1322 if (regs != NULL) {
1323 #if defined(__arm64__)
1324 current_cpu_datap()->ipi_pc = (uint64_t)get_saved_state_pc(regs);
1325 current_cpu_datap()->ipi_lr = (uint64_t)get_saved_state_lr(regs);
1326 current_cpu_datap()->ipi_fp = (uint64_t)get_saved_state_fp(regs);
1327 save_context = PSR64_IS_KERNEL(get_saved_state_cpsr(regs));
1328 #endif
1329 }
1330
1331 kstackptr = (vm_offset_t)current_thread()->machine.kstackptr;
1332
1333 #if defined(__arm64__)
1334 arm_kernel_saved_state_t *state = (arm_kernel_saved_state_t *)kstackptr;
1335
1336 if (save_context) {
1337 /* Save the interrupted context before acknowledging the signal */
1338 current_thread()->machine.kpcb = regs;
1339 } else if (regs) {
1340 /* zero old state so machine_trace_thread knows not to backtrace it */
1341 state->fp = 0;
1342 state->pc_was_in_userspace = true;
1343 state->lr = 0;
1344 state->sp = 0;
1345 state->ssbs = 0;
1346 state->uao = 0;
1347 state->dit = 0;
1348 }
1349 #endif
1350
1351 /*
1352 * When running in serial mode, the core capturing the dump may hold interrupts disabled
1353 * for a time longer than the timeout. That path includes logic to reset the timestamp
1354 * so that we do not eventually trigger the interrupt timeout assert().
1355 *
1356 * Here we check whether other cores have already gone over the timeout at this point
1357 * before spinning, so we at least cover the IPI reception path. After spinning, however,
1358 * we reset the timestamp so as to avoid hitting the interrupt timeout assert().
1359 */
1360 if ((serialmode & SERIALMODE_OUTPUT) || trap_is_stackshot) {
1361 INTERRUPT_MASKED_DEBUG_END();
1362 }
1363
1364 /*
1365 * Before we decrement debugger sync, do stackshot preflight work (if applicable).
1366 * Namely, we want to signal that we're available to do stackshot work, and we need to
1367 * signal so before the stackshot-calling CPU starts work.
1368 */
1369
1370 if (trap_is_stackshot) {
1371 stackshot_cpu_preflight();
1372 }
1373
1374 os_atomic_dec(&debugger_sync, release);
1375
1376 /* If we trapped because we're doing a stackshot, do our work first. */
1377 if (trap_is_stackshot) {
1378 stackshot_aux_cpu_entry();
1379 }
1380
1381
1382 wait_while_mp_kdp_trap(false);
1383
1384 /**
1385 * Alert the triggering CPU that this CPU is done spinning. The CPU that
1386 * signalled all of the other CPUs will wait (in DebuggerXCallReturn) for
1387 * all of the CPUs to exit the above loop before continuing.
1388 */
1389 os_atomic_dec(&debug_cpus_spinning, release);
1390
1391 #if SCHED_HYGIENE_DEBUG
1392 /*
1393 * We also abandon the measurement for preemption disable
1394 * timeouts, if any. Normally, time in interrupt handlers would be
1395 * subtracted from preemption disable time, and this will happen
1396 * up to this point here, but since we here "end" the interrupt
1397 * handler prematurely (from the point of view of interrupt masked
1398 * debugging), the time spinning would otherwise still be
1399 * attributed to preemption disable time, and potentially trigger
1400 * an event, which could be a panic.
1401 */
1402 abandon_preemption_disable_measurement();
1403 #endif /* SCHED_HYGIENE_DEBUG */
1404
1405 if ((serialmode & SERIALMODE_OUTPUT) || trap_is_stackshot) {
1406 INTERRUPT_MASKED_DEBUG_START(current_thread()->machine.int_handler_addr, current_thread()->machine.int_type);
1407 }
1408
1409 #if defined(__arm64__)
1410 current_thread()->machine.kpcb = NULL;
1411 #endif /* defined(__arm64__) */
1412
1413 /* Any cleanup for our pushed context should go here */
1414 }
1415
1416 void
DebuggerCall(unsigned int reason,void * ctx)1417 DebuggerCall(
1418 unsigned int reason,
1419 void *ctx)
1420 {
1421 #if !MACH_KDP
1422 #pragma unused(reason,ctx)
1423 #endif /* !MACH_KDP */
1424
1425 #if ALTERNATE_DEBUGGER
1426 alternate_debugger_enter();
1427 #endif
1428
1429 #if MACH_KDP
1430 kdp_trap(reason, (struct arm_saved_state *)ctx);
1431 #else
1432 /* TODO: decide what to do if no debugger config */
1433 #endif
1434 }
1435
1436 boolean_t
bootloader_valid_page(ppnum_t ppn)1437 bootloader_valid_page(ppnum_t ppn)
1438 {
1439 return pmap_bootloader_page(ppn);
1440 }
1441