1 /*
2 * Copyright (c) 2007-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <debug.h>
30 #include <mach_kdp.h>
31
32 #include <kern/thread.h>
33 #include <machine/pmap.h>
34 #include <device/device_types.h>
35
36 #include <mach/vm_param.h>
37 #include <mach/clock_types.h>
38 #include <mach/machine.h>
39 #include <mach/kmod.h>
40 #include <pexpert/boot.h>
41 #include <pexpert/pexpert.h>
42
43 #include <ptrauth.h>
44
45 #include <kern/misc_protos.h>
46 #include <kern/startup.h>
47 #include <kern/clock.h>
48 #include <kern/debug.h>
49 #include <kern/processor.h>
50 #include <kdp/kdp_core.h>
51 #if ALTERNATE_DEBUGGER
52 #include <arm64/alternate_debugger.h>
53 #endif
54 #include <machine/atomic.h>
55 #include <machine/trap.h>
56 #include <kern/spl.h>
57 #include <pexpert/pexpert.h>
58 #include <kdp/kdp_callout.h>
59 #include <kdp/kdp_dyld.h>
60 #include <kdp/kdp_internal.h>
61 #include <kdp/kdp_common.h>
62 #include <uuid/uuid.h>
63 #include <sys/codesign.h>
64 #include <sys/time.h>
65
66 #include <IOKit/IOPlatformExpert.h>
67 #include <IOKit/IOKitServer.h>
68
69 #include <mach/vm_prot.h>
70 #include <vm/vm_map.h>
71 #include <vm/pmap.h>
72 #include <vm/vm_shared_region.h>
73 #include <mach/time_value.h>
74 #include <machine/machparam.h> /* for btop */
75
76 #include <console/video_console.h>
77 #include <console/serial_protos.h>
78 #include <arm/cpu_data.h>
79 #include <arm/cpu_data_internal.h>
80 #include <arm/cpu_internal.h>
81 #include <arm/misc_protos.h>
82 #include <libkern/OSKextLibPrivate.h>
83 #include <vm/vm_kern.h>
84 #include <kern/kern_cdata.h>
85 #include <kern/ledger.h>
86
87
88 #if DEVELOPMENT || DEBUG
89 #include <kern/ext_paniclog.h>
90 #endif
91
92 #if CONFIG_EXCLAVES
93 #include <kern/exclaves_panic.h>
94 #endif
95
96 #if MACH_KDP
97 void kdp_trap(unsigned int, struct arm_saved_state *);
98 #endif
99
100 extern kern_return_t do_panic_stackshot(void *);
101 extern void kdp_snapshot_preflight(int pid, void * tracebuf,
102 uint32_t tracebuf_size, uint64_t flags,
103 kcdata_descriptor_t data_p,
104 uint64_t since_timestamp, uint32_t pagetable_mask);
105 extern int kdp_stack_snapshot_bytes_traced(void);
106 extern int kdp_stack_snapshot_bytes_uncompressed(void);
107
108 /*
109 * Increment the PANICLOG_VERSION if you change the format of the panic
110 * log in any way.
111 */
112 #define PANICLOG_VERSION 14
113 static struct kcdata_descriptor kc_panic_data;
114
115 extern char iBoot_version[];
116 #if defined(TARGET_OS_OSX) && defined(__arm64__)
117 extern char iBoot_Stage_2_version[];
118 #endif /* defined(TARGET_OS_OSX) && defined(__arm64__) */
119
120 extern volatile uint32_t debug_enabled;
121 extern unsigned int not_in_kdp;
122
123 extern int copyinframe(vm_address_t fp, uint32_t * frame);
124 extern void kdp_callouts(kdp_event_t event);
125
126 /* #include <sys/proc.h> */
127 #define MAXCOMLEN 16
128 struct proc;
129 extern int proc_pid(struct proc *p);
130 extern void proc_name_kdp(struct proc *, char *, int);
131
132 /*
133 * Make sure there's enough space to include the relevant bits in the format required
134 * within the space allocated for the panic version string in the panic header.
135 * The format required by OSAnalytics/DumpPanic is 'Product Version (OS Version)'.
136 */
137 #define PANIC_HEADER_VERSION_FMT_STR "%.14s (%.14s)"
138
139 extern const char version[];
140 extern char osversion[];
141 extern char osproductversion[];
142 extern char osreleasetype[];
143
144 #if defined(XNU_TARGET_OS_BRIDGE)
145 extern char macosproductversion[];
146 extern char macosversion[];
147 #endif
148
149 extern uint8_t gPlatformECID[8];
150 extern uint32_t gPlatformMemoryID;
151
152 extern uint64_t last_hwaccess_thread;
153
154 /*Choosing the size for gTargetTypeBuffer as 16 and size for gModelTypeBuffer as 32
155 * since the target name and model name typically doesn't exceed this size */
156 extern char gTargetTypeBuffer[16];
157 extern char gModelTypeBuffer[32];
158
159 extern struct timeval gIOLastSleepTime;
160 extern struct timeval gIOLastWakeTime;
161 extern boolean_t is_clock_configured;
162 extern boolean_t kernelcache_uuid_valid;
163 extern uuid_t kernelcache_uuid;
164 extern uuid_string_t bootsessionuuid_string;
165
166 extern uint64_t roots_installed;
167
168 /* Definitions for frame pointers */
169 #define FP_ALIGNMENT_MASK ((uint32_t)(0x3))
170 #define FP_LR_OFFSET ((uint32_t)4)
171 #define FP_LR_OFFSET64 ((uint32_t)8)
172 #define FP_MAX_NUM_TO_EVALUATE (50)
173
174 /* Timeout for all processors responding to debug crosscall */
175 MACHINE_TIMEOUT(debug_ack_timeout, "debug-ack", 240000, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
176
177 /* Forward functions definitions */
178 void panic_display_times(void);
179 void panic_print_symbol_name(vm_address_t search);
180
181
182 /* Global variables */
183 static uint32_t panic_bt_depth;
184 boolean_t PanicInfoSaved = FALSE;
185 boolean_t force_immediate_debug_halt = FALSE;
186 unsigned int debug_ack_timeout_count = 0;
187 volatile unsigned int debugger_sync = 0;
188 volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */
189 volatile unsigned int debug_cpus_spinning = 0; /* Number of signalled CPUs still spinning on mp_kdp_trap (in DebuggerXCall). */
190 unsigned int DebugContextCount = 0;
191
192 #if defined(__arm64__)
193 uint8_t PE_smc_stashed_x86_system_state = 0xFF;
194 uint8_t PE_smc_stashed_x86_power_state = 0xFF;
195 uint8_t PE_smc_stashed_x86_efi_boot_state = 0xFF;
196 uint8_t PE_smc_stashed_x86_shutdown_cause = 0xFF;
197 uint64_t PE_smc_stashed_x86_prev_power_transitions = UINT64_MAX;
198 uint32_t PE_pcie_stashed_link_state = UINT32_MAX;
199 uint64_t PE_nvram_stashed_x86_macos_slide = UINT64_MAX;
200 #endif
201
202
203 /*
204 * Backtrace a single frame.
205 */
206 static void
print_one_backtrace(pmap_t pmap,vm_offset_t topfp,const char * cur_marker,boolean_t is_64_bit,boolean_t print_kexts_in_backtrace)207 print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker,
208 boolean_t is_64_bit, boolean_t print_kexts_in_backtrace)
209 {
210 unsigned int i = 0;
211 addr64_t lr = 0;
212 addr64_t fp = topfp;
213 addr64_t fp_for_ppn = 0;
214 ppnum_t ppn = (ppnum_t)NULL;
215 vm_offset_t raddrs[FP_MAX_NUM_TO_EVALUATE] = { 0 };
216 bool dump_kernel_stack = (fp >= VM_MIN_KERNEL_ADDRESS);
217
218 #if defined(HAS_APPLE_PAC)
219 fp = (addr64_t)ptrauth_strip((void *)fp, ptrauth_key_frame_pointer);
220 #endif
221 do {
222 if ((fp == 0) || ((fp & FP_ALIGNMENT_MASK) != 0)) {
223 break;
224 }
225 if (dump_kernel_stack && ((fp < VM_MIN_KERNEL_ADDRESS) || (fp > VM_MAX_KERNEL_ADDRESS))) {
226 break;
227 }
228 if ((!dump_kernel_stack) && (fp >= VM_MIN_KERNEL_ADDRESS)) {
229 break;
230 }
231
232 /*
233 * Check to see if current address will result in a different
234 * ppn than previously computed (to avoid recomputation) via
235 * (addr) ^ fp_for_ppn) >> PAGE_SHIFT)
236 */
237 if ((((fp + FP_LR_OFFSET) ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) {
238 ppn = pmap_find_phys(pmap, fp + FP_LR_OFFSET);
239 fp_for_ppn = fp + (is_64_bit ? FP_LR_OFFSET64 : FP_LR_OFFSET);
240 }
241 if (ppn != (ppnum_t)NULL) {
242 if (is_64_bit) {
243 lr = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET64) & PAGE_MASK));
244 #if defined(HAS_APPLE_PAC)
245 /* return addresses on stack will be signed by arm64e ABI */
246 lr = (addr64_t) ptrauth_strip((void *)lr, ptrauth_key_return_address);
247 #endif
248 } else {
249 lr = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET) & PAGE_MASK));
250 }
251 } else {
252 if (is_64_bit) {
253 paniclog_append_noflush("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64);
254 } else {
255 paniclog_append_noflush("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET));
256 }
257 break;
258 }
259 if (((fp ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) {
260 ppn = pmap_find_phys(pmap, fp);
261 fp_for_ppn = fp;
262 }
263 if (ppn != (ppnum_t)NULL) {
264 if (is_64_bit) {
265 fp = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK));
266 #if defined(HAS_APPLE_PAC)
267 /* frame pointers on stack will be signed by arm64e ABI */
268 fp = (addr64_t) ptrauth_strip((void *)fp, ptrauth_key_frame_pointer);
269 #endif
270 } else {
271 fp = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK));
272 }
273 } else {
274 if (is_64_bit) {
275 paniclog_append_noflush("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp);
276 } else {
277 paniclog_append_noflush("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp);
278 }
279 break;
280 }
281 /*
282 * Counter 'i' may == FP_MAX_NUM_TO_EVALUATE when running one
283 * extra round to check whether we have all frames in order to
284 * indicate (in)complete backtrace below. This happens in a case
285 * where total frame count and FP_MAX_NUM_TO_EVALUATE are equal.
286 * Do not capture anything.
287 */
288 if (i < FP_MAX_NUM_TO_EVALUATE && lr) {
289 if (is_64_bit) {
290 paniclog_append_noflush("%s\t lr: 0x%016llx fp: 0x%016llx\n", cur_marker, lr, fp);
291 } else {
292 paniclog_append_noflush("%s\t lr: 0x%08x fp: 0x%08x\n", cur_marker, (uint32_t)lr, (uint32_t)fp);
293 }
294 raddrs[i] = lr;
295 }
296 } while ((++i <= FP_MAX_NUM_TO_EVALUATE) && (fp != topfp));
297
298 if (i > FP_MAX_NUM_TO_EVALUATE && fp != 0) {
299 paniclog_append_noflush("Backtrace continues...\n");
300 }
301
302 if (print_kexts_in_backtrace && i > 0) {
303 kmod_panic_dump(&raddrs[0], i);
304 }
305 }
306
307 #define SANE_TASK_LIMIT 256
308 #define TOP_RUNNABLE_LIMIT 5
309 #define PANICLOG_UUID_BUF_SIZE 256
310
311 extern void panic_print_vnodes(void);
312
313 static void
panic_display_tpidrs(void)314 panic_display_tpidrs(void)
315 {
316 #if defined(__arm64__)
317 paniclog_append_noflush("TPIDRx_ELy = {1: 0x%016llx 0: 0x%016llx 0ro: 0x%016llx }\n",
318 __builtin_arm_rsr64("TPIDR_EL1"), __builtin_arm_rsr64("TPIDR_EL0"),
319 __builtin_arm_rsr64("TPIDRRO_EL0"));
320 #endif //defined(__arm64__)
321 }
322
323 static void
panic_display_hung_cpus_help(void)324 panic_display_hung_cpus_help(void)
325 {
326 #if defined(__arm64__)
327 const uint32_t pcsr_offset = 0x90;
328
329 /*
330 * Print some info that might help in cases where nothing
331 * else does
332 */
333 const ml_topology_info_t *info = ml_get_topology_info();
334 if (info) {
335 unsigned i, retry;
336
337 for (i = 0; i < info->num_cpus; i++) {
338 if (!PE_cpu_power_check_kdp(i)) {
339 paniclog_append_noflush("CORE %u is offline, skipping\n", i);
340 continue;
341 }
342 if (info->cpus[i].cpu_UTTDBG_regs) {
343 volatile uint64_t *pcsr = (volatile uint64_t*)(info->cpus[i].cpu_UTTDBG_regs + pcsr_offset);
344 volatile uint32_t *pcsrTrigger = (volatile uint32_t*)pcsr;
345 uint64_t pc = 0;
346
347 // a number of retries are needed till this works
348 for (retry = 1024; retry && !pc; retry--) {
349 //a 32-bit read is required to make a PC sample be produced, else we'll only get a zero
350 (void)*pcsrTrigger;
351 pc = *pcsr;
352 }
353
354 //postprocessing (same as astris does)
355 if (pc >> 48) {
356 pc |= 0xffff000000000000ull;
357 }
358 paniclog_append_noflush("CORE %u recently retired instr at 0x%016llx\n", i, pc);
359 }
360 }
361 }
362 #endif //defined(__arm64__)
363 }
364
365
366 static void
panic_display_pvhs_locked(void)367 panic_display_pvhs_locked(void)
368 {
369 }
370
371 static void
panic_display_pvh_to_lock(void)372 panic_display_pvh_to_lock(void)
373 {
374 }
375
376 static void
panic_display_last_pc_lr(void)377 panic_display_last_pc_lr(void)
378 {
379 #if defined(__arm64__)
380 const int max_cpu = ml_get_max_cpu_number();
381
382 for (int cpu = 0; cpu <= max_cpu; cpu++) {
383 cpu_data_t *current_cpu_datap = cpu_datap(cpu);
384
385 if (current_cpu_datap == NULL) {
386 continue;
387 }
388
389 if (current_cpu_datap == getCpuDatap()) {
390 /**
391 * Skip printing the PC/LR if this is the CPU
392 * that initiated the panic.
393 */
394 paniclog_append_noflush("CORE %u is the one that panicked. Check the full backtrace for details.\n", cpu);
395 continue;
396 }
397
398 paniclog_append_noflush("CORE %u: PC=0x%016llx, LR=0x%016llx, FP=0x%016llx\n", cpu,
399 current_cpu_datap->ipi_pc, (uint64_t)VM_KERNEL_STRIP_PTR(current_cpu_datap->ipi_lr),
400 (uint64_t)VM_KERNEL_STRIP_PTR(current_cpu_datap->ipi_fp));
401 }
402 #endif
403 }
404
405 static void
do_print_all_backtraces(const char * message,uint64_t panic_options)406 do_print_all_backtraces(const char *message, uint64_t panic_options)
407 {
408 int logversion = PANICLOG_VERSION;
409 thread_t cur_thread = current_thread();
410 uintptr_t cur_fp;
411 task_t task;
412 struct proc *proc;
413 int print_vnodes = 0;
414 const char *nohilite_thread_marker = "\t";
415
416 /* end_marker_bytes set to 200 for printing END marker + stackshot summary info always */
417 int bytes_traced = 0, bytes_remaining = 0, end_marker_bytes = 200;
418 int bytes_uncompressed = 0;
419 uint64_t bytes_used = 0ULL;
420 int err = 0;
421 char *stackshot_begin_loc = NULL;
422 kc_format_t kc_format;
423 bool filesetKC = false;
424 #if CONFIG_EXT_PANICLOG
425 uint32_t ext_paniclog_bytes = 0;
426 #endif
427
428 #if defined(__arm64__)
429 __asm__ volatile ("add %0, xzr, fp":"=r"(cur_fp));
430 #else
431 #error Unknown architecture.
432 #endif
433 if (panic_bt_depth != 0) {
434 return;
435 }
436 panic_bt_depth++;
437
438 __unused bool result = PE_get_primary_kc_format(&kc_format);
439 assert(result == true);
440 filesetKC = kc_format == KCFormatFileset;
441
442 /* Truncate panic string to 1200 bytes */
443 paniclog_append_noflush("Debugger message: %.1200s\n", message);
444 if (debug_enabled) {
445 paniclog_append_noflush("Device: %s\n",
446 ('\0' != gTargetTypeBuffer[0]) ? gTargetTypeBuffer : "Not set yet");
447 paniclog_append_noflush("Hardware Model: %s\n",
448 ('\0' != gModelTypeBuffer[0]) ? gModelTypeBuffer:"Not set yet");
449 paniclog_append_noflush("ECID: %02X%02X%02X%02X%02X%02X%02X%02X\n", gPlatformECID[7],
450 gPlatformECID[6], gPlatformECID[5], gPlatformECID[4], gPlatformECID[3],
451 gPlatformECID[2], gPlatformECID[1], gPlatformECID[0]);
452 if (last_hwaccess_thread) {
453 paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread);
454 }
455 paniclog_append_noflush("Boot args: %s\n", PE_boot_args());
456 }
457 paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID);
458 paniclog_append_noflush("OS release type: %.256s\n",
459 ('\0' != osreleasetype[0]) ? osreleasetype : "Not set yet");
460 paniclog_append_noflush("OS version: %.256s\n",
461 ('\0' != osversion[0]) ? osversion : "Not set yet");
462 #if defined(XNU_TARGET_OS_BRIDGE)
463 paniclog_append_noflush("macOS version: %.256s\n",
464 ('\0' != macosversion[0]) ? macosversion : "Not set");
465 #endif
466 paniclog_append_noflush("Kernel version: %.512s\n", version);
467
468 #if CONFIG_EXCLAVES
469 exclaves_panic_append_info();
470 #endif
471
472 if (kernelcache_uuid_valid) {
473 if (filesetKC) {
474 paniclog_append_noflush("Fileset Kernelcache UUID: ");
475 } else {
476 paniclog_append_noflush("KernelCache UUID: ");
477 }
478 for (size_t index = 0; index < sizeof(uuid_t); index++) {
479 paniclog_append_noflush("%02X", kernelcache_uuid[index]);
480 }
481 paniclog_append_noflush("\n");
482 }
483 panic_display_kernel_uuid();
484
485 if (bootsessionuuid_string[0] != '\0') {
486 paniclog_append_noflush("Boot session UUID: %s\n", bootsessionuuid_string);
487 } else {
488 paniclog_append_noflush("Boot session UUID not yet initialized\n");
489 }
490
491 paniclog_append_noflush("iBoot version: %.128s\n", iBoot_version);
492 #if defined(TARGET_OS_OSX) && defined(__arm64__)
493 paniclog_append_noflush("iBoot Stage 2 version: %.128s\n", iBoot_Stage_2_version);
494 #endif /* defined(TARGET_OS_OSX) && defined(__arm64__) */
495
496 paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES");
497 paniclog_append_noflush("roots installed: %lld\n", roots_installed);
498 #if defined(XNU_TARGET_OS_BRIDGE)
499 paniclog_append_noflush("x86 EFI Boot State: ");
500 if (PE_smc_stashed_x86_efi_boot_state != 0xFF) {
501 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_efi_boot_state);
502 } else {
503 paniclog_append_noflush("not available\n");
504 }
505 paniclog_append_noflush("x86 System State: ");
506 if (PE_smc_stashed_x86_system_state != 0xFF) {
507 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_system_state);
508 } else {
509 paniclog_append_noflush("not available\n");
510 }
511 paniclog_append_noflush("x86 Power State: ");
512 if (PE_smc_stashed_x86_power_state != 0xFF) {
513 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_power_state);
514 } else {
515 paniclog_append_noflush("not available\n");
516 }
517 paniclog_append_noflush("x86 Shutdown Cause: ");
518 if (PE_smc_stashed_x86_shutdown_cause != 0xFF) {
519 paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_shutdown_cause);
520 } else {
521 paniclog_append_noflush("not available\n");
522 }
523 paniclog_append_noflush("x86 Previous Power Transitions: ");
524 if (PE_smc_stashed_x86_prev_power_transitions != UINT64_MAX) {
525 paniclog_append_noflush("0x%llx\n", PE_smc_stashed_x86_prev_power_transitions);
526 } else {
527 paniclog_append_noflush("not available\n");
528 }
529 paniclog_append_noflush("PCIeUp link state: ");
530 if (PE_pcie_stashed_link_state != UINT32_MAX) {
531 paniclog_append_noflush("0x%x\n", PE_pcie_stashed_link_state);
532 } else {
533 paniclog_append_noflush("not available\n");
534 }
535 paniclog_append_noflush("macOS kernel slide: ");
536 if (PE_nvram_stashed_x86_macos_slide != UINT64_MAX) {
537 paniclog_append_noflush("%#llx\n", PE_nvram_stashed_x86_macos_slide);
538 } else {
539 paniclog_append_noflush("not available\n");
540 }
541 #endif
542 if (panic_data_buffers != NULL) {
543 paniclog_append_noflush("%s data: ", panic_data_buffers->producer_name);
544 uint8_t *panic_buffer_data = (uint8_t *) panic_data_buffers->buf;
545 for (int i = 0; i < panic_data_buffers->len; i++) {
546 paniclog_append_noflush("%02X", panic_buffer_data[i]);
547 }
548 paniclog_append_noflush("\n");
549 }
550 paniclog_append_noflush("Paniclog version: %d\n", logversion);
551
552 panic_display_kernel_aslr();
553 panic_display_times();
554 panic_display_zalloc();
555 panic_display_hung_cpus_help();
556 panic_display_tpidrs();
557 panic_display_pvhs_locked();
558 panic_display_pvh_to_lock();
559 panic_display_last_pc_lr();
560 #if CONFIG_ECC_LOGGING
561 panic_display_ecc_errors();
562 #endif /* CONFIG_ECC_LOGGING */
563 panic_display_compressor_stats();
564
565 #if DEVELOPMENT || DEBUG
566 if (cs_debug_unsigned_exec_failures != 0 || cs_debug_unsigned_mmap_failures != 0) {
567 paniclog_append_noflush("Unsigned code exec failures: %u\n", cs_debug_unsigned_exec_failures);
568 paniclog_append_noflush("Unsigned code mmap failures: %u\n", cs_debug_unsigned_mmap_failures);
569 }
570 #endif
571
572 // Highlight threads that used high amounts of CPU in the panic log if requested (historically requested for watchdog panics)
573 if (panic_options & DEBUGGER_OPTION_PRINT_CPU_USAGE_PANICLOG) {
574 thread_t top_runnable[5] = {0};
575 thread_t thread;
576 int total_cpu_usage = 0;
577
578 print_vnodes = 1;
579
580
581 for (thread = (thread_t)queue_first(&threads);
582 PANIC_VALIDATE_PTR(thread) && !queue_end(&threads, (queue_entry_t)thread);
583 thread = (thread_t)queue_next(&thread->threads)) {
584 total_cpu_usage += thread->cpu_usage;
585
586 // Look for the 5 runnable threads with highest priority
587 if (thread->state & TH_RUN) {
588 int k;
589 thread_t comparison_thread = thread;
590
591 for (k = 0; k < TOP_RUNNABLE_LIMIT; k++) {
592 if (top_runnable[k] == 0) {
593 top_runnable[k] = comparison_thread;
594 break;
595 } else if (comparison_thread->sched_pri > top_runnable[k]->sched_pri) {
596 thread_t temp = top_runnable[k];
597 top_runnable[k] = comparison_thread;
598 comparison_thread = temp;
599 } // if comparison thread has higher priority than previously saved thread
600 } // loop through highest priority runnable threads
601 } // Check if thread is runnable
602 } // Loop through all threads
603
604 // Print the relevant info for each thread identified
605 paniclog_append_noflush("Total cpu_usage: %d\n", total_cpu_usage);
606 paniclog_append_noflush("Thread task pri cpu_usage\n");
607
608 for (int i = 0; i < TOP_RUNNABLE_LIMIT; i++) {
609 if (top_runnable[i] &&
610 panic_get_thread_proc_task(top_runnable[i], &task, &proc) && proc) {
611 char name[MAXCOMLEN + 1];
612 proc_name_kdp(proc, name, sizeof(name));
613 paniclog_append_noflush("%p %s %d %d\n",
614 top_runnable[i], name, top_runnable[i]->sched_pri, top_runnable[i]->cpu_usage);
615 }
616 } // Loop through highest priority runnable threads
617 paniclog_append_noflush("\n");
618 }
619
620 // print current task info
621 if (panic_get_thread_proc_task(cur_thread, &task, &proc)) {
622 if (PANIC_VALIDATE_PTR(task->map) &&
623 PANIC_VALIDATE_PTR(task->map->pmap)) {
624 ledger_amount_t resident = 0;
625 if (task != kernel_task) {
626 ledger_get_balance(task->ledger, task_ledgers.phys_mem, &resident);
627 resident >>= VM_MAP_PAGE_SHIFT(task->map);
628 }
629 paniclog_append_noflush("Panicked task %p: %lld pages, %d threads: ",
630 task, resident, task->thread_count);
631 } else {
632 paniclog_append_noflush("Panicked task %p: %d threads: ",
633 task, task->thread_count);
634 }
635
636 if (proc) {
637 char name[MAXCOMLEN + 1];
638 proc_name_kdp(proc, name, sizeof(name));
639 paniclog_append_noflush("pid %d: %s", proc_pid(proc), name);
640 } else {
641 paniclog_append_noflush("unknown task");
642 }
643
644 paniclog_append_noflush("\n");
645 }
646
647 if (cur_fp < VM_MAX_KERNEL_ADDRESS) {
648 paniclog_append_noflush("Panicked thread: %p, backtrace: 0x%llx, tid: %llu\n",
649 cur_thread, (addr64_t)cur_fp, thread_tid(cur_thread));
650 #if __LP64__
651 print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, TRUE, filesetKC);
652 #else
653 print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, FALSE, filesetKC);
654 #endif
655 } else {
656 paniclog_append_noflush("Could not print panicked thread backtrace:"
657 "frame pointer outside kernel vm.\n");
658 }
659
660 paniclog_append_noflush("\n");
661 if (filesetKC) {
662 kext_dump_panic_lists(&paniclog_append_noflush);
663 paniclog_append_noflush("\n");
664 }
665 panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_panic_log_offset;
666 /* set the os version data in the panic header in the format 'Product Version (OS Version)' (only if they have been set) */
667 if ((osversion[0] != '\0') && (osproductversion[0] != '\0')) {
668 snprintf((char *)&panic_info->eph_os_version, sizeof(panic_info->eph_os_version), PANIC_HEADER_VERSION_FMT_STR,
669 osproductversion, osversion);
670 }
671 #if defined(XNU_TARGET_OS_BRIDGE)
672 if ((macosversion[0] != '\0') && (macosproductversion[0] != '\0')) {
673 snprintf((char *)&panic_info->eph_macos_version, sizeof(panic_info->eph_macos_version), PANIC_HEADER_VERSION_FMT_STR,
674 macosproductversion, macosversion);
675 }
676 #endif
677 if (bootsessionuuid_string[0] != '\0') {
678 memcpy(panic_info->eph_bootsessionuuid_string, bootsessionuuid_string,
679 sizeof(panic_info->eph_bootsessionuuid_string));
680 }
681 panic_info->eph_roots_installed = roots_installed;
682
683 if (debug_ack_timeout_count) {
684 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_DEBUGGERSYNC;
685 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
686 paniclog_append_noflush("!! debugger synchronization failed, no stackshot !!\n");
687 } else if (stackshot_active()) {
688 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_NESTED;
689 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
690 paniclog_append_noflush("!! panicked during stackshot, skipping panic stackshot !!\n");
691 } else {
692 /* Align the stackshot buffer to an 8-byte address (especially important for armv7k devices) */
693 debug_buf_ptr += (8 - ((uintptr_t)debug_buf_ptr % 8));
694 stackshot_begin_loc = debug_buf_ptr;
695
696 bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)stackshot_begin_loc - (uintptr_t)debug_buf_base);
697 err = kcdata_memory_static_init(&kc_panic_data, (mach_vm_address_t)debug_buf_ptr,
698 KCDATA_BUFFER_BEGIN_COMPRESSED, bytes_remaining - end_marker_bytes,
699 KCFLAG_USE_MEMCOPY);
700 if (err == KERN_SUCCESS) {
701 uint64_t stackshot_flags = (STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_LOADINFO | STACKSHOT_KCDATA_FORMAT |
702 STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_FROM_PANIC | STACKSHOT_DO_COMPRESS |
703 STACKSHOT_DISABLE_LATENCY_INFO | STACKSHOT_NO_IO_STATS | STACKSHOT_THREAD_WAITINFO | STACKSHOT_GET_DQ |
704 STACKSHOT_COLLECT_SHAREDCACHE_LAYOUT);
705
706 err = kcdata_init_compress(&kc_panic_data, KCDATA_BUFFER_BEGIN_STACKSHOT, kdp_memcpy, KCDCT_ZLIB);
707 if (err != KERN_SUCCESS) {
708 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COMPRESS_FAILED;
709 stackshot_flags &= ~STACKSHOT_DO_COMPRESS;
710 }
711 if (filesetKC) {
712 stackshot_flags |= STACKSHOT_SAVE_KEXT_LOADINFO;
713 }
714
715 kdp_snapshot_preflight(-1, stackshot_begin_loc, bytes_remaining - end_marker_bytes,
716 stackshot_flags, &kc_panic_data, 0, 0);
717 err = do_panic_stackshot(NULL);
718 bytes_traced = kdp_stack_snapshot_bytes_traced();
719 if (bytes_traced > 0 && !err) {
720 debug_buf_ptr += bytes_traced;
721 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_SUCCEEDED;
722 panic_info->eph_stackshot_offset = PE_get_offset_into_panic_region(stackshot_begin_loc);
723 panic_info->eph_stackshot_len = bytes_traced;
724
725 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
726 if (stackshot_flags & STACKSHOT_DO_COMPRESS) {
727 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_DATA_COMPRESSED;
728 bytes_uncompressed = kdp_stack_snapshot_bytes_uncompressed();
729 paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d (Uncompressed %d) **\n", bytes_traced, bytes_uncompressed);
730 } else {
731 paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d **\n", bytes_traced);
732 }
733 } else {
734 bytes_used = kcdata_memory_get_used_bytes(&kc_panic_data);
735 if (bytes_used > 0) {
736 /* Zero out the stackshot data */
737 bzero(stackshot_begin_loc, bytes_used);
738 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_INCOMPLETE;
739
740 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
741 paniclog_append_noflush("\n** Stackshot Incomplete ** Bytes Filled %llu **\n", bytes_used);
742 } else {
743 bzero(stackshot_begin_loc, bytes_used);
744 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR;
745
746 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
747 paniclog_append_noflush("\n!! Stackshot Failed !! Bytes Traced %d, err %d\n", bytes_traced, err);
748 }
749 }
750 } else {
751 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR;
752 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
753 paniclog_append_noflush("\n!! Stackshot Failed !!\nkcdata_memory_static_init returned %d", err);
754 }
755 }
756
757 #if CONFIG_EXT_PANICLOG
758 // Write ext paniclog at the end of the paniclog region.
759 ext_paniclog_bytes = ext_paniclog_write_panicdata();
760 panic_info->eph_ext_paniclog_offset = (ext_paniclog_bytes != 0) ?
761 PE_get_offset_into_panic_region((debug_buf_base + debug_buf_size) - ext_paniclog_bytes) :
762 0;
763 panic_info->eph_ext_paniclog_len = ext_paniclog_bytes;
764 #endif
765
766 assert(panic_info->eph_other_log_offset != 0);
767
768 if (print_vnodes != 0) {
769 panic_print_vnodes();
770 }
771
772 panic_bt_depth--;
773 }
774
775 /*
776 * Entry to print_all_backtraces is serialized by the debugger lock
777 */
778 static void
print_all_backtraces(const char * message,uint64_t panic_options)779 print_all_backtraces(const char *message, uint64_t panic_options)
780 {
781 unsigned int initial_not_in_kdp = not_in_kdp;
782
783 cpu_data_t * cpu_data_ptr = getCpuDatap();
784
785 assert(cpu_data_ptr->PAB_active == FALSE);
786 cpu_data_ptr->PAB_active = TRUE;
787
788 /*
789 * Because print all backtraces uses the pmap routines, it needs to
790 * avoid taking pmap locks. Right now, this is conditionalized on
791 * not_in_kdp.
792 */
793 not_in_kdp = 0;
794 do_print_all_backtraces(message, panic_options);
795
796 not_in_kdp = initial_not_in_kdp;
797
798 cpu_data_ptr->PAB_active = FALSE;
799 }
800
801 void
panic_display_times()802 panic_display_times()
803 {
804 if (kdp_clock_is_locked()) {
805 paniclog_append_noflush("Warning: clock is locked. Can't get time\n");
806 return;
807 }
808
809 extern lck_ticket_t clock_lock;
810 extern lck_grp_t clock_lock_grp;
811
812 if ((is_clock_configured) && (lck_ticket_lock_try(&clock_lock, &clock_lock_grp))) {
813 clock_sec_t secs, boot_secs;
814 clock_usec_t usecs, boot_usecs;
815
816 lck_ticket_unlock(&clock_lock);
817
818 clock_get_calendar_microtime(&secs, &usecs);
819 clock_get_boottime_microtime(&boot_secs, &boot_usecs);
820
821 paniclog_append_noflush("mach_absolute_time: 0x%llx\n", mach_absolute_time());
822 paniclog_append_noflush("Epoch Time: sec usec\n");
823 paniclog_append_noflush(" Boot : 0x%08x 0x%08x\n", (unsigned int)boot_secs, (unsigned int)boot_usecs);
824 paniclog_append_noflush(" Sleep : 0x%08x 0x%08x\n", (unsigned int)gIOLastSleepTime.tv_sec, (unsigned int)gIOLastSleepTime.tv_usec);
825 paniclog_append_noflush(" Wake : 0x%08x 0x%08x\n", (unsigned int)gIOLastWakeTime.tv_sec, (unsigned int)gIOLastWakeTime.tv_usec);
826 paniclog_append_noflush(" Calendar: 0x%08x 0x%08x\n\n", (unsigned int)secs, (unsigned int)usecs);
827 }
828 }
829
830 void
panic_print_symbol_name(vm_address_t search)831 panic_print_symbol_name(vm_address_t search)
832 {
833 #pragma unused(search)
834 // empty stub. Really only used on x86_64.
835 return;
836 }
837
838 void
SavePanicInfo(const char * message,__unused void * panic_data,uint64_t panic_options)839 SavePanicInfo(
840 const char *message, __unused void *panic_data, uint64_t panic_options)
841 {
842 /*
843 * This should be initialized by the time we get here, but
844 * if it is not, asserting about it will be of no use (it will
845 * come right back to here), so just loop right here and now.
846 * This prevents early-boot panics from becoming recursive and
847 * thus makes them easier to debug. If you attached to a device
848 * and see your PC here, look down a few frames to see your
849 * early-boot panic there.
850 */
851 while (!panic_info || panic_info->eph_panic_log_offset == 0) {
852 // rdar://87170225 (PanicHardening: audit panic code for naked spinloops)
853 // rdar://88094367 (Add test hooks for panic at different stages in XNU)
854 ;
855 }
856
857 if (panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
858 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_BUTTON_RESET_PANIC;
859 }
860
861 if (panic_options & DEBUGGER_OPTION_COPROC_INITIATED_PANIC) {
862 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC;
863 }
864
865 #if defined(XNU_TARGET_OS_BRIDGE)
866 panic_info->eph_x86_power_state = PE_smc_stashed_x86_power_state;
867 panic_info->eph_x86_efi_boot_state = PE_smc_stashed_x86_efi_boot_state;
868 panic_info->eph_x86_system_state = PE_smc_stashed_x86_system_state;
869 #endif
870
871 /*
872 * On newer targets, panic data is stored directly into the iBoot panic region.
873 * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the
874 * panic CRC so that iBoot can hopefully find *something* useful in the panic region.
875 */
876 if (PanicInfoSaved && (debug_buf_base >= (char*)gPanicBase) && (debug_buf_base < (char*)gPanicBase + gPanicSize)) {
877 unsigned int pi_size = (unsigned int)(debug_buf_ptr - gPanicBase);
878 PE_update_panic_crc((unsigned char*)gPanicBase, &pi_size);
879 PE_sync_panic_buffers(); // extra precaution; panic path likely isn't reliable if we're here
880 }
881
882 if (PanicInfoSaved || (debug_buf_size == 0)) {
883 return;
884 }
885
886 PanicInfoSaved = TRUE;
887
888
889 print_all_backtraces(message, panic_options);
890
891 assert(panic_info->eph_panic_log_len != 0);
892 panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset;
893
894 PEHaltRestart(kPEPanicSync);
895
896 /*
897 * Notifies registered IOPlatformPanicAction callbacks
898 * (which includes one to disable the memcache) and flushes
899 * the buffer contents from the cache
900 */
901 paniclog_flush();
902 }
903
904 void
paniclog_flush()905 paniclog_flush()
906 {
907 unsigned int panicbuf_length = 0;
908
909 panicbuf_length = (unsigned int)(debug_buf_ptr - gPanicBase);
910 if (!debug_buf_ptr || !panicbuf_length) {
911 return;
912 }
913
914 /*
915 * Updates the log length of the last part of the panic log.
916 */
917 panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset;
918
919 /*
920 * Updates the metadata at the beginning of the panic buffer,
921 * updates the CRC.
922 */
923 PE_update_panic_crc((unsigned char *)gPanicBase, &panicbuf_length);
924
925 /*
926 * This is currently unused by platform KEXTs on embedded but is
927 * kept for compatibility with the published IOKit interfaces.
928 */
929 PESavePanicInfo((unsigned char *)gPanicBase, panicbuf_length);
930
931 PE_sync_panic_buffers();
932 }
933
934 /*
935 * @function DebuggerXCallEnter
936 *
937 * @abstract IPI other cores so this core can run in a single-threaded context.
938 *
939 * @discussion This function should be called with the debugger lock held. It
940 * signals the other cores to go into a busy loop so this core can run in a
941 * single-threaded context and inspect kernel memory.
942 *
943 * @param proceed_on_sync_failure If true, then go ahead and try to debug even
944 * if we can't synch with the other cores. This is inherently unsafe and should
945 * only be used if the kernel is going down in flames anyway.
946 *
947 * @param is_stackshot If true, this is a stackshot request.
948 *
949 * @result returns KERN_OPERATION_TIMED_OUT if synchronization times out and
950 * proceed_on_sync_failure is false.
951 */
952 kern_return_t
DebuggerXCallEnter(boolean_t proceed_on_sync_failure,bool is_stackshot)953 DebuggerXCallEnter(
954 boolean_t proceed_on_sync_failure, bool is_stackshot)
955 {
956 uint64_t max_mabs_time, current_mabs_time;
957 int cpu;
958 int max_cpu;
959 cpu_data_t *target_cpu_datap;
960 cpu_data_t *cpu_data_ptr = getCpuDatap();
961
962 /* Check for nested debugger entry. */
963 cpu_data_ptr->debugger_active++;
964 if (cpu_data_ptr->debugger_active != 1) {
965 return KERN_SUCCESS;
966 }
967
968 /*
969 * If debugger_sync is not 0, someone responded excessively late to the last
970 * debug request (we zero the sync variable in the return function). Zero it
971 * again here. This should prevent us from getting out of sync (heh) and
972 * timing out on every entry to the debugger if we timeout once.
973 */
974
975 debugger_sync = 0;
976 mp_kdp_trap = 1;
977 debug_cpus_spinning = 0;
978
979 #pragma unused(is_stackshot)
980
981 /*
982 * Try to signal all CPUs (except ourselves, of course). Use debugger_sync to
983 * synchronize with every CPU that we appeared to signal successfully (cpu_signal
984 * is not synchronous).
985 */
986 max_cpu = ml_get_max_cpu_number();
987
988 boolean_t immediate_halt = FALSE;
989 if (proceed_on_sync_failure && force_immediate_debug_halt) {
990 immediate_halt = TRUE;
991 }
992
993 if (!immediate_halt) {
994 for (cpu = 0; cpu <= max_cpu; cpu++) {
995 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
996
997 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
998 continue;
999 }
1000
1001 kern_return_t ret = cpu_signal(target_cpu_datap, SIGPdebug, (void *)NULL, NULL);
1002 if (ret == KERN_SUCCESS) {
1003 os_atomic_inc(&debugger_sync, relaxed);
1004 os_atomic_inc(&debug_cpus_spinning, relaxed);
1005 } else if (proceed_on_sync_failure) {
1006 kprintf("cpu_signal failed in DebuggerXCallEnter\n");
1007 }
1008 }
1009
1010 max_mabs_time = os_atomic_load(&debug_ack_timeout, relaxed);
1011
1012 if (max_mabs_time > 0) {
1013 current_mabs_time = mach_absolute_time();
1014 max_mabs_time += current_mabs_time;
1015 assert(max_mabs_time > current_mabs_time);
1016 }
1017
1018 /*
1019 * Wait for DEBUG_ACK_TIMEOUT ns for a response from everyone we IPI'd. If we
1020 * timeout, that is simply too bad; we don't have a true NMI, and one CPU may be
1021 * uninterruptibly spinning on someone else. The best we can hope for is that
1022 * all other CPUs have either responded or are spinning in a context that is
1023 * debugger safe.
1024 */
1025 while ((debugger_sync != 0) && (max_mabs_time == 0 || current_mabs_time < max_mabs_time)) {
1026 current_mabs_time = mach_absolute_time();
1027 }
1028 }
1029
1030 if (!proceed_on_sync_failure && (max_mabs_time > 0 && current_mabs_time >= max_mabs_time)) {
1031 __builtin_arm_dmb(DMB_ISH);
1032 for (cpu = 0; cpu <= max_cpu; cpu++) {
1033 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1034
1035 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1036 continue;
1037 }
1038 if (!(target_cpu_datap->cpu_signal & SIGPdebug)) {
1039 continue;
1040 }
1041 if (processor_array[cpu]->state <= PROCESSOR_PENDING_OFFLINE) {
1042 /*
1043 * This is a processor that was successfully sent a SIGPdebug signal
1044 * but which hasn't acknowledged it because it went offline with
1045 * interrupts disabled before the IPI was delivered, so count it
1046 * here.
1047 */
1048 os_atomic_dec(&debugger_sync, relaxed);
1049 kprintf("%s>found CPU %d offline, debugger_sync=%d\n", __FUNCTION__, cpu, debugger_sync);
1050 continue;
1051 }
1052
1053 kprintf("%s>Debugger synch pending on cpu %d\n", __FUNCTION__, cpu);
1054 }
1055
1056 if (debugger_sync == 0) {
1057 return KERN_SUCCESS;
1058 } else {
1059 DebuggerXCallReturn();
1060 kprintf("%s>returning KERN_OPERATION_TIMED_OUT\n", __FUNCTION__);
1061 return KERN_OPERATION_TIMED_OUT;
1062 }
1063 } else if (immediate_halt || (max_mabs_time > 0 && current_mabs_time >= max_mabs_time)) {
1064 /*
1065 * For the moment, we're aiming for a timeout that the user shouldn't notice,
1066 * but will be sufficient to let the other core respond.
1067 */
1068 __builtin_arm_dmb(DMB_ISH);
1069 for (cpu = 0; cpu <= max_cpu; cpu++) {
1070 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1071
1072 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) {
1073 continue;
1074 }
1075 if (!(target_cpu_datap->cpu_signal & SIGPdebug) && !immediate_halt) {
1076 continue;
1077 }
1078 if (proceed_on_sync_failure) {
1079 paniclog_append_noflush("Attempting to forcibly halt cpu %d\n", cpu);
1080 dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu(cpu, 0);
1081 if (halt_status < 0) {
1082 paniclog_append_noflush("cpu %d failed to halt with error %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1083 } else {
1084 if (halt_status > 0) {
1085 paniclog_append_noflush("cpu %d halted with warning %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1086 }
1087 target_cpu_datap->halt_status = CPU_HALTED;
1088 }
1089 } else {
1090 kprintf("Debugger synch pending on cpu %d\n", cpu);
1091 }
1092 }
1093 if (proceed_on_sync_failure) {
1094 for (cpu = 0; cpu <= max_cpu; cpu++) {
1095 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1096
1097 if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr) ||
1098 (target_cpu_datap->halt_status == CPU_NOT_HALTED)) {
1099 continue;
1100 }
1101 dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu_with_state(cpu,
1102 NSEC_PER_SEC, &target_cpu_datap->halt_state);
1103 if ((halt_status < 0) || (halt_status == DBGWRAP_WARN_CPU_OFFLINE)) {
1104 paniclog_append_noflush("Unable to obtain state for cpu %d with status %d: %s\n", cpu, halt_status, ml_dbgwrap_strerror(halt_status));
1105 } else {
1106 paniclog_append_noflush("cpu %d successfully halted\n", cpu);
1107 target_cpu_datap->halt_status = CPU_HALTED_WITH_STATE;
1108 }
1109 }
1110 if (immediate_halt) {
1111 paniclog_append_noflush("Immediate halt requested on all cores\n");
1112 } else {
1113 paniclog_append_noflush("Debugger synchronization timed out; waited %llu nanoseconds\n",
1114 os_atomic_load(&debug_ack_timeout, relaxed));
1115 }
1116 debug_ack_timeout_count++;
1117 return KERN_SUCCESS;
1118 } else {
1119 DebuggerXCallReturn();
1120 return KERN_OPERATION_TIMED_OUT;
1121 }
1122 } else {
1123 return KERN_SUCCESS;
1124 }
1125 }
1126
1127 /*
1128 * @function DebuggerXCallReturn
1129 *
1130 * @abstract Resume normal multicore operation after DebuggerXCallEnter()
1131 *
1132 * @discussion This function should be called with debugger lock held.
1133 */
1134 void
DebuggerXCallReturn(void)1135 DebuggerXCallReturn(
1136 void)
1137 {
1138 cpu_data_t *cpu_data_ptr = getCpuDatap();
1139 uint64_t max_mabs_time, current_mabs_time;
1140
1141 cpu_data_ptr->debugger_active--;
1142 if (cpu_data_ptr->debugger_active != 0) {
1143 return;
1144 }
1145
1146 mp_kdp_trap = 0;
1147 debugger_sync = 0;
1148
1149 max_mabs_time = os_atomic_load(&debug_ack_timeout, relaxed);
1150
1151 if (max_mabs_time > 0) {
1152 current_mabs_time = mach_absolute_time();
1153 max_mabs_time += current_mabs_time;
1154 assert(max_mabs_time > current_mabs_time);
1155 }
1156
1157 /*
1158 * Wait for other CPUs to stop spinning on mp_kdp_trap (see DebuggerXCall).
1159 * It's possible for one or more CPUs to not decrement debug_cpus_spinning,
1160 * since they may be stuck somewhere else with interrupts disabled.
1161 * Wait for DEBUG_ACK_TIMEOUT ns for a response and move on if we don't get it.
1162 *
1163 * Note that the same is done in DebuggerXCallEnter, when we wait for other
1164 * CPUS to update debugger_sync. If we time out, let's hope for all CPUs to be
1165 * spinning in a debugger-safe context
1166 */
1167 while ((os_atomic_load_exclusive(&debug_cpus_spinning, relaxed) != 0) &&
1168 (max_mabs_time == 0 || current_mabs_time < max_mabs_time)) {
1169 __builtin_arm_wfe();
1170 current_mabs_time = mach_absolute_time();
1171 }
1172 os_atomic_clear_exclusive();
1173 }
1174
1175 extern void wait_while_mp_kdp_trap(bool check_SIGPdebug);
1176 /*
1177 * Spin while mp_kdp_trap is set.
1178 *
1179 * processor_offline() calls this with check_SIGPdebug=true
1180 * to break out of the spin loop if the cpu has SIGPdebug
1181 * pending.
1182 */
1183 void
wait_while_mp_kdp_trap(bool check_SIGPdebug)1184 wait_while_mp_kdp_trap(bool check_SIGPdebug)
1185 {
1186 bool found_mp_kdp_trap = false;
1187 bool found_SIGPdebug = false;
1188
1189 while (os_atomic_load_exclusive(&mp_kdp_trap, relaxed) != 0) {
1190 found_mp_kdp_trap = true;
1191 if (check_SIGPdebug && cpu_has_SIGPdebug_pending()) {
1192 found_SIGPdebug = true;
1193 break;
1194 }
1195 __builtin_arm_wfe();
1196 }
1197 os_atomic_clear_exclusive();
1198
1199 if (check_SIGPdebug && found_mp_kdp_trap) {
1200 kprintf("%s>found_mp_kdp_trap=true found_SIGPdebug=%s\n", __FUNCTION__, found_SIGPdebug ? "true" : "false");
1201 }
1202 }
1203
1204 void
DebuggerXCall(void * ctx)1205 DebuggerXCall(
1206 void *ctx)
1207 {
1208 boolean_t save_context = FALSE;
1209 vm_offset_t kstackptr = 0;
1210 arm_saved_state_t *regs = (arm_saved_state_t *) ctx;
1211
1212 if (regs != NULL) {
1213 #if defined(__arm64__)
1214 current_cpu_datap()->ipi_pc = (uint64_t)get_saved_state_pc(regs);
1215 current_cpu_datap()->ipi_lr = (uint64_t)get_saved_state_lr(regs);
1216 current_cpu_datap()->ipi_fp = (uint64_t)get_saved_state_fp(regs);
1217 save_context = PSR64_IS_KERNEL(get_saved_state_cpsr(regs));
1218 #endif
1219 }
1220
1221 kstackptr = (vm_offset_t)current_thread()->machine.kstackptr;
1222
1223 #if defined(__arm64__)
1224 arm_kernel_saved_state_t *state = (arm_kernel_saved_state_t *)kstackptr;
1225
1226 if (save_context) {
1227 /* Save the interrupted context before acknowledging the signal */
1228 current_thread()->machine.kpcb = regs;
1229 } else if (regs) {
1230 /* zero old state so machine_trace_thread knows not to backtrace it */
1231 state->fp = 0;
1232 state->pc_was_in_userspace = true;
1233 state->lr = 0;
1234 state->sp = 0;
1235 state->ssbs = 0;
1236 state->uao = 0;
1237 state->dit = 0;
1238 }
1239 #endif
1240
1241 /*
1242 * When running in serial mode, the core capturing the dump may hold interrupts disabled
1243 * for a time longer than the timeout. That path includes logic to reset the timestamp
1244 * so that we do not eventually trigger the interrupt timeout assert().
1245 *
1246 * Here we check whether other cores have already gone over the timeout at this point
1247 * before spinning, so we at least cover the IPI reception path. After spinning, however,
1248 * we reset the timestamp so as to avoid hitting the interrupt timeout assert().
1249 */
1250 if ((serialmode & SERIALMODE_OUTPUT) || stackshot_active()) {
1251 INTERRUPT_MASKED_DEBUG_END();
1252 }
1253
1254 os_atomic_dec(&debugger_sync, relaxed);
1255
1256
1257 wait_while_mp_kdp_trap(false);
1258
1259 /**
1260 * Alert the triggering CPU that this CPU is done spinning. The CPU that
1261 * signalled all of the other CPUs will wait (in DebuggerXCallReturn) for
1262 * all of the CPUs to exit the above loop before continuing.
1263 */
1264 os_atomic_dec(&debug_cpus_spinning, relaxed);
1265
1266 #if SCHED_HYGIENE_DEBUG
1267 /*
1268 * We also abandon the measurement for preemption disable
1269 * timeouts, if any. Normally, time in interrupt handlers would be
1270 * subtracted from preemption disable time, and this will happen
1271 * up to this point here, but since we here "end" the interrupt
1272 * handler prematurely (from the point of view of interrupt masked
1273 * debugging), the time spinning would otherwise still be
1274 * attributed to preemption disable time, and potentially trigger
1275 * an event, which could be a panic.
1276 */
1277 abandon_preemption_disable_measurement();
1278 #endif /* SCHED_HYGIENE_DEBUG */
1279
1280 if ((serialmode & SERIALMODE_OUTPUT) || stackshot_active()) {
1281 INTERRUPT_MASKED_DEBUG_START(current_thread()->machine.int_handler_addr, current_thread()->machine.int_type);
1282 }
1283
1284 #if defined(__arm64__)
1285 current_thread()->machine.kpcb = NULL;
1286 #endif /* defined(__arm64__) */
1287
1288 /* Any cleanup for our pushed context should go here */
1289 }
1290
1291 void
DebuggerCall(unsigned int reason,void * ctx)1292 DebuggerCall(
1293 unsigned int reason,
1294 void *ctx)
1295 {
1296 #if !MACH_KDP
1297 #pragma unused(reason,ctx)
1298 #endif /* !MACH_KDP */
1299
1300 #if ALTERNATE_DEBUGGER
1301 alternate_debugger_enter();
1302 #endif
1303
1304 #if MACH_KDP
1305 kdp_trap(reason, (struct arm_saved_state *)ctx);
1306 #else
1307 /* TODO: decide what to do if no debugger config */
1308 #endif
1309 }
1310
1311 boolean_t
bootloader_valid_page(ppnum_t ppn)1312 bootloader_valid_page(ppnum_t ppn)
1313 {
1314 return pmap_bootloader_page(ppn);
1315 }
1316