1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #include <mach_assert.h>
58 #include <mach_kdp.h>
59 #include <kdp/kdp.h>
60 #include <kdp/kdp_core.h>
61 #include <kdp/kdp_internal.h>
62 #include <kdp/kdp_callout.h>
63 #include <kern/cpu_number.h>
64 #include <kern/kalloc.h>
65 #include <kern/percpu.h>
66 #include <kern/spl.h>
67 #include <kern/thread.h>
68 #include <kern/assert.h>
69 #include <kern/sched_prim.h>
70 #include <kern/socd_client.h>
71 #include <kern/misc_protos.h>
72 #include <kern/clock.h>
73 #include <kern/telemetry.h>
74 #include <kern/trap_telemetry.h>
75 #include <kern/ecc.h>
76 #include <kern/kern_stackshot.h>
77 #include <kern/kern_cdata.h>
78 #include <kern/zalloc_internal.h>
79 #include <kern/iotrace.h>
80 #include <pexpert/device_tree.h>
81 #include <vm/vm_kern_xnu.h>
82 #include <vm/vm_map.h>
83 #include <vm/pmap.h>
84 #include <vm/vm_compressor_xnu.h>
85 #include <stdarg.h>
86 #include <stdatomic.h>
87 #include <sys/pgo.h>
88 #include <console/serial_protos.h>
89 #include <IOKit/IOBSD.h>
90 #include <libkern/crc.h>
91
92 #if !(MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING)
93 #include <kdp/kdp_udp.h>
94 #endif
95 #include <kern/processor.h>
96
97 #if defined(__i386__) || defined(__x86_64__)
98 #include <IOKit/IOBSD.h>
99
100 #include <i386/cpu_threads.h>
101 #include <i386/pmCPU.h>
102 #include <i386/lbr.h>
103 #endif
104
105 #include <IOKit/IOPlatformExpert.h>
106 #include <machine/machine_cpu.h>
107 #include <machine/pal_routines.h>
108
109 #include <sys/kdebug.h>
110 #include <libkern/OSKextLibPrivate.h>
111 #include <libkern/OSAtomic.h>
112 #include <libkern/kernel_mach_header.h>
113 #include <libkern/section_keywords.h>
114 #include <uuid/uuid.h>
115 #include <mach_debug/zone_info.h>
116 #include <mach/resource_monitors.h>
117 #include <machine/machine_routines.h>
118 #include <sys/proc_require.h>
119 #include <vm/vm_compressor_internal.h>
120
121 #include <os/log_private.h>
122
123 #include <kern/ext_paniclog.h>
124
125 #if defined(__arm64__)
126 #include <pexpert/pexpert.h> /* For gPanicBase */
127 #include <arm/caches_internal.h>
128 #include <arm/misc_protos.h>
129 extern volatile struct xnu_hw_shmem_dbg_command_info *hwsd_info;
130 #endif
131
132 #include <san/kcov.h>
133
134 #if CONFIG_XNUPOST
135 #include <tests/xnupost.h>
136 #endif
137
138 extern int vsnprintf(char *, size_t, const char *, va_list);
139
140 #if CONFIG_CSR
141 #include <sys/csr.h>
142 #endif
143
144 #if CONFIG_EXCLAVES
145 #include <xnuproxy/panic.h>
146 #include "exclaves_panic.h"
147 #endif
148
149 #if CONFIG_SPTM
150 #include <arm64/sptm/sptm.h>
151 #include <arm64/sptm/pmap/pmap_data.h>
152 #endif /* CONFIG_SPTM */
153
154 extern int IODTGetLoaderInfo( const char *key, void **infoAddr, int *infosize );
155 extern void IODTFreeLoaderInfo( const char *key, void *infoAddr, int infoSize );
156 extern unsigned int debug_boot_arg;
157
158 unsigned int halt_in_debugger = 0;
159 unsigned int current_debugger = 0;
160 unsigned int active_debugger = 0;
161 SECURITY_READ_ONLY_LATE(unsigned int) panicDebugging = FALSE;
162 unsigned int kernel_debugger_entry_count = 0;
163
164 #if DEVELOPMENT || DEBUG
165 unsigned int panic_test_failure_mode = PANIC_TEST_FAILURE_MODE_BADPTR;
166 unsigned int panic_test_action_count = 1;
167 unsigned int panic_test_case = PANIC_TEST_CASE_DISABLED;
168 #endif
169
170 #if defined(__arm64__)
171 struct additional_panic_data_buffer *panic_data_buffers = NULL;
172 #endif
173
174 #if defined(__arm64__)
175 /*
176 * Magic number; this should be identical to the armv7 encoding for trap.
177 */
178 #define TRAP_DEBUGGER __asm__ volatile(".long 0xe7ffdeff")
179 #elif defined (__x86_64__)
180 #define TRAP_DEBUGGER __asm__("int3")
181 #else
182 #error No TRAP_DEBUGGER for this architecture
183 #endif
184
185 #if defined(__i386__) || defined(__x86_64__)
186 #define panic_stop() pmCPUHalt(PM_HALT_PANIC)
187 #else
188 #define panic_stop() panic_spin_forever()
189 #endif
190
191 /*
192 * More than enough for any typical format string passed to panic();
193 * anything longer will be truncated but that's better than nothing.
194 */
195 #define EARLY_PANIC_BUFLEN 256
196
197 struct debugger_state {
198 uint64_t db_panic_options;
199 debugger_op db_current_op;
200 boolean_t db_proceed_on_sync_failure;
201 const char *db_message;
202 const char *db_panic_str;
203 va_list *db_panic_args;
204 void *db_panic_data_ptr;
205 unsigned long db_panic_caller;
206 const char *db_panic_initiator;
207 /* incremented whenever we panic or call Debugger (current CPU panic level) */
208 uint32_t db_entry_count;
209 kern_return_t db_op_return;
210 };
211 static struct debugger_state PERCPU_DATA(debugger_state);
212 struct kernel_panic_reason PERCPU_DATA(panic_reason);
213
214 /* __pure2 is correct if this function is called with preemption disabled */
215 static inline __pure2 struct debugger_state *
current_debugger_state(void)216 current_debugger_state(void)
217 {
218 return PERCPU_GET(debugger_state);
219 }
220
221 #define CPUDEBUGGEROP current_debugger_state()->db_current_op
222 #define CPUDEBUGGERMSG current_debugger_state()->db_message
223 #define CPUPANICSTR current_debugger_state()->db_panic_str
224 #define CPUPANICARGS current_debugger_state()->db_panic_args
225 #define CPUPANICOPTS current_debugger_state()->db_panic_options
226 #define CPUPANICDATAPTR current_debugger_state()->db_panic_data_ptr
227 #define CPUDEBUGGERSYNC current_debugger_state()->db_proceed_on_sync_failure
228 #define CPUDEBUGGERCOUNT current_debugger_state()->db_entry_count
229 #define CPUDEBUGGERRET current_debugger_state()->db_op_return
230 #define CPUPANICCALLER current_debugger_state()->db_panic_caller
231 #define CPUPANICINITIATOR current_debugger_state()->db_panic_initiator
232
233
234 /*
235 * Usage:
236 * panic_test_action_count is in the context of other flags, e.g. for IO errors it is "succeed this many times then fail" and for nesting it is "panic this many times then succeed"
237 * panic_test_failure_mode is a bit map of things to do
238 * panic_test_case is what sort of test we are injecting
239 *
240 * For more details see definitions in debugger.h
241 *
242 * Note that not all combinations are sensible, but some actions can be combined, e.g.
243 * - BADPTR+SPIN with action count = 3 will cause panic->panic->spin
244 * - BADPTR with action count = 2 will cause 2 nested panics (in addition to the initial panic)
245 * - IO_ERR with action 15 will cause 14 successful IOs, then fail on the next one
246 */
247 #if DEVELOPMENT || DEBUG
248 #define INJECT_NESTED_PANIC_IF_REQUESTED(requested) \
249 MACRO_BEGIN \
250 if ((panic_test_case & requested) && panic_test_action_count) { \
251 panic_test_action_count--; \
252 volatile int *panic_test_badpointer = (int *)4; \
253 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_SPIN) && (!panic_test_action_count)) { printf("inject spin...\n"); while(panic_test_badpointer); } \
254 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_BADPTR) && (panic_test_action_count+1)) { printf("inject badptr...\n"); *panic_test_badpointer = 0; } \
255 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_PANIC) && (panic_test_action_count+1)) { printf("inject panic...\n"); panic("nested panic level %d", panic_test_action_count); } \
256 } \
257 MACRO_END
258
259 #endif /* DEVELOPMENT || DEBUG */
260
261 debugger_op debugger_current_op = DBOP_NONE;
262 const char *debugger_panic_str = NULL;
263 va_list *debugger_panic_args = NULL;
264 void *debugger_panic_data = NULL;
265 uint64_t debugger_panic_options = 0;
266 const char *debugger_message = NULL;
267 unsigned long debugger_panic_caller = 0;
268 const char *debugger_panic_initiator = "";
269
270 void panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args,
271 unsigned int reason, void *ctx, uint64_t panic_options_mask, void *panic_data,
272 unsigned long panic_caller, const char *panic_initiator) __dead2 __printflike(1, 0);
273 static void kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags);
274 void panic_spin_forever(void) __dead2;
275 void panic_stackshot_release_lock(void);
276 extern void PE_panic_hook(const char*);
277 extern int sync_internal(void);
278
279 #define NESTEDDEBUGGERENTRYMAX 5
280 static TUNABLE(unsigned int, max_debugger_entry_count, "nested_panic_max",
281 NESTEDDEBUGGERENTRYMAX);
282
283 SECURITY_READ_ONLY_LATE(bool) awl_scratch_reg_supported = false;
284 static bool PERCPU_DATA(hv_entry_detected); // = false
285 static void awl_set_scratch_reg_hv_bit(void);
286 void awl_mark_hv_entry(void);
287 static bool awl_pm_state_change_cbk(void *param, enum cpu_event event, unsigned int cpu_or_cluster);
288
289 #if !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING
290 static boolean_t device_corefile_valid_on_ephemeral(void);
291 #endif /* !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING */
292
293 #if defined(__arm64__)
294 #define DEBUG_BUF_SIZE (4096)
295
296 /* debug_buf is directly linked with iBoot panic region for arm targets */
297 char *debug_buf_base = NULL;
298 char *debug_buf_ptr = NULL;
299 unsigned int debug_buf_size = 0;
300
301 SECURITY_READ_ONLY_LATE(boolean_t) kdp_explicitly_requested = FALSE;
302 #else /* defined(__arm64__) */
303 #define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data))
304 /* EXTENDED_DEBUG_BUF_SIZE definition is now in debug.h */
305 static_assert(((EXTENDED_DEBUG_BUF_SIZE % PANIC_FLUSH_BOUNDARY) == 0), "Extended debug buf size must match SMC alignment requirements");
306
307 char debug_buf[DEBUG_BUF_SIZE];
308 struct macos_panic_header *panic_info = (struct macos_panic_header *)debug_buf;
309 char *debug_buf_base = (debug_buf + offsetof(struct macos_panic_header, mph_data));
310 char *debug_buf_ptr = (debug_buf + offsetof(struct macos_panic_header, mph_data));
311
312 /*
313 * We don't include the size of the panic header in the length of the data we actually write.
314 * On co-processor platforms, we lose sizeof(struct macos_panic_header) bytes from the end of
315 * the end of the log because we only support writing (3*PAGESIZE) bytes.
316 */
317 unsigned int debug_buf_size = (DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));
318
319 boolean_t extended_debug_log_enabled = FALSE;
320 #endif /* defined(__arm64__) */
321
322 #if defined(XNU_TARGET_OS_OSX)
323 #define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace"
324 #else
325 #define KDBG_TRACE_PANIC_FILENAME "/var/log/panic.trace"
326 #endif
327
328 static inline boolean_t debug_fatal_panic_begin(void);
329
330 /* Debugger state */
331 atomic_int debugger_cpu = DEBUGGER_NO_CPU;
332 boolean_t debugger_allcpus_halted = FALSE;
333 boolean_t debugger_safe_to_return = TRUE;
334 unsigned int debugger_context = 0;
335
336 static char model_name[64];
337 unsigned char *kernel_uuid;
338
339 boolean_t kernelcache_uuid_valid = FALSE;
340 uuid_t kernelcache_uuid;
341 uuid_string_t kernelcache_uuid_string;
342
343 boolean_t pageablekc_uuid_valid = FALSE;
344 uuid_t pageablekc_uuid;
345 uuid_string_t pageablekc_uuid_string;
346
347 boolean_t auxkc_uuid_valid = FALSE;
348 uuid_t auxkc_uuid;
349 uuid_string_t auxkc_uuid_string;
350
351
352 /*
353 * By default we treat Debugger() the same as calls to panic(), unless
354 * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set.
355 * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported.
356 *
357 * Return from Debugger() is currently only implemented on x86
358 */
359 static boolean_t debugger_is_panic = TRUE;
360
361 TUNABLE(unsigned int, debug_boot_arg, "debug", 0);
362
363 TUNABLE_DEV_WRITEABLE(unsigned int, verbose_panic_flow_logging, "verbose_panic_flow_logging", 0);
364
365 char kernel_uuid_string[37]; /* uuid_string_t */
366 char kernelcache_uuid_string[37]; /* uuid_string_t */
367 char panic_disk_error_description[512];
368 size_t panic_disk_error_description_size = sizeof(panic_disk_error_description);
369
370 extern unsigned int write_trace_on_panic;
371 int kext_assertions_enable =
372 #if DEBUG || DEVELOPMENT
373 TRUE;
374 #else
375 FALSE;
376 #endif
377
378 /*
379 * Maintain the physically-contiguous carveouts for the carveout bootargs.
380 */
381 TUNABLE_WRITEABLE(boolean_t, phys_carveout_core, "phys_carveout_core", 1);
382
383 TUNABLE(uint32_t, phys_carveout_mb, "phys_carveout_mb", 0);
384 SECURITY_READ_ONLY_LATE(vm_offset_t) phys_carveout = 0;
385 SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout_pa = 0;
386 SECURITY_READ_ONLY_LATE(size_t) phys_carveout_size = 0;
387
388
389 #if CONFIG_SPTM && (DEVELOPMENT || DEBUG)
390 /**
391 * Extra debug state which is set when panic lockdown is initiated.
392 * This information is intended to help when debugging issues with the panic
393 * path.
394 */
395 struct panic_lockdown_initiator_state debug_panic_lockdown_initiator_state;
396 #endif /* CONFIG_SPTM && (DEVELOPMENT || DEBUG) */
397
398 /*
399 * Returns whether kernel debugging is expected to be restricted
400 * on the device currently based on CSR or other platform restrictions.
401 */
402 boolean_t
kernel_debugging_restricted(void)403 kernel_debugging_restricted(void)
404 {
405 #if XNU_TARGET_OS_OSX
406 #if CONFIG_CSR
407 if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) != 0) {
408 return TRUE;
409 }
410 #endif /* CONFIG_CSR */
411 return FALSE;
412 #else /* XNU_TARGET_OS_OSX */
413 return FALSE;
414 #endif /* XNU_TARGET_OS_OSX */
415 }
416
417 __startup_func
418 static void
panic_init(void)419 panic_init(void)
420 {
421 unsigned long uuidlen = 0;
422 void *uuid;
423
424 uuid = getuuidfromheader(&_mh_execute_header, &uuidlen);
425 if ((uuid != NULL) && (uuidlen == sizeof(uuid_t))) {
426 kernel_uuid = uuid;
427 uuid_unparse_upper(*(uuid_t *)uuid, kernel_uuid_string);
428 }
429
430 /*
431 * Take the value of the debug boot-arg into account
432 */
433 #if MACH_KDP
434 if (!kernel_debugging_restricted() && debug_boot_arg) {
435 if (debug_boot_arg & DB_HALT) {
436 halt_in_debugger = 1;
437 }
438
439 #if defined(__arm64__)
440 if (debug_boot_arg & DB_NMI) {
441 panicDebugging = TRUE;
442 }
443 #else
444 panicDebugging = TRUE;
445 #endif /* defined(__arm64__) */
446 }
447
448 #if defined(__arm64__)
449 char kdpname[80];
450
451 kdp_explicitly_requested = PE_parse_boot_argn("kdp_match_name", kdpname, sizeof(kdpname));
452 #endif /* defined(__arm64__) */
453
454 #endif /* MACH_KDP */
455
456 #if defined (__x86_64__)
457 /*
458 * By default we treat Debugger() the same as calls to panic(), unless
459 * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set.
460 * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported.
461 * This is because writing an on-device corefile is a destructive operation.
462 *
463 * Return from Debugger() is currently only implemented on x86
464 */
465 if (PE_i_can_has_debugger(NULL) && !(debug_boot_arg & DB_KERN_DUMP_ON_NMI)) {
466 debugger_is_panic = FALSE;
467 }
468 #endif
469 }
470 STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, panic_init);
471
472 #if defined (__x86_64__)
473 void
extended_debug_log_init(void)474 extended_debug_log_init(void)
475 {
476 assert(coprocessor_paniclog_flush);
477 /*
478 * Allocate an extended panic log buffer that has space for the panic
479 * stackshot at the end. Update the debug buf pointers appropriately
480 * to point at this new buffer.
481 *
482 * iBoot pre-initializes the panic region with the NULL character. We set this here
483 * so we can accurately calculate the CRC for the region without needing to flush the
484 * full region over SMC.
485 */
486 char *new_debug_buf = kalloc_data(EXTENDED_DEBUG_BUF_SIZE, Z_WAITOK | Z_ZERO);
487
488 panic_info = (struct macos_panic_header *)new_debug_buf;
489 debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data));
490 debug_buf_size = (EXTENDED_DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));
491
492 extended_debug_log_enabled = TRUE;
493
494 /*
495 * Insert a compiler barrier so we don't free the other panic stackshot buffer
496 * until after we've marked the new one as available
497 */
498 __compiler_barrier();
499 kmem_free(kernel_map, panic_stackshot_buf, panic_stackshot_buf_len);
500 panic_stackshot_buf = 0;
501 panic_stackshot_buf_len = 0;
502 }
503 #endif /* defined (__x86_64__) */
504
505 void
debug_log_init(void)506 debug_log_init(void)
507 {
508 #if defined(__arm64__)
509 if (!gPanicBase) {
510 printf("debug_log_init: Error!! gPanicBase is still not initialized\n");
511 return;
512 }
513 /* Shift debug buf start location and size by the length of the panic header */
514 debug_buf_base = (char *)gPanicBase + sizeof(struct embedded_panic_header);
515 debug_buf_ptr = debug_buf_base;
516 debug_buf_size = gPanicSize - sizeof(struct embedded_panic_header);
517
518 #if CONFIG_EXT_PANICLOG
519 ext_paniclog_init();
520 #endif
521 #else
522 kern_return_t kr = KERN_SUCCESS;
523 bzero(panic_info, DEBUG_BUF_SIZE);
524
525 assert(debug_buf_base != NULL);
526 assert(debug_buf_ptr != NULL);
527 assert(debug_buf_size != 0);
528
529 /*
530 * We allocate a buffer to store a panic time stackshot. If we later discover that this is a
531 * system that supports flushing a stackshot via an extended debug log (see above), we'll free this memory
532 * as it's not necessary on this platform. This information won't be available until the IOPlatform has come
533 * up.
534 */
535 kr = kmem_alloc(kernel_map, &panic_stackshot_buf, PANIC_STACKSHOT_BUFSIZE,
536 KMA_DATA_SHARED | KMA_ZERO, VM_KERN_MEMORY_DIAG);
537 assert(kr == KERN_SUCCESS);
538 if (kr == KERN_SUCCESS) {
539 panic_stackshot_buf_len = PANIC_STACKSHOT_BUFSIZE;
540 }
541 #endif
542 }
543
544 void
phys_carveout_init(void)545 phys_carveout_init(void)
546 {
547 if (!PE_i_can_has_debugger(NULL)) {
548 return;
549 }
550
551 #if __arm__ || __arm64__
552 #if DEVELOPMENT || DEBUG
553 #endif /* DEVELOPMENT || DEBUG */
554 #endif /* __arm__ || __arm64__ */
555
556 struct carveout {
557 const char *name;
558 vm_offset_t *va;
559 uint32_t requested_size;
560 uintptr_t *pa;
561 size_t *allocated_size;
562 uint64_t present;
563 } carveouts[] = {
564 {
565 "phys_carveout",
566 &phys_carveout,
567 phys_carveout_mb,
568 &phys_carveout_pa,
569 &phys_carveout_size,
570 phys_carveout_mb != 0,
571
572 /* Before Donan, XNU allocates the panic-trace carveout. */
573 }
574 };
575
576 for (int i = 0; i < (sizeof(carveouts) / sizeof(struct carveout)); i++) {
577 if (carveouts[i].present) {
578 size_t temp_carveout_size = 0;
579 if (os_mul_overflow(carveouts[i].requested_size, 1024 * 1024, &temp_carveout_size)) {
580 panic("%s_mb size overflowed (%uMB)",
581 carveouts[i].name, carveouts[i].requested_size);
582 return;
583 }
584
585 kmem_alloc_contig(kernel_map, carveouts[i].va,
586 temp_carveout_size, PAGE_MASK, 0, 0,
587 KMA_NOFAIL | KMA_PERMANENT | KMA_NOPAGEWAIT | KMA_DATA_SHARED |
588 KMA_NOSOFTLIMIT,
589 VM_KERN_MEMORY_DIAG);
590
591 *carveouts[i].pa = kvtophys(*carveouts[i].va);
592 *carveouts[i].allocated_size = temp_carveout_size;
593 }
594 }
595 }
596
597 boolean_t
debug_is_in_phys_carveout(vm_map_offset_t va)598 debug_is_in_phys_carveout(vm_map_offset_t va)
599 {
600 return phys_carveout_size && va >= phys_carveout &&
601 va < (phys_carveout + phys_carveout_size);
602 }
603
604 boolean_t
debug_can_coredump_phys_carveout(void)605 debug_can_coredump_phys_carveout(void)
606 {
607 return phys_carveout_core;
608 }
609
610 static boolean_t
DebuggerLock(void)611 DebuggerLock(void)
612 {
613 int my_cpu = cpu_number();
614 int debugger_exp_cpu = DEBUGGER_NO_CPU;
615 assert(ml_get_interrupts_enabled() == FALSE);
616
617 if (atomic_load(&debugger_cpu) == my_cpu) {
618 return true;
619 }
620
621 if (!atomic_compare_exchange_strong(&debugger_cpu, &debugger_exp_cpu, my_cpu)) {
622 return false;
623 }
624
625 return true;
626 }
627
628 static void
DebuggerUnlock(void)629 DebuggerUnlock(void)
630 {
631 assert(atomic_load_explicit(&debugger_cpu, memory_order_relaxed) == cpu_number());
632
633 /*
634 * We don't do an atomic exchange here in case
635 * there's another CPU spinning to acquire the debugger_lock
636 * and we never get a chance to update it. We already have the
637 * lock so we can simply store DEBUGGER_NO_CPU and follow with
638 * a barrier.
639 */
640 atomic_store(&debugger_cpu, DEBUGGER_NO_CPU);
641 OSMemoryBarrier();
642
643 return;
644 }
645
646 static kern_return_t
DebuggerHaltOtherCores(boolean_t proceed_on_failure,bool is_stackshot)647 DebuggerHaltOtherCores(boolean_t proceed_on_failure, bool is_stackshot)
648 {
649 #if defined(__arm64__)
650 return DebuggerXCallEnter(proceed_on_failure, is_stackshot);
651 #else /* defined(__arm64__) */
652 #pragma unused(proceed_on_failure)
653 mp_kdp_enter(proceed_on_failure, is_stackshot);
654 return KERN_SUCCESS;
655 #endif
656 }
657
658 static void
DebuggerResumeOtherCores(void)659 DebuggerResumeOtherCores(void)
660 {
661 #if defined(__arm64__)
662 DebuggerXCallReturn();
663 #else /* defined(__arm64__) */
664 mp_kdp_exit();
665 #endif
666 }
667
668 __printflike(3, 0)
669 static void
DebuggerSaveState(debugger_op db_op,const char * db_message,const char * db_panic_str,va_list * db_panic_args,uint64_t db_panic_options,void * db_panic_data_ptr,boolean_t db_proceed_on_sync_failure,unsigned long db_panic_caller,const char * db_panic_initiator)670 DebuggerSaveState(debugger_op db_op, const char *db_message, const char *db_panic_str,
671 va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
672 boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller, const char *db_panic_initiator)
673 {
674 CPUDEBUGGEROP = db_op;
675
676 /*
677 * Note:
678 * if CPUDEBUGGERCOUNT == 1 then we are in the normal case - record the panic data
679 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR == NULL then we are in a nested panic that happened before DebuggerSaveState was called, so store the nested panic data
680 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR != NULL then we are in a nested panic that happened after DebuggerSaveState was called, so leave the original panic data
681 *
682 * TODO: is it safe to flatten this to if (CPUPANICSTR == NULL)?
683 */
684 if (CPUDEBUGGERCOUNT == 1 || CPUPANICSTR == NULL) {
685 CPUDEBUGGERMSG = db_message;
686 CPUPANICSTR = db_panic_str;
687 CPUPANICARGS = db_panic_args;
688 CPUPANICDATAPTR = db_panic_data_ptr;
689 CPUPANICCALLER = db_panic_caller;
690 CPUPANICINITIATOR = db_panic_initiator;
691
692 #if CONFIG_EXCLAVES
693 char *panic_str;
694 if (exclaves_panic_get_string(&panic_str) == KERN_SUCCESS) {
695 CPUPANICSTR = panic_str;
696 }
697 #endif
698 }
699
700 CPUDEBUGGERSYNC = db_proceed_on_sync_failure;
701 CPUDEBUGGERRET = KERN_SUCCESS;
702
703 /* Reset these on any nested panics */
704 // follow up in rdar://88497308 (nested panics should not clobber panic flags)
705 CPUPANICOPTS = db_panic_options;
706
707 return;
708 }
709
710 /*
711 * Save the requested debugger state/action into the current processor's
712 * percu state and trap to the debugger.
713 */
714 kern_return_t
DebuggerTrapWithState(debugger_op db_op,const char * db_message,const char * db_panic_str,va_list * db_panic_args,uint64_t db_panic_options,void * db_panic_data_ptr,boolean_t db_proceed_on_sync_failure,unsigned long db_panic_caller,const char * db_panic_initiator)715 DebuggerTrapWithState(debugger_op db_op, const char *db_message, const char *db_panic_str,
716 va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
717 boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller, const char* db_panic_initiator)
718 {
719 kern_return_t ret;
720
721 #if defined(__arm64__) && (DEVELOPMENT || DEBUG)
722 if (!PE_arm_debug_and_trace_initialized()) {
723 /*
724 * In practice this can only happen if we panicked very early,
725 * when only the boot CPU is online and before it has finished
726 * initializing the debug and trace infrastructure. We're going
727 * to hang soon, so let's at least make sure the message passed
728 * to panic() is actually logged.
729 */
730 char buf[EARLY_PANIC_BUFLEN];
731 vsnprintf(buf, EARLY_PANIC_BUFLEN, db_panic_str, *db_panic_args);
732 paniclog_append_noflush("%s\n", buf);
733 }
734 #endif
735
736 assert(ml_get_interrupts_enabled() == FALSE);
737 DebuggerSaveState(db_op, db_message, db_panic_str, db_panic_args,
738 db_panic_options, db_panic_data_ptr,
739 db_proceed_on_sync_failure, db_panic_caller, db_panic_initiator);
740
741 /*
742 * On ARM this generates an uncategorized exception -> sleh code ->
743 * DebuggerCall -> kdp_trap -> handle_debugger_trap
744 * So that is how XNU ensures that only one core can panic.
745 * The rest of the cores are halted by IPI if possible; if that
746 * fails it will fall back to dbgwrap.
747 */
748 TRAP_DEBUGGER;
749
750 ret = CPUDEBUGGERRET;
751
752 DebuggerSaveState(DBOP_NONE, NULL, NULL, NULL, 0, NULL, FALSE, 0, NULL);
753
754 return ret;
755 }
756
757 void __attribute__((noinline))
Assert(const char * file,int line,const char * expression)758 Assert(const char*file, int line, const char *expression)
759 {
760 panic_plain("%s:%d Assertion failed: %s", file, line, expression);
761 }
762
763 void
panic_assert_format(char * buf,size_t len,struct mach_assert_hdr * hdr,long a,long b)764 panic_assert_format(char *buf, size_t len, struct mach_assert_hdr *hdr, long a, long b)
765 {
766 struct mach_assert_default *adef;
767 struct mach_assert_3x *a3x;
768
769 static_assert(MACH_ASSERT_TRAP_CODE == XNU_HARD_TRAP_ASSERT_FAILURE);
770
771 switch (hdr->type) {
772 case MACH_ASSERT_DEFAULT:
773 adef = __container_of(hdr, struct mach_assert_default, hdr);
774 snprintf(buf, len, "%s:%d Assertion failed: %s",
775 hdr->filename, hdr->lineno, adef->expr);
776 break;
777
778 case MACH_ASSERT_3P:
779 a3x = __container_of(hdr, struct mach_assert_3x, hdr);
780 snprintf(buf, len, "%s:%d Assertion failed: "
781 "%s %s %s (%p %s %p)",
782 hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
783 (void *)a, a3x->op, (void *)b);
784 break;
785
786 case MACH_ASSERT_3S:
787 a3x = __container_of(hdr, struct mach_assert_3x, hdr);
788 snprintf(buf, len, "%s:%d Assertion failed: "
789 "%s %s %s (0x%lx %s 0x%lx, %ld %s %ld)",
790 hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
791 a, a3x->op, b, a, a3x->op, b);
792 break;
793
794 case MACH_ASSERT_3U:
795 a3x = __container_of(hdr, struct mach_assert_3x, hdr);
796 snprintf(buf, len, "%s:%d Assertion failed: "
797 "%s %s %s (0x%lx %s 0x%lx, %lu %s %lu)",
798 hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
799 a, a3x->op, b, a, a3x->op, b);
800 break;
801 }
802 }
803
804 boolean_t
debug_is_current_cpu_in_panic_state(void)805 debug_is_current_cpu_in_panic_state(void)
806 {
807 return current_debugger_state()->db_entry_count > 0;
808 }
809
810 #if defined(__arm64__)
811 /*
812 * Helper function to compute kernel text exec slide and base values
813 */
814 static void
get_kernel_text_exec_slide_and_base(unsigned long * exec_slide,unsigned long * exec_base)815 get_kernel_text_exec_slide_and_base(unsigned long *exec_slide, unsigned long *exec_base)
816 {
817 extern vm_offset_t segTEXTEXECB, vm_kernel_slide;
818 void *kch = PE_get_kc_header(KCKindPrimary);
819
820 if (kch && exec_slide && exec_base) {
821 *exec_slide = (unsigned long)segTEXTEXECB - (unsigned long)kch + vm_kernel_slide;
822 *exec_base = (unsigned long)segTEXTEXECB;
823 }
824 }
825 #endif /* defined(__arm64__) */
826
827 /*
828 * check if we are in a nested panic, report findings, take evasive action where necessary
829 *
830 * see also PE_update_panicheader_nestedpanic
831 */
832 static void
check_and_handle_nested_panic(uint64_t panic_options_mask,unsigned long panic_caller,const char * db_panic_str,va_list * db_panic_args)833 check_and_handle_nested_panic(uint64_t panic_options_mask, unsigned long panic_caller, const char *db_panic_str, va_list *db_panic_args)
834 {
835 if ((CPUDEBUGGERCOUNT > 1) && (CPUDEBUGGERCOUNT < max_debugger_entry_count)) {
836 // Note: this is the first indication in the panic log or serial that we are off the rails...
837 //
838 // if we panic *before* the paniclog is finalized then this will end up in the ips report with a panic_caller addr that gives us a clue
839 // if we panic *after* the log is finalized then we will only see it in the serial log
840 //
841 paniclog_append_noflush("Nested panic detected - entry count: %d panic_caller: 0x%016lx\n", CPUDEBUGGERCOUNT, panic_caller);
842
843 #if defined(__arm64__)
844 // Print kernel slide and base information for nested panics in order to enable symbolication
845 unsigned long kernel_text_exec_slide = 0, kernel_text_exec_base = 0;
846 get_kernel_text_exec_slide_and_base(&kernel_text_exec_slide, &kernel_text_exec_base);
847 paniclog_append_noflush("Kernel text exec slide: 0x%016lx\n", kernel_text_exec_slide);
848 paniclog_append_noflush("Kernel text exec base: 0x%016lx\n", kernel_text_exec_base);
849 #endif /* defined(__arm64__) */
850 print_curr_backtrace();
851 paniclog_flush();
852
853 // print the *new* panic string to the console, we might not get it by other means...
854 // TODO: I tried to write this stuff to the paniclog, but the serial output gets corrupted and the panicstring in the ips file is <mysterious>
855 // rdar://87846117 (NestedPanic: output panic string to paniclog)
856 if (db_panic_str) {
857 printf("Nested panic string:\n");
858 #pragma clang diagnostic push
859 #pragma clang diagnostic ignored "-Wformat-nonliteral"
860 #pragma clang diagnostic ignored "-Wformat"
861 _doprnt(db_panic_str, db_panic_args, PE_kputc, 0);
862 #pragma clang diagnostic pop
863 printf("\n<end nested panic string>\n");
864 }
865 }
866
867 // Stage 1 bailout
868 //
869 // Try to complete the normal panic flow, i.e. try to make sure the callouts happen and we flush the paniclog. If this fails with another nested
870 // panic then we will land in Stage 2 below...
871 //
872 if (CPUDEBUGGERCOUNT == max_debugger_entry_count) {
873 uint32_t panic_details = 0;
874
875 // if this is a force-reset panic then capture a log and reboot immediately.
876 if (panic_options_mask & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
877 panic_details |= kPanicDetailsForcePowerOff;
878 }
879
880 // normally the kPEPanicBegin is sent from debugger_collect_diagnostics(), but we might nested-panic before we get
881 // there. To be safe send another notification, the function called below will only send kPEPanicBegin if it has not yet been sent.
882 //
883 PEHaltRestartInternal(kPEPanicBegin, panic_details);
884
885 paniclog_append_noflush("Nested panic count exceeds limit %d, machine will reset or spin\n", max_debugger_entry_count);
886 PE_update_panicheader_nestedpanic();
887 paniclog_flush();
888
889 if (!panicDebugging) {
890 // note that this will also send kPEPanicEnd
891 kdp_machine_reboot_type(kPEPanicRestartCPU, panic_options_mask);
892 }
893
894 // prints to console
895 paniclog_append_noflush("\nNested panic stall. Stage 1 bailout. Please go to https://panic.apple.com to report this panic\n");
896 panic_spin_forever();
897 }
898
899 // Stage 2 bailout
900 //
901 // Things are severely hosed, we have nested to the point of bailout and then nested again during the bailout path. Try to issue
902 // a chipreset as quickly as possible, hopefully something in the panic log is salvageable, since we flushed it during Stage 1.
903 //
904 if (CPUDEBUGGERCOUNT == max_debugger_entry_count + 1) {
905 if (!panicDebugging) {
906 // note that:
907 // - this code path should be audited for prints, as that is a common cause of nested panics
908 // - this code path should take the fastest route to the actual reset, and not call any un-necessary code
909 kdp_machine_reboot_type(kPEPanicRestartCPU, panic_options_mask & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS);
910 }
911
912 // prints to console, but another nested panic will land in Stage 3 where we simply spin, so that is sort of ok...
913 paniclog_append_noflush("\nIn Nested panic stall. Stage 2 bailout. Please go to https://panic.apple.com to report this panic\n");
914 panic_spin_forever();
915 }
916
917 // Stage 3 bailout
918 //
919 // We are done here, we were unable to reset the platform without another nested panic. Spin until the watchdog kicks in.
920 //
921 if (CPUDEBUGGERCOUNT > max_debugger_entry_count + 1) {
922 kdp_machine_reboot_type(kPEHangCPU, 0);
923 }
924 }
925
926 void
Debugger(const char * message)927 Debugger(const char *message)
928 {
929 DebuggerWithContext(0, NULL, message, DEBUGGER_OPTION_NONE, (unsigned long)(char *)__builtin_return_address(0));
930 }
931
932 /*
933 * Enter the Debugger
934 *
935 * This is similar to, but not the same as a panic
936 *
937 * Key differences:
938 * - we get here from a debugger entry action (e.g. NMI)
939 * - the system is resumable on x86 (in theory, however it is not clear if this is tested)
940 * - rdar://57738811 (xnu: support resume from debugger via KDP on arm devices)
941 *
942 */
943 void
DebuggerWithContext(unsigned int reason,void * ctx,const char * message,uint64_t debugger_options_mask,unsigned long debugger_caller)944 DebuggerWithContext(unsigned int reason, void *ctx, const char *message,
945 uint64_t debugger_options_mask, unsigned long debugger_caller)
946 {
947 spl_t previous_interrupts_state;
948 boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers;
949
950 #if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
951 read_lbr();
952 #endif
953 previous_interrupts_state = ml_set_interrupts_enabled(FALSE);
954 disable_preemption();
955
956 /* track depth of debugger/panic entry */
957 CPUDEBUGGERCOUNT++;
958
959 /* emit a tracepoint as early as possible in case of hang */
960 SOCD_TRACE_XNU(PANIC,
961 ((CPUDEBUGGERCOUNT <= 2) ? SOCD_TRACE_MODE_STICKY_TRACEPOINT : SOCD_TRACE_MODE_NONE),
962 PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)),
963 VALUE(debugger_options_mask),
964 ADDR(message),
965 ADDR(debugger_caller));
966
967 /* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
968 check_and_handle_nested_panic(debugger_options_mask, debugger_caller, message, NULL);
969
970 /* Handle any necessary platform specific actions before we proceed */
971 PEInitiatePanic();
972
973 #if DEVELOPMENT || DEBUG
974 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_ENTRY);
975 #endif
976
977 PE_panic_hook(message);
978
979 doprnt_hide_pointers = FALSE;
980
981 if (ctx != NULL) {
982 DebuggerSaveState(DBOP_DEBUGGER, message,
983 NULL, NULL, debugger_options_mask, NULL, TRUE, 0, "");
984 handle_debugger_trap(reason, 0, 0, ctx);
985 DebuggerSaveState(DBOP_NONE, NULL, NULL,
986 NULL, 0, NULL, FALSE, 0, "");
987 } else {
988 DebuggerTrapWithState(DBOP_DEBUGGER, message,
989 NULL, NULL, debugger_options_mask, NULL, TRUE, 0, NULL);
990 }
991
992 /* resume from the debugger */
993
994 CPUDEBUGGERCOUNT--;
995 doprnt_hide_pointers = old_doprnt_hide_pointers;
996 enable_preemption();
997 ml_set_interrupts_enabled(previous_interrupts_state);
998 }
999
1000 static struct kdp_callout {
1001 struct kdp_callout * callout_next;
1002 kdp_callout_fn_t callout_fn;
1003 boolean_t callout_in_progress;
1004 void * callout_arg;
1005 } * kdp_callout_list = NULL;
1006
1007 /*
1008 * Called from kernel context to register a kdp event callout.
1009 */
1010 void
kdp_register_callout(kdp_callout_fn_t fn,void * arg)1011 kdp_register_callout(kdp_callout_fn_t fn, void * arg)
1012 {
1013 struct kdp_callout * kcp;
1014 struct kdp_callout * list_head;
1015
1016 kcp = zalloc_permanent_type(struct kdp_callout);
1017
1018 kcp->callout_fn = fn;
1019 kcp->callout_arg = arg;
1020 kcp->callout_in_progress = FALSE;
1021
1022 /* Lock-less list insertion using compare and exchange. */
1023 do {
1024 list_head = kdp_callout_list;
1025 kcp->callout_next = list_head;
1026 } while (!OSCompareAndSwapPtr(list_head, kcp, &kdp_callout_list));
1027 }
1028
1029 static void
kdp_callouts(kdp_event_t event)1030 kdp_callouts(kdp_event_t event)
1031 {
1032 struct kdp_callout *kcp = kdp_callout_list;
1033
1034 while (kcp) {
1035 if (!kcp->callout_in_progress) {
1036 kcp->callout_in_progress = TRUE;
1037 kcp->callout_fn(kcp->callout_arg, event);
1038 kcp->callout_in_progress = FALSE;
1039 }
1040 kcp = kcp->callout_next;
1041 }
1042 }
1043
1044 #if defined(__arm64__)
1045 /*
1046 * Register an additional buffer with data to include in the panic log
1047 *
1048 * <rdar://problem/50137705> tracks supporting more than one buffer
1049 *
1050 * Note that producer_name and buf should never be de-allocated as we reference these during panic.
1051 */
1052 void
register_additional_panic_data_buffer(const char * producer_name,void * buf,int len)1053 register_additional_panic_data_buffer(const char *producer_name, void *buf, int len)
1054 {
1055 if (panic_data_buffers != NULL) {
1056 panic("register_additional_panic_data_buffer called with buffer already registered");
1057 }
1058
1059 if (producer_name == NULL || (strlen(producer_name) == 0)) {
1060 panic("register_additional_panic_data_buffer called with invalid producer_name");
1061 }
1062
1063 if (buf == NULL) {
1064 panic("register_additional_panic_data_buffer called with invalid buffer pointer");
1065 }
1066
1067 if ((len <= 0) || (len > ADDITIONAL_PANIC_DATA_BUFFER_MAX_LEN)) {
1068 panic("register_additional_panic_data_buffer called with invalid length");
1069 }
1070
1071 struct additional_panic_data_buffer *new_panic_data_buffer = zalloc_permanent_type(struct additional_panic_data_buffer);
1072 new_panic_data_buffer->producer_name = producer_name;
1073 new_panic_data_buffer->buf = buf;
1074 new_panic_data_buffer->len = len;
1075
1076 if (!OSCompareAndSwapPtr(NULL, new_panic_data_buffer, &panic_data_buffers)) {
1077 panic("register_additional_panic_data_buffer called with buffer already registered");
1078 }
1079
1080 return;
1081 }
1082 #endif /* defined(__arm64__) */
1083
1084 /*
1085 * An overview of the xnu panic path:
1086 *
1087 * Several panic wrappers (panic(), panic_with_options(), etc.) all funnel into panic_trap_to_debugger().
1088 * panic_trap_to_debugger() sets the panic state in the current processor's debugger_state prior
1089 * to trapping into the debugger. Once we trap to the debugger, we end up in handle_debugger_trap()
1090 * which tries to acquire the panic lock by atomically swapping the current CPU number into debugger_cpu.
1091 * debugger_cpu acts as a synchronization point, from which the winning CPU can halt the other cores and
1092 * continue to debugger_collect_diagnostics() where we write the paniclog, corefile (if appropriate) and proceed
1093 * according to the device's boot-args.
1094 */
1095 #undef panic
1096 void
panic(const char * str,...)1097 panic(const char *str, ...)
1098 {
1099 va_list panic_str_args;
1100
1101 va_start(panic_str_args, str);
1102 panic_trap_to_debugger(str, &panic_str_args, 0, NULL, 0, NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
1103 va_end(panic_str_args);
1104 }
1105
1106 void
panic_with_data(uuid_t uuid,void * addr,uint32_t len,uint64_t debugger_options_mask,const char * str,...)1107 panic_with_data(uuid_t uuid, void *addr, uint32_t len, uint64_t debugger_options_mask, const char *str, ...)
1108 {
1109 va_list panic_str_args;
1110
1111 ext_paniclog_panic_with_data(uuid, addr, len);
1112
1113 #if CONFIG_EXCLAVES
1114 /*
1115 * Before trapping, inform the exclaves scheduler that we're going down
1116 * so it can grab an exclaves stackshot.
1117 */
1118 if ((debugger_options_mask & DEBUGGER_OPTION_USER_WATCHDOG) != 0 &&
1119 exclaves_get_boot_stage() != EXCLAVES_BOOT_STAGE_NONE) {
1120 (void) exclaves_scheduler_request_watchdog_panic();
1121 }
1122 #endif /* CONFIG_EXCLAVES */
1123
1124 va_start(panic_str_args, str);
1125 panic_trap_to_debugger(str, &panic_str_args, 0, NULL, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1126 NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
1127 va_end(panic_str_args);
1128 }
1129
1130 void
panic_with_options(unsigned int reason,void * ctx,uint64_t debugger_options_mask,const char * str,...)1131 panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
1132 {
1133 va_list panic_str_args;
1134
1135 #if CONFIG_EXCLAVES
1136 /*
1137 * Before trapping, inform the exclaves scheduler that we're going down
1138 * so it can grab an exclaves stackshot.
1139 */
1140 if ((debugger_options_mask & DEBUGGER_OPTION_USER_WATCHDOG) != 0 &&
1141 exclaves_get_boot_stage() != EXCLAVES_BOOT_STAGE_NONE) {
1142 (void) exclaves_scheduler_request_watchdog_panic();
1143 }
1144 #endif /* CONFIG_EXCLAVES */
1145
1146 va_start(panic_str_args, str);
1147 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1148 NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
1149 va_end(panic_str_args);
1150 }
1151
1152 void
panic_with_options_and_initiator(const char * initiator,unsigned int reason,void * ctx,uint64_t debugger_options_mask,const char * str,...)1153 panic_with_options_and_initiator(const char* initiator, unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
1154 {
1155 va_list panic_str_args;
1156
1157 va_start(panic_str_args, str);
1158 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1159 NULL, (unsigned long)(char *)__builtin_return_address(0), initiator);
1160 va_end(panic_str_args);
1161 }
1162
1163 boolean_t
panic_validate_ptr(void * ptr,vm_size_t size,const char * what)1164 panic_validate_ptr(void *ptr, vm_size_t size, const char *what)
1165 {
1166 if (ptr == NULL) {
1167 paniclog_append_noflush("NULL %s pointer\n", what);
1168 return false;
1169 }
1170
1171 if (!ml_validate_nofault((vm_offset_t)ptr, size)) {
1172 paniclog_append_noflush("Invalid %s pointer: %p (size %d)\n",
1173 what, ptr, (uint32_t)size);
1174 return false;
1175 }
1176
1177 return true;
1178 }
1179
1180 boolean_t
panic_get_thread_proc_task(struct thread * thread,struct task ** task,struct proc ** proc)1181 panic_get_thread_proc_task(struct thread *thread, struct task **task, struct proc **proc)
1182 {
1183 if (!PANIC_VALIDATE_PTR(thread)) {
1184 return false;
1185 }
1186
1187 if (!PANIC_VALIDATE_PTR(thread->t_tro)) {
1188 return false;
1189 }
1190
1191 if (!PANIC_VALIDATE_PTR(thread->t_tro->tro_task)) {
1192 return false;
1193 }
1194
1195 if (task) {
1196 *task = thread->t_tro->tro_task;
1197 }
1198
1199 if (!panic_validate_ptr(thread->t_tro->tro_proc,
1200 sizeof(struct proc *), "bsd_info")) {
1201 *proc = NULL;
1202 } else {
1203 *proc = thread->t_tro->tro_proc;
1204 }
1205
1206 return true;
1207 }
1208
1209 #if defined (__x86_64__)
1210 /*
1211 * panic_with_thread_context() is used on x86 platforms to specify a different thread that should be backtraced in the paniclog.
1212 * We don't generally need this functionality on embedded platforms because embedded platforms include a panic time stackshot
1213 * from customer devices. We plumb the thread pointer via the debugger trap mechanism and backtrace the kernel stack from the
1214 * thread when writing the panic log.
1215 *
1216 * NOTE: panic_with_thread_context() should be called with an explicit thread reference held on the passed thread.
1217 */
1218 void
panic_with_thread_context(unsigned int reason,void * ctx,uint64_t debugger_options_mask,thread_t thread,const char * str,...)1219 panic_with_thread_context(unsigned int reason, void *ctx, uint64_t debugger_options_mask, thread_t thread, const char *str, ...)
1220 {
1221 va_list panic_str_args;
1222 __assert_only os_ref_count_t th_ref_count;
1223
1224 assert_thread_magic(thread);
1225 th_ref_count = os_ref_get_count_raw(&thread->ref_count);
1226 assertf(th_ref_count > 0, "panic_with_thread_context called with invalid thread %p with refcount %u", thread, th_ref_count);
1227
1228 /* Take a reference on the thread so it doesn't disappear by the time we try to backtrace it */
1229 thread_reference(thread);
1230
1231 va_start(panic_str_args, str);
1232 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, ((debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK) | DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE),
1233 thread, (unsigned long)(char *)__builtin_return_address(0), "");
1234
1235 va_end(panic_str_args);
1236 }
1237 #endif /* defined (__x86_64__) */
1238
1239 #pragma clang diagnostic push
1240 #pragma clang diagnostic ignored "-Wmissing-noreturn"
1241 __mockable void
panic_trap_to_debugger(const char * panic_format_str,va_list * panic_args,unsigned int reason,void * ctx,uint64_t panic_options_mask,void * panic_data_ptr,unsigned long panic_caller,const char * panic_initiator)1242 panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsigned int reason, void *ctx,
1243 uint64_t panic_options_mask, void *panic_data_ptr, unsigned long panic_caller, const char *panic_initiator)
1244 {
1245 #pragma clang diagnostic pop
1246
1247 #if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
1248 read_lbr();
1249 #endif
1250
1251 /* For very early panics before XNU serial initialization. */
1252 if (PE_kputc == NULL) {
1253 char buf[EARLY_PANIC_BUFLEN];
1254 vsnprintf(buf, EARLY_PANIC_BUFLEN, panic_format_str, *panic_args);
1255 paniclog_append_noflush("panic: %s\n", buf);
1256 paniclog_append_noflush("Kernel panicked very early before serial init, spinning forever...\n");
1257 panic_spin_forever();
1258 }
1259
1260 /* optionally call sync, to reduce lost logs on restart, avoid on recursive panic. Unsafe due to unbounded sync() duration */
1261 if ((panic_options_mask & DEBUGGER_OPTION_SYNC_ON_PANIC_UNSAFE) && (CPUDEBUGGERCOUNT == 0)) {
1262 sync_internal();
1263 }
1264
1265 /* Turn off I/O tracing once we've panicked */
1266 iotrace_disable();
1267
1268 /* call machine-layer panic handler */
1269 ml_panic_trap_to_debugger(panic_format_str, panic_args, reason, ctx, panic_options_mask, panic_caller, panic_initiator);
1270
1271 /* track depth of debugger/panic entry */
1272 CPUDEBUGGERCOUNT++;
1273
1274 __unused uint32_t panic_initiator_crc = panic_initiator ? crc32(0, panic_initiator, strnlen(panic_initiator, MAX_PANIC_INITIATOR_SIZE)) : 0;
1275
1276 /* emit a tracepoint as early as possible in case of hang */
1277 SOCD_TRACE_XNU(PANIC,
1278 ((CPUDEBUGGERCOUNT <= 2) ? SOCD_TRACE_MODE_STICKY_TRACEPOINT : SOCD_TRACE_MODE_NONE),
1279 PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)),
1280 PACK_2X32(VALUE(panic_initiator_crc), VALUE(panic_options_mask & 0xFFFFFFFF)),
1281 ADDR(panic_format_str),
1282 ADDR(panic_caller));
1283
1284 /* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
1285 check_and_handle_nested_panic(panic_options_mask, panic_caller, panic_format_str, panic_args);
1286
1287 /* If we're in a stackshot, signal that we've started panicking and wait for other CPUs to coalesce and spin before proceeding */
1288 stackshot_cpu_signal_panic();
1289
1290 /* Handle any necessary platform specific actions before we proceed */
1291 PEInitiatePanic();
1292
1293 #if DEVELOPMENT || DEBUG
1294 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_ENTRY);
1295 #endif
1296
1297 PE_panic_hook(panic_format_str);
1298
1299 #if defined (__x86_64__)
1300 plctrace_disable();
1301 #endif
1302
1303 if (write_trace_on_panic && kdebug_enable) {
1304 if (get_preemption_level() == 0 && !ml_at_interrupt_context()) {
1305 ml_set_interrupts_enabled(TRUE);
1306 KDBG_RELEASE(TRACE_PANIC);
1307 kdbg_dump_trace_to_file(KDBG_TRACE_PANIC_FILENAME, false);
1308 }
1309 }
1310
1311 ml_set_interrupts_enabled(FALSE);
1312 disable_preemption();
1313
1314 if (!debug_fatal_panic_begin()) {
1315 /*
1316 * This CPU lost the race to be the first to panic. Re-enable
1317 * interrupts and dead loop here awaiting the debugger xcall from
1318 * the CPU that first panicked.
1319 */
1320 ml_set_interrupts_enabled(TRUE);
1321 panic_stop();
1322 }
1323
1324 #if defined (__x86_64__)
1325 pmSafeMode(x86_lcpu(), PM_SAFE_FL_SAFE);
1326 #endif /* defined (__x86_64__) */
1327
1328 /* Never hide pointers from panic logs. */
1329 doprnt_hide_pointers = FALSE;
1330
1331 if (ctx != NULL) {
1332 /*
1333 * We called into panic from a trap, no need to trap again. Set the
1334 * state on the current CPU and then jump to handle_debugger_trap.
1335 */
1336 DebuggerSaveState(DBOP_PANIC, "panic",
1337 panic_format_str, panic_args,
1338 panic_options_mask, panic_data_ptr, TRUE, panic_caller, panic_initiator);
1339 handle_debugger_trap(reason, 0, 0, ctx);
1340 }
1341
1342 #if defined(__arm64__) && !APPLEVIRTUALPLATFORM
1343 /*
1344 * Signal to fastsim that it should open debug ports (nop on hardware)
1345 */
1346 __asm__ volatile ("hint #0x45");
1347 #endif /* defined(__arm64__) && !APPLEVIRTUALPLATFORM */
1348
1349 DebuggerTrapWithState(DBOP_PANIC, "panic", panic_format_str,
1350 panic_args, panic_options_mask, panic_data_ptr, TRUE, panic_caller, panic_initiator);
1351
1352 /*
1353 * Not reached.
1354 */
1355 panic_stop();
1356 __builtin_unreachable();
1357 }
1358
1359 /* We rely on this symbol being visible in the debugger for triage automation */
1360 void __attribute__((noinline, optnone))
panic_spin_forever(void)1361 panic_spin_forever(void)
1362 {
1363 for (;;) {
1364 #if defined(__arm__) || defined(__arm64__)
1365 /* On arm32, which doesn't have a WFE timeout, this may not return. But that should be OK on this path. */
1366 __builtin_arm_wfe();
1367 #else
1368 cpu_pause();
1369 #endif
1370 }
1371 }
1372
1373 void
panic_stackshot_release_lock(void)1374 panic_stackshot_release_lock(void)
1375 {
1376 assert(!not_in_kdp);
1377 DebuggerUnlock();
1378 }
1379
1380 static void
kdp_machine_reboot_type(unsigned int type,uint64_t debugger_flags)1381 kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags)
1382 {
1383 if ((type == kPEPanicRestartCPU) && (debugger_flags & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS)) {
1384 PEHaltRestart(kPEPanicRestartCPUNoCallouts);
1385 } else {
1386 PEHaltRestart(type);
1387 }
1388 halt_all_cpus(TRUE);
1389 }
1390
1391 void
kdp_machine_reboot(void)1392 kdp_machine_reboot(void)
1393 {
1394 kdp_machine_reboot_type(kPEPanicRestartCPU, 0);
1395 }
1396
1397 static __attribute__((unused)) void
panic_debugger_log(const char * string,...)1398 panic_debugger_log(const char *string, ...)
1399 {
1400 va_list panic_debugger_log_args;
1401
1402 va_start(panic_debugger_log_args, string);
1403 #pragma clang diagnostic push
1404 #pragma clang diagnostic ignored "-Wformat-nonliteral"
1405 #pragma clang diagnostic ignored "-Wformat"
1406 _doprnt(string, &panic_debugger_log_args, consdebug_putc, 16);
1407 #pragma clang diagnostic pop
1408 va_end(panic_debugger_log_args);
1409
1410 #if defined(__arm64__)
1411 paniclog_flush();
1412 #endif
1413 }
1414
1415 /*
1416 * Gather and save diagnostic information about a panic (or Debugger call).
1417 *
1418 * On embedded, Debugger and Panic are treated very similarly -- WDT uses Debugger so we can
1419 * theoretically return from it. On desktop, Debugger is treated as a conventional debugger -- i.e no
1420 * paniclog is written and no core is written unless we request a core on NMI.
1421 *
1422 * This routine handles kicking off local coredumps, paniclogs, calling into the Debugger/KDP (if it's configured),
1423 * and calling out to any other functions we have for collecting diagnostic info.
1424 */
1425 static void
debugger_collect_diagnostics(unsigned int exception,unsigned int code,unsigned int subcode,void * state)1426 debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
1427 {
1428 #if DEVELOPMENT || DEBUG
1429 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_PRELOG);
1430 #endif
1431
1432 #if defined(__x86_64__)
1433 kprintf("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
1434 #endif
1435 /*
1436 * DB_HALT (halt_in_debugger) can be requested on startup, we shouldn't generate
1437 * a coredump/paniclog for this type of debugger entry. If KDP isn't configured,
1438 * we'll just spin in kdp_raise_exception.
1439 */
1440 if (debugger_current_op == DBOP_DEBUGGER && halt_in_debugger) {
1441 kdp_raise_exception(exception, code, subcode, state);
1442 if (debugger_safe_to_return && !debugger_is_panic) {
1443 return;
1444 }
1445 }
1446
1447 #ifdef CONFIG_KCOV
1448 /* Try not to break core dump path by sanitizer. */
1449 kcov_panic_disable();
1450 #endif
1451
1452 if ((debugger_current_op == DBOP_PANIC) ||
1453 ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1454 /*
1455 * Attempt to notify listeners once and only once that we've started
1456 * panicking. Only do this for Debugger() calls if we're treating
1457 * Debugger() calls like panic().
1458 */
1459 uint32_t panic_details = 0;
1460 /* if this is a force-reset panic then capture a log and reboot immediately. */
1461 if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
1462 panic_details |= kPanicDetailsForcePowerOff;
1463 }
1464 PEHaltRestartInternal(kPEPanicBegin, panic_details);
1465
1466 /*
1467 * Set the begin pointer in the panic log structure. We key off of this
1468 * static variable rather than contents from the panic header itself in case someone
1469 * has stomped over the panic_info structure. Also initializes the header magic.
1470 */
1471 static boolean_t began_writing_paniclog = FALSE;
1472 if (!began_writing_paniclog) {
1473 PE_init_panicheader();
1474 began_writing_paniclog = TRUE;
1475 }
1476
1477 if (CPUDEBUGGERCOUNT > 1) {
1478 /*
1479 * we are in a nested panic. Record the nested bit in panic flags and do some housekeeping
1480 */
1481 PE_update_panicheader_nestedpanic();
1482 paniclog_flush();
1483 }
1484 }
1485
1486 /*
1487 * Write panic string if this was a panic.
1488 *
1489 * TODO: Consider moving to SavePanicInfo as this is part of the panic log.
1490 */
1491 if (debugger_current_op == DBOP_PANIC) {
1492 paniclog_append_noflush("panic(cpu %u caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller);
1493 if (debugger_panic_str) {
1494 #pragma clang diagnostic push
1495 #pragma clang diagnostic ignored "-Wformat-nonliteral"
1496 #pragma clang diagnostic ignored "-Wformat"
1497 _doprnt(debugger_panic_str, debugger_panic_args, consdebug_putc, 0);
1498 #pragma clang diagnostic pop
1499 }
1500 paniclog_append_noflush("\n");
1501 }
1502 #if defined(__x86_64__)
1503 else if (((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1504 paniclog_append_noflush("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
1505 }
1506
1507 /*
1508 * Debugger() is treated like panic() on embedded -- for example we use it for WDT
1509 * panics (so we need to write a paniclog). On desktop Debugger() is used in the
1510 * conventional sense.
1511 */
1512 if (debugger_current_op == DBOP_PANIC || ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic))
1513 #endif /* __x86_64__ */
1514 {
1515 kdp_callouts(KDP_EVENT_PANICLOG);
1516
1517 /*
1518 * Write paniclog and panic stackshot (if supported)
1519 * TODO: Need to clear panic log when return from debugger
1520 * hooked up for embedded
1521 */
1522 SavePanicInfo(debugger_message, debugger_panic_data, debugger_panic_options, debugger_panic_initiator);
1523
1524 #if DEVELOPMENT || DEBUG
1525 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_POSTLOG);
1526 #endif
1527
1528 /* DEBUGGER_OPTION_PANICLOGANDREBOOT is used for two finger resets on embedded so we get a paniclog */
1529 if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
1530 PEHaltRestart(kPEPanicDiagnosticsDone);
1531 PEHaltRestart(kPEPanicRestartCPUNoCallouts);
1532 }
1533 }
1534
1535 #if CONFIG_KDP_INTERACTIVE_DEBUGGING
1536 /*
1537 * If reboot on panic is enabled and the caller of panic indicated that we should skip
1538 * local coredumps, don't try to write these and instead go straight to reboot. This
1539 * allows us to persist any data that's stored in the panic log.
1540 */
1541 if ((debugger_panic_options & DEBUGGER_OPTION_SKIP_LOCAL_COREDUMP) &&
1542 (debug_boot_arg & DB_REBOOT_POST_CORE)) {
1543 PEHaltRestart(kPEPanicDiagnosticsDone);
1544 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1545 }
1546
1547 /*
1548 * Consider generating a local corefile if the infrastructure is configured
1549 * and we haven't disabled on-device coredumps.
1550 */
1551 if (on_device_corefile_enabled()) {
1552 #if CONFIG_SPTM
1553 /* We want to skip taking a local core dump if this is a panic from SPTM/TXM/cL4. */
1554 extern uint8_t sptm_supports_local_coredump;
1555 bool sptm_interrupted = false;
1556 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET(pmap_sptm_percpu);
1557 (void)sptm_get_cpu_state(sptm_pcpu->sptm_cpu_id, CPUSTATE_SPTM_INTERRUPTED, &sptm_interrupted);
1558 #endif
1559 if (!kdp_has_polled_corefile()) {
1560 if (debug_boot_arg & (DB_KERN_DUMP_ON_PANIC | DB_KERN_DUMP_ON_NMI)) {
1561 paniclog_append_noflush("skipping local kernel core because core file could not be opened prior to panic (mode : 0x%x, error : 0x%x)\n",
1562 kdp_polled_corefile_mode(), kdp_polled_corefile_error());
1563 #if defined(__arm64__)
1564 if (kdp_polled_corefile_mode() == kIOPolledCoreFileModeUnlinked) {
1565 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREFILE_UNLINKED;
1566 }
1567 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1568 paniclog_flush();
1569 #else /* defined(__arm64__) */
1570 if (panic_info->mph_panic_log_offset != 0) {
1571 if (kdp_polled_corefile_mode() == kIOPolledCoreFileModeUnlinked) {
1572 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COREFILE_UNLINKED;
1573 }
1574 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1575 paniclog_flush();
1576 }
1577 #endif /* defined(__arm64__) */
1578 }
1579 }
1580 #if XNU_MONITOR
1581 else if (pmap_get_cpu_data()->ppl_state != PPL_STATE_KERNEL) {
1582 paniclog_append_noflush("skipping local kernel core because the PPL is not in KERNEL state\n");
1583 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1584 paniclog_flush();
1585 }
1586 #elif CONFIG_SPTM
1587 else if (!sptm_supports_local_coredump) {
1588 paniclog_append_noflush("skipping local kernel core because the SPTM is in PANIC state and can't support core dump generation\n");
1589 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1590 paniclog_flush();
1591 } else if (sptm_interrupted) {
1592 paniclog_append_noflush("skipping local kernel core because the SPTM is in INTERRUPTED state and can't support core dump generation\n");
1593 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1594 paniclog_flush();
1595 }
1596 #endif /* XNU_MONITOR */
1597 else {
1598 int ret = -1;
1599
1600 #if defined (__x86_64__)
1601 /* On x86 we don't do a coredump on Debugger unless the DB_KERN_DUMP_ON_NMI boot-arg is specified. */
1602 if (debugger_current_op != DBOP_DEBUGGER || (debug_boot_arg & DB_KERN_DUMP_ON_NMI))
1603 #endif
1604 {
1605 /*
1606 * Doing an on-device coredump leaves the disk driver in a state
1607 * that can not be resumed.
1608 */
1609 debugger_safe_to_return = FALSE;
1610 begin_panic_transfer();
1611 ret = kern_dump(KERN_DUMP_DISK);
1612 abort_panic_transfer();
1613
1614 #if DEVELOPMENT || DEBUG
1615 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_POSTCORE);
1616 #endif
1617 }
1618
1619 /*
1620 * If DB_REBOOT_POST_CORE is set, then reboot if coredump is sucessfully saved
1621 * or if option to ignore failures is set.
1622 */
1623 if ((debug_boot_arg & DB_REBOOT_POST_CORE) &&
1624 ((ret == 0) || (debugger_panic_options & DEBUGGER_OPTION_ATTEMPTCOREDUMPANDREBOOT))) {
1625 PEHaltRestart(kPEPanicDiagnosticsDone);
1626 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1627 }
1628 }
1629 }
1630
1631 if (debugger_current_op == DBOP_PANIC ||
1632 ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1633 PEHaltRestart(kPEPanicDiagnosticsDone);
1634 }
1635
1636 if (debug_boot_arg & DB_REBOOT_ALWAYS) {
1637 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1638 }
1639
1640 /* If KDP is configured, try to trap to the debugger */
1641 #if defined(__arm64__)
1642 if (kdp_explicitly_requested && (current_debugger != NO_CUR_DB)) {
1643 #else
1644 if (current_debugger != NO_CUR_DB) {
1645 #endif
1646 kdp_raise_exception(exception, code, subcode, state);
1647 /*
1648 * Only return if we entered via Debugger and it's safe to return
1649 * (we halted the other cores successfully, this isn't a nested panic, etc)
1650 */
1651 if (debugger_current_op == DBOP_DEBUGGER &&
1652 debugger_safe_to_return &&
1653 kernel_debugger_entry_count == 1 &&
1654 !debugger_is_panic) {
1655 return;
1656 }
1657 }
1658
1659 #if defined(__arm64__)
1660 if (PE_i_can_has_debugger(NULL) && panicDebugging) {
1661 /*
1662 * Print panic string at the end of serial output
1663 * to make panic more obvious when someone connects a debugger
1664 */
1665 if (debugger_panic_str) {
1666 panic_debugger_log("Original panic string:\n");
1667 panic_debugger_log("panic(cpu %u caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller);
1668 #pragma clang diagnostic push
1669 #pragma clang diagnostic ignored "-Wformat-nonliteral"
1670 #pragma clang diagnostic ignored "-Wformat"
1671 _doprnt(debugger_panic_str, debugger_panic_args, consdebug_putc, 0);
1672 #pragma clang diagnostic pop
1673 panic_debugger_log("\n");
1674 }
1675
1676 /* If panic debugging is configured and we're on a dev fused device, spin for astris to connect */
1677 panic_spin_shmcon();
1678 }
1679 #endif /* defined(__arm64__) */
1680
1681 #else /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1682
1683 PEHaltRestart(kPEPanicDiagnosticsDone);
1684
1685 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1686
1687 if (!panicDebugging) {
1688 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1689 }
1690
1691 paniclog_append_noflush("\nPlease go to https://panic.apple.com to report this panic\n");
1692 panic_spin_forever();
1693 }
1694
1695 #if SCHED_HYGIENE_DEBUG
1696 uint64_t debugger_trap_timestamps[9];
1697 # define DEBUGGER_TRAP_TIMESTAMP(i) debugger_trap_timestamps[i] = mach_absolute_time();
1698 #else
1699 # define DEBUGGER_TRAP_TIMESTAMP(i)
1700 #endif /* SCHED_HYGIENE_DEBUG */
1701
1702 void
1703 handle_debugger_trap(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
1704 {
1705 unsigned int initial_not_in_kdp = not_in_kdp;
1706 kern_return_t ret = KERN_SUCCESS;
1707 debugger_op db_prev_op = debugger_current_op;
1708
1709 if (!DebuggerLock()) {
1710 /*
1711 * We lost the race to be the first to panic.
1712 * Return here so that we will enter the panic stop
1713 * infinite loop and take the debugger IPI from the
1714 * first CPU that got the debugger lock.
1715 */
1716 return;
1717 }
1718
1719 DEBUGGER_TRAP_TIMESTAMP(0);
1720
1721 ret = DebuggerHaltOtherCores(CPUDEBUGGERSYNC, (CPUDEBUGGEROP == DBOP_STACKSHOT));
1722
1723 DEBUGGER_TRAP_TIMESTAMP(1);
1724
1725 #if SCHED_HYGIENE_DEBUG
1726 if (serialmode & SERIALMODE_OUTPUT) {
1727 ml_spin_debug_reset(current_thread());
1728 }
1729 #endif /* SCHED_HYGIENE_DEBUG */
1730 if (ret != KERN_SUCCESS) {
1731 CPUDEBUGGERRET = ret;
1732 DebuggerUnlock();
1733 return;
1734 }
1735
1736 /* Update the global panic/debugger nested entry level */
1737 kernel_debugger_entry_count = CPUDEBUGGERCOUNT;
1738 if (kernel_debugger_entry_count > 0) {
1739 console_suspend();
1740 }
1741
1742 /*
1743 * TODO: Should we do anything special for nested panics here? i.e. if we've trapped more than twice
1744 * should we call into the debugger if it's configured and then reboot if the panic log has been written?
1745 */
1746
1747 if (CPUDEBUGGEROP == DBOP_NONE) {
1748 /* If there was no debugger context setup, we trapped due to a software breakpoint */
1749 debugger_current_op = DBOP_BREAKPOINT;
1750 } else {
1751 /* Not safe to return from a nested panic/debugger call */
1752 if (debugger_current_op == DBOP_PANIC ||
1753 debugger_current_op == DBOP_DEBUGGER) {
1754 debugger_safe_to_return = FALSE;
1755 }
1756
1757 debugger_current_op = CPUDEBUGGEROP;
1758
1759 /* Only overwrite the panic message if there is none already - save the data from the first call */
1760 if (debugger_panic_str == NULL) {
1761 debugger_panic_str = CPUPANICSTR;
1762 debugger_panic_args = CPUPANICARGS;
1763 debugger_panic_data = CPUPANICDATAPTR;
1764 debugger_message = CPUDEBUGGERMSG;
1765 debugger_panic_caller = CPUPANICCALLER;
1766 debugger_panic_initiator = CPUPANICINITIATOR;
1767 }
1768
1769 debugger_panic_options = CPUPANICOPTS;
1770 }
1771
1772 /*
1773 * Clear the op from the processor debugger context so we can handle
1774 * breakpoints in the debugger
1775 */
1776 CPUDEBUGGEROP = DBOP_NONE;
1777
1778 DEBUGGER_TRAP_TIMESTAMP(2);
1779
1780 kdp_callouts(KDP_EVENT_ENTER);
1781 not_in_kdp = 0;
1782
1783 DEBUGGER_TRAP_TIMESTAMP(3);
1784
1785 #if defined(__arm64__) && CONFIG_KDP_INTERACTIVE_DEBUGGING
1786 shmem_mark_as_busy();
1787 #endif
1788
1789 if (debugger_current_op == DBOP_BREAKPOINT) {
1790 kdp_raise_exception(exception, code, subcode, state);
1791 } else if (debugger_current_op == DBOP_STACKSHOT) {
1792 CPUDEBUGGERRET = do_stackshot(NULL);
1793 #if PGO
1794 } else if (debugger_current_op == DBOP_RESET_PGO_COUNTERS) {
1795 CPUDEBUGGERRET = do_pgo_reset_counters();
1796 #endif
1797 } else {
1798 /* note: this is the panic path... */
1799 if (!debug_fatal_panic_begin()) {
1800 /*
1801 * This CPU lost the race to be the first to panic. Re-enable
1802 * interrupts and dead loop here awaiting the debugger xcall from
1803 * the CPU that first panicked.
1804 */
1805 ml_set_interrupts_enabled(TRUE);
1806 panic_stop();
1807 }
1808 #if defined(__arm64__) && (DEBUG || DEVELOPMENT)
1809 if (!PE_arm_debug_and_trace_initialized()) {
1810 paniclog_append_noflush("kernel panicked before debug and trace infrastructure initialized!\n"
1811 "spinning forever...\n");
1812 panic_spin_forever();
1813 }
1814 #endif
1815 debugger_collect_diagnostics(exception, code, subcode, state);
1816 }
1817
1818 #if defined(__arm64__) && CONFIG_KDP_INTERACTIVE_DEBUGGING
1819 shmem_unmark_as_busy();
1820 #endif
1821
1822 DEBUGGER_TRAP_TIMESTAMP(4);
1823
1824 not_in_kdp = initial_not_in_kdp;
1825 kdp_callouts(KDP_EVENT_EXIT);
1826
1827 DEBUGGER_TRAP_TIMESTAMP(5);
1828
1829 if (debugger_current_op != DBOP_BREAKPOINT) {
1830 debugger_panic_str = NULL;
1831 debugger_panic_args = NULL;
1832 debugger_panic_data = NULL;
1833 debugger_panic_options = 0;
1834 debugger_message = NULL;
1835 }
1836
1837 /* Restore the previous debugger state */
1838 debugger_current_op = db_prev_op;
1839
1840 DEBUGGER_TRAP_TIMESTAMP(6);
1841
1842 DebuggerResumeOtherCores();
1843
1844 DEBUGGER_TRAP_TIMESTAMP(7);
1845
1846 DebuggerUnlock();
1847
1848 DEBUGGER_TRAP_TIMESTAMP(8);
1849
1850 return;
1851 }
1852
1853 __attribute__((noinline, not_tail_called))
1854 void
1855 log(__unused int level, char *fmt, ...)
1856 {
1857 void *caller = __builtin_return_address(0);
1858 va_list listp;
1859 va_list listp2;
1860
1861
1862 #ifdef lint
1863 level++;
1864 #endif /* lint */
1865 #ifdef MACH_BSD
1866 va_start(listp, fmt);
1867 va_copy(listp2, listp);
1868
1869 disable_preemption();
1870 _doprnt(fmt, &listp, cons_putc_locked, 0);
1871 enable_preemption();
1872
1873 va_end(listp);
1874
1875 #pragma clang diagnostic push
1876 #pragma clang diagnostic ignored "-Wformat-nonliteral"
1877 #pragma clang diagnostic ignored "-Wformat"
1878 os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, listp2, caller);
1879 #pragma clang diagnostic pop
1880 va_end(listp2);
1881 #endif
1882 }
1883
1884 /*
1885 * Per <rdar://problem/24974766>, skip appending log messages to
1886 * the new logging infrastructure in contexts where safety is
1887 * uncertain. These contexts include:
1888 * - When we're in the debugger
1889 * - We're in a panic
1890 * - Interrupts are disabled
1891 * - Or Pre-emption is disabled
1892 * In all the above cases, it is potentially unsafe to log messages.
1893 */
1894
1895 boolean_t
1896 oslog_is_safe(void)
1897 {
1898 return kernel_debugger_entry_count == 0 &&
1899 not_in_kdp == 1 &&
1900 get_preemption_level() == 0 &&
1901 ml_get_interrupts_enabled() == TRUE;
1902 }
1903
1904 boolean_t
1905 debug_mode_active(void)
1906 {
1907 return (0 != kernel_debugger_entry_count != 0) || (0 == not_in_kdp);
1908 }
1909
1910 void
1911 debug_putc(char c)
1912 {
1913 if ((debug_buf_size != 0) &&
1914 ((debug_buf_ptr - debug_buf_base) < (int)debug_buf_size) &&
1915 (!is_debug_ptr_in_ext_paniclog())) {
1916 *debug_buf_ptr = c;
1917 debug_buf_ptr++;
1918 }
1919 }
1920
1921 #if defined (__x86_64__)
1922 struct pasc {
1923 unsigned a: 7;
1924 unsigned b: 7;
1925 unsigned c: 7;
1926 unsigned d: 7;
1927 unsigned e: 7;
1928 unsigned f: 7;
1929 unsigned g: 7;
1930 unsigned h: 7;
1931 } __attribute__((packed));
1932
1933 typedef struct pasc pasc_t;
1934
1935 /*
1936 * In-place packing routines -- inefficient, but they're called at most once.
1937 * Assumes "buflen" is a multiple of 8. Used for compressing paniclogs on x86.
1938 */
1939 int
1940 packA(char *inbuf, uint32_t length, uint32_t buflen)
1941 {
1942 unsigned int i, j = 0;
1943 pasc_t pack;
1944
1945 length = MIN(((length + 7) & ~7), buflen);
1946
1947 for (i = 0; i < length; i += 8) {
1948 pack.a = inbuf[i];
1949 pack.b = inbuf[i + 1];
1950 pack.c = inbuf[i + 2];
1951 pack.d = inbuf[i + 3];
1952 pack.e = inbuf[i + 4];
1953 pack.f = inbuf[i + 5];
1954 pack.g = inbuf[i + 6];
1955 pack.h = inbuf[i + 7];
1956 bcopy((char *) &pack, inbuf + j, 7);
1957 j += 7;
1958 }
1959 return j;
1960 }
1961
1962 void
1963 unpackA(char *inbuf, uint32_t length)
1964 {
1965 pasc_t packs;
1966 unsigned i = 0;
1967 length = (length * 8) / 7;
1968
1969 while (i < length) {
1970 packs = *(pasc_t *)&inbuf[i];
1971 bcopy(&inbuf[i + 7], &inbuf[i + 8], MAX(0, (int) (length - i - 8)));
1972 inbuf[i++] = packs.a;
1973 inbuf[i++] = packs.b;
1974 inbuf[i++] = packs.c;
1975 inbuf[i++] = packs.d;
1976 inbuf[i++] = packs.e;
1977 inbuf[i++] = packs.f;
1978 inbuf[i++] = packs.g;
1979 inbuf[i++] = packs.h;
1980 }
1981 }
1982 #endif /* defined (__x86_64__) */
1983
1984 extern char *proc_name_address(void *);
1985 extern char *proc_longname_address(void *);
1986
1987 __private_extern__ void
1988 panic_display_process_name(void)
1989 {
1990 proc_name_t proc_name = {};
1991 struct proc *cbsd_info = NULL;
1992 task_t ctask = NULL;
1993 vm_size_t size;
1994
1995 if (!panic_get_thread_proc_task(current_thread(), &ctask, &cbsd_info)) {
1996 goto out;
1997 }
1998
1999 if (cbsd_info == NULL) {
2000 goto out;
2001 }
2002
2003 size = ml_nofault_copy((vm_offset_t)proc_longname_address(cbsd_info),
2004 (vm_offset_t)&proc_name, sizeof(proc_name));
2005
2006 if (size == 0 || proc_name[0] == '\0') {
2007 size = ml_nofault_copy((vm_offset_t)proc_name_address(cbsd_info),
2008 (vm_offset_t)&proc_name,
2009 MIN(sizeof(command_t), sizeof(proc_name)));
2010 if (size > 0) {
2011 proc_name[size - 1] = '\0';
2012 }
2013 }
2014
2015 out:
2016 proc_name[sizeof(proc_name) - 1] = '\0';
2017 paniclog_append_noflush("\nProcess name corresponding to current thread (%p): %s\n",
2018 current_thread(), proc_name[0] != '\0' ? proc_name : "Unknown");
2019 }
2020
2021 unsigned
2022 panic_active(void)
2023 {
2024 return debugger_current_op == DBOP_PANIC ||
2025 (debugger_current_op == DBOP_DEBUGGER && debugger_is_panic);
2026 }
2027
2028 void
2029 populate_model_name(char *model_string)
2030 {
2031 strlcpy(model_name, model_string, sizeof(model_name));
2032 }
2033
2034 void
2035 panic_display_model_name(void)
2036 {
2037 char tmp_model_name[sizeof(model_name)];
2038
2039 if (ml_nofault_copy((vm_offset_t) &model_name, (vm_offset_t) &tmp_model_name, sizeof(model_name)) != sizeof(model_name)) {
2040 return;
2041 }
2042
2043 tmp_model_name[sizeof(tmp_model_name) - 1] = '\0';
2044
2045 if (tmp_model_name[0] != 0) {
2046 paniclog_append_noflush("System model name: %s\n", tmp_model_name);
2047 }
2048 }
2049
2050 void
2051 panic_display_kernel_uuid(void)
2052 {
2053 char tmp_kernel_uuid[sizeof(kernel_uuid_string)];
2054
2055 if (ml_nofault_copy((vm_offset_t) &kernel_uuid_string, (vm_offset_t) &tmp_kernel_uuid, sizeof(kernel_uuid_string)) != sizeof(kernel_uuid_string)) {
2056 return;
2057 }
2058
2059 if (tmp_kernel_uuid[0] != '\0') {
2060 paniclog_append_noflush("Kernel UUID: %s\n", tmp_kernel_uuid);
2061 }
2062 }
2063
2064 #if CONFIG_SPTM
2065 static void
2066 panic_display_component_uuid(char const *component_name, void *component_address)
2067 {
2068 uuid_t *component_uuid;
2069 unsigned long component_uuid_len = 0;
2070 uuid_string_t component_uuid_string;
2071
2072 component_uuid = getuuidfromheader((kernel_mach_header_t *)component_address, &component_uuid_len);
2073
2074 if (component_uuid != NULL && component_uuid_len == sizeof(uuid_t)) {
2075 uuid_unparse_upper(*component_uuid, component_uuid_string);
2076 paniclog_append_noflush("%s UUID: %s\n", component_name, component_uuid_string);
2077 }
2078 }
2079 #endif /* CONFIG_SPTM */
2080
2081 void
2082 panic_display_kernel_aslr(void)
2083 {
2084 #if CONFIG_SPTM
2085 {
2086 struct debug_header const *dh = SPTMArgs->debug_header;
2087
2088 paniclog_append_noflush("Debug Header address: %p\n", dh);
2089
2090 if (dh != NULL) {
2091 void *component_address;
2092
2093 paniclog_append_noflush("Debug Header entry count: %d\n", dh->count);
2094
2095 switch (dh->count) {
2096 default: // 3 or more
2097 component_address = dh->image[DEBUG_HEADER_ENTRY_TXM];
2098 paniclog_append_noflush("TXM load address: %p\n", component_address);
2099
2100 panic_display_component_uuid("TXM", component_address);
2101 OS_FALLTHROUGH;
2102 case 2:
2103 component_address = dh->image[DEBUG_HEADER_ENTRY_XNU];
2104 paniclog_append_noflush("Debug Header kernelcache load address: %p\n", component_address);
2105
2106 panic_display_component_uuid("Debug Header kernelcache", component_address);
2107 OS_FALLTHROUGH;
2108 case 1:
2109 component_address = dh->image[DEBUG_HEADER_ENTRY_SPTM];
2110 paniclog_append_noflush("SPTM load address: %p\n", component_address);
2111
2112 panic_display_component_uuid("SPTM", component_address);
2113 OS_FALLTHROUGH;
2114 case 0:
2115 ; // nothing to print
2116 }
2117 }
2118 }
2119 #endif /* CONFIG_SPTM */
2120
2121 kc_format_t kc_format;
2122
2123 PE_get_primary_kc_format(&kc_format);
2124
2125 if (kc_format == KCFormatFileset) {
2126 void *kch = PE_get_kc_header(KCKindPrimary);
2127 paniclog_append_noflush("KernelCache slide: 0x%016lx\n", (unsigned long) vm_kernel_slide);
2128 paniclog_append_noflush("KernelCache base: %p\n", (void*) kch);
2129 paniclog_append_noflush("Kernel slide: 0x%016lx\n", vm_kernel_stext - (unsigned long)kch + vm_kernel_slide);
2130 paniclog_append_noflush("Kernel text base: %p\n", (void *) vm_kernel_stext);
2131 #if defined(__arm64__)
2132 unsigned long kernel_text_exec_slide = 0, kernel_text_exec_base = 0;
2133 get_kernel_text_exec_slide_and_base(&kernel_text_exec_slide, &kernel_text_exec_base);
2134 paniclog_append_noflush("Kernel text exec slide: 0x%016lx\n", kernel_text_exec_slide);
2135 paniclog_append_noflush("Kernel text exec base: 0x%016lx\n", kernel_text_exec_base);
2136 #endif /* defined(__arm64__) */
2137 } else if (vm_kernel_slide) {
2138 paniclog_append_noflush("Kernel slide: 0x%016lx\n", (unsigned long) vm_kernel_slide);
2139 paniclog_append_noflush("Kernel text base: %p\n", (void *)vm_kernel_stext);
2140 } else {
2141 paniclog_append_noflush("Kernel text base: %p\n", (void *)vm_kernel_stext);
2142 }
2143 }
2144
2145 void
2146 panic_display_hibb(void)
2147 {
2148 #if defined(__i386__) || defined (__x86_64__)
2149 paniclog_append_noflush("__HIB text base: %p\n", (void *) vm_hib_base);
2150 #endif
2151 }
2152
2153 #if CONFIG_ECC_LOGGING
2154 __private_extern__ void
2155 panic_display_ecc_errors(void)
2156 {
2157 uint32_t count = ecc_log_get_correction_count();
2158
2159 if (count > 0) {
2160 paniclog_append_noflush("ECC Corrections:%u\n", count);
2161 }
2162 }
2163 #endif /* CONFIG_ECC_LOGGING */
2164
2165 extern int vm_num_swap_files;
2166
2167 void
2168 panic_display_compressor_stats(void)
2169 {
2170 int isswaplow = vm_swap_low_on_space();
2171 #if CONFIG_FREEZE
2172 uint32_t incore_seg_count;
2173 uint32_t incore_compressed_pages;
2174 if (freezer_incore_cseg_acct) {
2175 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
2176 incore_compressed_pages = c_segment_pages_compressed_incore;
2177 } else {
2178 incore_seg_count = c_segment_count;
2179 incore_compressed_pages = c_segment_pages_compressed;
2180 }
2181
2182 paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
2183 (incore_compressed_pages * 100) / c_segment_pages_compressed_limit,
2184 (incore_compressed_pages > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
2185 (incore_seg_count * 100) / c_segments_limit,
2186 (incore_seg_count > c_segments_nearing_limit) ? "BAD":"OK",
2187 vm_num_swap_files,
2188 isswaplow ? "LOW":"OK");
2189 #else /* CONFIG_FREEZE */
2190 paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
2191 (c_segment_pages_compressed * 100) / c_segment_pages_compressed_limit,
2192 (c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
2193 (c_segment_count * 100) / c_segments_limit,
2194 (c_segment_count > c_segments_nearing_limit) ? "BAD":"OK",
2195 vm_num_swap_files,
2196 isswaplow ? "LOW":"OK");
2197 #endif /* CONFIG_FREEZE */
2198 }
2199
2200 #if !CONFIG_TELEMETRY
2201 int
2202 telemetry_gather(user_addr_t buffer __unused, uint32_t *length __unused, bool mark __unused)
2203 {
2204 return KERN_NOT_SUPPORTED;
2205 }
2206 #endif
2207
2208 #include <machine/machine_cpu.h>
2209
2210 TUNABLE(uint32_t, kern_feature_overrides, "validation_disables", 0);
2211
2212 __startup_func
2213 static void
2214 kern_feature_override_init(void)
2215 {
2216 /*
2217 * update kern_feature_override based on the serverperfmode=1 boot-arg
2218 * being present, but do not look at the device-tree setting on purpose.
2219 *
2220 * scale_setup() will update serverperfmode=1 based on the DT later.
2221 */
2222
2223 if (serverperfmode) {
2224 kern_feature_overrides |= KF_SERVER_PERF_MODE_OVRD;
2225 }
2226 }
2227 STARTUP(TUNABLES, STARTUP_RANK_LAST, kern_feature_override_init);
2228
2229 #if MACH_ASSERT
2230 STATIC_IF_KEY_DEFINE_TRUE(mach_assert);
2231 #endif
2232
2233 #if SCHED_HYGIENE_DEBUG
2234 STATIC_IF_KEY_DEFINE_TRUE(sched_debug_pmc);
2235 STATIC_IF_KEY_DEFINE_TRUE(sched_debug_preemption_disable);
2236 STATIC_IF_KEY_DEFINE_TRUE(sched_debug_interrupt_disable);
2237 #endif /* SCHED_HYGIENE_DEBUG */
2238
2239 __static_if_init_func
2240 static void
2241 kern_feature_override_apply(const char *args)
2242 {
2243 uint64_t kf_ovrd;
2244
2245 /*
2246 * Compute the value of kern_feature_override like it will look like
2247 * after kern_feature_override_init().
2248 */
2249 kf_ovrd = static_if_boot_arg_uint64(args, "validation_disables", 0);
2250 if (static_if_boot_arg_uint64(args, "serverperfmode", 0)) {
2251 kf_ovrd |= KF_SERVER_PERF_MODE_OVRD;
2252 }
2253
2254 #if DEBUG_RW
2255 lck_rw_assert_init(args, kf_ovrd);
2256 #endif /* DEBUG_RW */
2257 #if MACH_ASSERT
2258 if (kf_ovrd & KF_MACH_ASSERT_OVRD) {
2259 static_if_key_disable(mach_assert);
2260 }
2261 #endif /* MACH_ASSERT */
2262 #if SCHED_HYGIENE_DEBUG
2263 if ((int64_t)static_if_boot_arg_uint64(args, "wdt", 0) != -1) {
2264 if (kf_ovrd & KF_SCHED_HYGIENE_DEBUG_PMC_OVRD) {
2265 static_if_key_disable(sched_debug_pmc);
2266 }
2267 if (kf_ovrd & KF_PREEMPTION_DISABLED_DEBUG_OVRD) {
2268 static_if_key_disable(sched_debug_preemption_disable);
2269 if (kf_ovrd & KF_INTERRUPT_MASKED_DEBUG_OVRD) {
2270 static_if_key_disable(sched_debug_interrupt_disable);
2271 }
2272 }
2273 }
2274 #endif /* SCHED_HYGIENE_DEBUG */
2275 }
2276 STATIC_IF_INIT(kern_feature_override_apply);
2277
2278 boolean_t
2279 kern_feature_override(uint32_t fmask)
2280 {
2281 return (kern_feature_overrides & fmask) == fmask;
2282 }
2283
2284 #if !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING
2285 static boolean_t
2286 device_corefile_valid_on_ephemeral(void)
2287 {
2288 #ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
2289 DTEntry node;
2290 const uint32_t *value = NULL;
2291 unsigned int size = 0;
2292 if (kSuccess != SecureDTLookupEntry(NULL, "/product", &node)) {
2293 return TRUE;
2294 }
2295 if (kSuccess != SecureDTGetProperty(node, "ephemeral-data-mode", (void const **) &value, &size)) {
2296 return TRUE;
2297 }
2298
2299 if (size != sizeof(uint32_t)) {
2300 return TRUE;
2301 }
2302
2303 if ((*value) && (kern_dump_should_enforce_encryption() == true)) {
2304 return FALSE;
2305 }
2306 #endif /* ifdef CONFIG_KDP_COREDUMP_ENCRYPTION */
2307
2308 return TRUE;
2309 }
2310 #endif /* !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING */
2311
2312 boolean_t
2313 on_device_corefile_enabled(void)
2314 {
2315 assert(startup_phase >= STARTUP_SUB_TUNABLES);
2316 #if CONFIG_KDP_INTERACTIVE_DEBUGGING
2317 if (debug_boot_arg == 0) {
2318 return FALSE;
2319 }
2320 if (debug_boot_arg & DB_DISABLE_LOCAL_CORE) {
2321 return FALSE;
2322 }
2323 #if !XNU_TARGET_OS_OSX
2324 if (device_corefile_valid_on_ephemeral() == FALSE) {
2325 return FALSE;
2326 }
2327 /*
2328 * outside of macOS, if there's a debug boot-arg set and local
2329 * cores aren't explicitly disabled, we always write a corefile.
2330 */
2331 return TRUE;
2332 #else /* !XNU_TARGET_OS_OSX */
2333 /*
2334 * on macOS, if corefiles on panic are requested and local cores
2335 * aren't disabled we write a local core.
2336 */
2337 if (debug_boot_arg & (DB_KERN_DUMP_ON_NMI | DB_KERN_DUMP_ON_PANIC)) {
2338 return TRUE;
2339 }
2340 #endif /* !XNU_TARGET_OS_OSX */
2341 #endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
2342 return FALSE;
2343 }
2344
2345 boolean_t
2346 panic_stackshot_to_disk_enabled(void)
2347 {
2348 assert(startup_phase >= STARTUP_SUB_TUNABLES);
2349 #if defined(__x86_64__)
2350 if (PEGetCoprocessorVersion() < kCoprocessorVersion2) {
2351 /* Only enabled on pre-Gibraltar machines where it hasn't been disabled explicitly */
2352 if ((debug_boot_arg != 0) && (debug_boot_arg & DB_DISABLE_STACKSHOT_TO_DISK)) {
2353 return FALSE;
2354 }
2355
2356 return TRUE;
2357 }
2358 #endif
2359 return FALSE;
2360 }
2361
2362 const char *
2363 sysctl_debug_get_preoslog(size_t *size)
2364 {
2365 int result = 0;
2366 void *preoslog_pa = NULL;
2367 int preoslog_size = 0;
2368
2369 result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
2370 if (result || preoslog_pa == NULL || preoslog_size == 0) {
2371 kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
2372 *size = 0;
2373 return NULL;
2374 }
2375
2376 /*
2377 * Beware:
2378 * On release builds, we would need to call IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size) to free the preoslog buffer.
2379 * On Development & Debug builds, we retain the buffer so it can be extracted from coredumps.
2380 */
2381 *size = preoslog_size;
2382 return (char *)(ml_static_ptovirt((vm_offset_t)(preoslog_pa)));
2383 }
2384
2385 void
2386 sysctl_debug_free_preoslog(void)
2387 {
2388 #if RELEASE
2389 int result = 0;
2390 void *preoslog_pa = NULL;
2391 int preoslog_size = 0;
2392
2393 result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
2394 if (result || preoslog_pa == NULL || preoslog_size == 0) {
2395 kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
2396 return;
2397 }
2398
2399 IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size);
2400 #else
2401 /* On Development & Debug builds, we retain the buffer so it can be extracted from coredumps. */
2402 #endif // RELEASE
2403 }
2404
2405 #if HAS_UPSI_FAILURE_INJECTION
2406 uint64_t xnu_upsi_injection_stage = 0;
2407 uint64_t xnu_upsi_injection_action = 0;
2408
2409 __attribute__((optnone)) static void
2410 SPINNING_FOREVER(void)
2411 {
2412 // Decided to disable optimizations on this function instead of using a
2413 // volatile bool for the deadloop.
2414 // This simplifies the process of using the deadloop as an LLDB attach point
2415 bool loop = true;
2416
2417 while (loop) {
2418 }
2419 return;
2420 }
2421
2422 void
2423 check_for_failure_injection(failure_injection_stage_t current_stage)
2424 {
2425 // Can't call this function with the default initialization for xnu_upsi_injection_stage
2426 assert(current_stage != 0);
2427
2428 // Check condition to inject a panic/stall/hang
2429 if (current_stage != xnu_upsi_injection_stage) {
2430 return;
2431 }
2432
2433 // Do the requested action
2434 switch (xnu_upsi_injection_action) {
2435 case INJECTION_ACTION_PANIC:
2436 panic("Test panic at stage 0x%llx", current_stage);
2437 case INJECTION_ACTION_WATCHDOG_TIMEOUT:
2438 case INJECTION_ACTION_DEADLOOP:
2439 SPINNING_FOREVER();
2440 break;
2441 default:
2442 break;
2443 }
2444 }
2445 #endif // HAS_UPSI_FAILURE_INJECTION
2446
2447 #define AWL_HV_ENTRY_FLAG (0x1)
2448
2449 static inline void
2450 awl_set_scratch_reg_hv_bit(void)
2451 {
2452 #if defined(__arm64__)
2453 #define WATCHDOG_DIAG0 "S3_5_c15_c2_6"
2454 uint64_t awl_diag0 = __builtin_arm_rsr64(WATCHDOG_DIAG0);
2455 awl_diag0 |= AWL_HV_ENTRY_FLAG;
2456 __builtin_arm_wsr64(WATCHDOG_DIAG0, awl_diag0);
2457 #endif // defined(__arm64__)
2458 }
2459
2460 void
2461 awl_mark_hv_entry(void)
2462 {
2463 if (__probable(*PERCPU_GET(hv_entry_detected) || !awl_scratch_reg_supported)) {
2464 return;
2465 }
2466 *PERCPU_GET(hv_entry_detected) = true;
2467
2468 awl_set_scratch_reg_hv_bit();
2469 }
2470
2471 /*
2472 * Awl WatchdogDiag0 is not restored by hardware when coming out of reset,
2473 * so restore it manually.
2474 */
2475 static bool
2476 awl_pm_state_change_cbk(void *param __unused, enum cpu_event event, unsigned int cpu_or_cluster __unused)
2477 {
2478 if (event == CPU_BOOTED) {
2479 if (*PERCPU_GET(hv_entry_detected)) {
2480 awl_set_scratch_reg_hv_bit();
2481 }
2482 }
2483
2484 return true;
2485 }
2486
2487 /*
2488 * Identifies and sets a flag if AWL Scratch0/1 exists in the system, subscribes
2489 * for a callback to restore register after hibernation
2490 */
2491 __startup_func
2492 static void
2493 set_awl_scratch_exists_flag_and_subscribe_for_pm(void)
2494 {
2495 DTEntry base = NULL;
2496
2497 if (SecureDTLookupEntry(NULL, "/arm-io/wdt", &base) != kSuccess) {
2498 return;
2499 }
2500 const uint8_t *data = NULL;
2501 unsigned int data_size = sizeof(uint8_t);
2502
2503 if (base != NULL && SecureDTGetProperty(base, "awl-scratch-supported", (const void **)&data, &data_size) == kSuccess) {
2504 for (unsigned int i = 0; i < data_size; i++) {
2505 if (data[i] != 0) {
2506 awl_scratch_reg_supported = true;
2507 cpu_event_register_callback(awl_pm_state_change_cbk, NULL);
2508 break;
2509 }
2510 }
2511 }
2512 }
2513 STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, set_awl_scratch_exists_flag_and_subscribe_for_pm);
2514
2515 /**
2516 * Signal that the system is going down for a panic. Returns true if it is safe to
2517 * proceed with the panic flow, false if we should re-enable interrupts and spin
2518 * to allow another CPU to proceed with its panic flow.
2519 *
2520 * This function is idempotent when called from the same CPU; in the normal
2521 * panic case it is invoked twice, since it needs to be invoked in the case
2522 * where we enter the panic flow outside of panic() from DebuggerWithContext().
2523 */
2524 static inline boolean_t
2525 debug_fatal_panic_begin(void)
2526 {
2527 #if CONFIG_SPTM
2528 /*
2529 * Since we're going down, initiate panic lockdown.
2530 *
2531 * Whether or not this call to panic lockdown can be subverted is murky.
2532 * This doesn't really matter, however, because any security critical panics
2533 * events will have already initiated lockdown from the exception vector
2534 * before calling panic. Thus, lockdown from panic itself is fine as merely
2535 * a "best effort".
2536 */
2537 #if DEVELOPMENT || DEBUG
2538 panic_lockdown_record_debug_data();
2539 #endif /* DEVELOPMENT || DEBUG */
2540 sptm_xnu_panic_begin();
2541
2542 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET(pmap_sptm_percpu);
2543 uint16_t sptm_cpu_id = sptm_pcpu->sptm_cpu_id;
2544 uint64_t sptm_panicking_cpu_id;
2545
2546 if (sptm_get_panicking_cpu_id(&sptm_panicking_cpu_id) == LIBSPTM_SUCCESS &&
2547 sptm_panicking_cpu_id != sptm_cpu_id) {
2548 return false;
2549 }
2550 #endif /* CONFIG_SPTM */
2551 return true;
2552 }
2553