1 /*
2 * Copyright (c) 2007-2023 Apple Inc. All rights reserved.
3 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
4 */
5
6
7 /* Required to know if we must compile the file. */
8 #include <pexpert/arm64/board_config.h>
9
10 /* Generic headers. */
11 #include <pexpert/pexpert.h>
12 #include <pexpert/device_tree.h>
13 #include <machine/machine_routines.h>
14 #include <sys/sysctl.h>
15 #include <kern/clock.h>
16
17 /* Dev headers. */
18 #if DEVELOPMENT || DEBUG
19 #include <kern/simple_lock.h>
20 #include <os/hash.h>
21 #endif /* DEVELOPMENT || DEBUG */
22
23 /* Trace-specific headers. */
24
25 /********
26 * Logs *
27 ********/
28
29 #define PANIC_TRACE_LOG 1
30 #define panic_trace_error(msg, args...) { if (panic_trace_debug == 1) kprintf("panic_trace: " msg "\n", ##args); else if (panic_trace_debug == 2) printf("panic_trace: " msg "\n", ##args); }
31 #if PANIC_TRACE_LOG
32 #define panic_trace_log(msg, args...) { if (panic_trace_debug) panic_trace_error(msg, ##args); }
33 #else
34 #define panic_trace_log(msg, args...)
35 #endif /* PANIC_TRACE_LOG */
36
37 /************
38 * Externals *
39 ************/
40
41 /*
42 * Soc base physical address.
43 * Set by pe_identify_machine.c:pe_arm_map_interrupt_controller during
44 * early boot, null before.
45 */
46 extern vm_offset_t gSocPhys;
47
48 /*******
49 * Logs *
50 *******/
51
52 #if DEVELOPMENT || DEBUG
53 #ifndef CT_DFT_LOGS_ON
54 #define CT_DFT_LOGS_ON 0
55 #endif /* CT_DFT_LOGS_ON */
56 #endif /* DEVELOPMENT || DEBUG */
57
58 /****************
59 * Default state *
60 ****************/
61
62 #if DEVELOPMENT || DEBUG
63
64 /*
65 * When supported, panic-trace is enabled by default on some platforms.
66 * This section defines on which platform it is enabled..
67 */
68
69 /* Opensource -> disabled. */
70 #define DEFAULT_PANIC_TRACE_MODE panic_trace_disabled
71
72 #endif /* DEVELOPMENT || DEBUG */
73
74 /**********
75 * Globals *
76 **********/
77
78 #if DEVELOPMENT || DEBUG
79 boolean_t panic_trace_disabled_for_rdar107003520 = FALSE;
80 #endif /* DEVELOPMENT || DEBUG */
81
82 static boolean_t debug_and_trace_initialized = false;
83
84 #if DEVELOPMENT || DEBUG
85 static boolean_t _panic_trace_always_enabled = false;
86
87 static boolean_t _panic_trace_stress_racks = false;
88 #endif /* DEVELOPMENT || DEBUG */
89
90 /************
91 * Boot-args *
92 ************/
93
94 #if DEVELOPMENT || DEBUG
95 /*
96 * Panic trace state.
97 * Has a double meaning :
98 * - at system init, it gives the expected tracing state.
99 * -> init code uses that to enable tracing.
100 * - after system init, used to report the tracing state.
101 */
102 TUNABLE_DT_WRITEABLE(panic_trace_t, panic_trace, "/arm-io/cpu-debug-interface",
103 "panic-trace-mode", "panic_trace", DEFAULT_PANIC_TRACE_MODE, TUNABLE_DT_NONE);
104
105 /*
106 * Panic trace debug state. See 'Logs' section above.
107 */
108 TUNABLE_WRITEABLE(boolean_t, panic_trace_debug, "panic_trace_debug", CT_DFT_LOGS_ON);
109
110 #endif /* DEVELOPMENT || DEBUG */
111
112 /********
113 * Locks *
114 ********/
115
116 /* Panic trace lock. */
117
118 /****************
119 * Debug command *
120 ****************/
121
122 #if DEVELOPMENT || DEBUG
123
124 decl_simple_lock_data(, panic_hook_lock);
125
126 TUNABLE(unsigned int, bootarg_stop_clocks, "stop_clocks", 0);
127
128 // The command buffer contains the converted commands from the device tree for commanding cpu_halt, enable_trace, etc.
129 #define DEBUG_COMMAND_BUFFER_SIZE 256
130 typedef struct command_buffer_element {
131 uintptr_t address;
132 uintptr_t address_pa;
133 uintptr_t value;
134 union cpu_selector {
135 uint16_t mask;
136 struct cpu_range {
137 uint8_t min_cpu;
138 uint8_t max_cpu;
139 } range;
140 } destination_cpu_selector;
141 uint16_t delay_us;
142 bool cpu_selector_is_range;
143 bool is_32bit;
144 } command_buffer_element_t;
145
146 #define CPU_SELECTOR_SHIFT (16)
147 #define CPU_SELECTOR_MASK (0xFFFF << CPU_SELECTOR_SHIFT)
148 #define REGISTER_OFFSET_MASK ((1 << CPU_SELECTOR_SHIFT) - 1)
149 #define REGISTER_OFFSET(register_prop) (register_prop & REGISTER_OFFSET_MASK)
150 #define CPU_SELECTOR(register_offset) ((register_offset & CPU_SELECTOR_MASK) >> CPU_SELECTOR_SHIFT) // Upper 16bits holds the cpu selector
151 #define MAX_WINDOW_SIZE 0xFFFF
152 #define DELAY_SHIFT (32)
153 #define DELAY_MASK (0xFFFFULL << DELAY_SHIFT)
154 #define DELAY_US(register_offset) ((register_offset & DELAY_MASK) >> DELAY_SHIFT)
155 #define CPU_SELECTOR_ISRANGE_MASK (1ULL << 62)
156 #define REGISTER_32BIT_MASK (1ULL << 63)
157 #define ALL_CPUS 0x0000
158 #define RESET_VIRTUAL_ADDRESS_WINDOW 0xFFFFFFFF
159
160 #define REGISTER_IS_32BIT(register_offset) ((register_offset & REGISTER_32BIT_MASK) != 0)
161 #define REGISTER_SIZE(register_offset) (REGISTER_IS_32BIT(register_offset) ? sizeof(uint32_t) : sizeof(uintptr_t))
162 #define CPU_SELECTOR_IS_RANGE(register_offset) ((register_offset & CPU_SELECTOR_ISRANGE_MASK) != 0)
163 #define CPU_SELECTOR_MIN_CPU(register_offset) ((CPU_SELECTOR(register_offset) & 0xff00) >> 8)
164 #define CPU_SELECTOR_MAX_CPU(register_offset) (CPU_SELECTOR(register_offset) & 0x00ff)
165
166 // Record which CPU is currently running one of our debug commands, so we can trap panic reentrancy to PE_arm_debug_panic_hook.
167 static int running_debug_command_on_cpu_number = -1;
168
169
170 // Determine whether the current debug command is intended for this CPU.
171 static inline bool
is_running_cpu_selected(command_buffer_element_t * command)172 is_running_cpu_selected(command_buffer_element_t *command)
173 {
174 assert(running_debug_command_on_cpu_number >= 0);
175 if (command->cpu_selector_is_range) {
176 return running_debug_command_on_cpu_number >= command->destination_cpu_selector.range.min_cpu
177 && running_debug_command_on_cpu_number <= command->destination_cpu_selector.range.max_cpu;
178 } else if (command->destination_cpu_selector.mask == ALL_CPUS) {
179 return true;
180 } else {
181 return !!(command->destination_cpu_selector.mask & (1 << running_debug_command_on_cpu_number));
182 }
183 }
184
185
186 // Pointers into debug_command_buffer for each operation. Assumes runtime will init them to zero.
187 static command_buffer_element_t *enable_stop_clocks;
188 static command_buffer_element_t *stop_clocks;
189
190 boolean_t
PE_arm_debug_and_trace_initialized(void)191 PE_arm_debug_and_trace_initialized(void)
192 {
193 return debug_and_trace_initialized;
194 }
195
196 static void
pe_init_debug_command(DTEntry entryP,command_buffer_element_t ** command_buffer,const char * entry_name)197 pe_init_debug_command(DTEntry entryP, command_buffer_element_t **command_buffer, const char* entry_name)
198 {
199 // statically allocate to prevent needing alloc at runtime
200 static command_buffer_element_t debug_command_buffer[DEBUG_COMMAND_BUFFER_SIZE];
201 static command_buffer_element_t *next_command_buffer_entry = debug_command_buffer;
202
203 // record this pointer but don't assign it to *command_buffer yet, in case we panic while half-initialized
204 command_buffer_element_t *command_starting_index = next_command_buffer_entry;
205
206 uintptr_t const *reg_prop;
207 uint32_t prop_size, reg_window_size = 0;
208 uintptr_t base_address_pa = 0, debug_reg_window = 0;
209
210 if (command_buffer == 0) {
211 panic_trace_log("%s: %s: no hook to assign this command to\n", __func__, entry_name);
212 return;
213 }
214
215 if (SecureDTGetProperty(entryP, entry_name, (void const **)®_prop, &prop_size) != kSuccess) {
216 panic("%s: %s: failed to read property from device tree", __func__, entry_name);
217 }
218
219 if (prop_size % (2 * sizeof(*reg_prop))) {
220 panic("%s: %s: property size %u bytes is not a multiple of %lu",
221 __func__, entry_name, prop_size, 2 * sizeof(*reg_prop));
222 }
223
224 // convert to real virt addresses and stuff commands into debug_command_buffer
225 for (; prop_size; reg_prop += 2, prop_size -= 2 * sizeof(*reg_prop)) {
226 if (*reg_prop == RESET_VIRTUAL_ADDRESS_WINDOW) {
227 debug_reg_window = 0; // Create a new window
228 } else if (debug_reg_window == 0) {
229 // create a window from virtual address to the specified physical address
230 base_address_pa = gSocPhys + *reg_prop;
231 reg_window_size = ((uint32_t)*(reg_prop + 1));
232 if (reg_window_size > MAX_WINDOW_SIZE) {
233 panic("%s: %s: %#x-byte window at #%lx exceeds maximum size of %#x",
234 __func__, entry_name, reg_window_size, base_address_pa, MAX_WINDOW_SIZE );
235 }
236 debug_reg_window = ml_io_map(base_address_pa, reg_window_size);
237 assert(debug_reg_window);
238 panic_trace_log("%s: %s: %#x bytes at %#lx mapped to %#lx\n",
239 __func__, entry_name, reg_window_size, base_address_pa, debug_reg_window );
240 } else {
241 if ((REGISTER_OFFSET(*reg_prop) + REGISTER_SIZE(*reg_prop)) > reg_window_size) {
242 panic("%s: %s[%ld]: %#lx(+%lu)-byte offset from %#lx exceeds allocated size of %#x",
243 __func__, entry_name, next_command_buffer_entry - command_starting_index,
244 REGISTER_OFFSET(*reg_prop), REGISTER_SIZE(*reg_prop), base_address_pa, reg_window_size );
245 }
246
247 if (next_command_buffer_entry - debug_command_buffer >= DEBUG_COMMAND_BUFFER_SIZE - 1) {
248 // can't use the very last entry, since we need it to terminate the command
249 panic("%s: %s[%ld]: out of space in command buffer",
250 __func__, entry_name, next_command_buffer_entry - command_starting_index );
251 }
252
253 next_command_buffer_entry->address = debug_reg_window + REGISTER_OFFSET(*reg_prop);
254 next_command_buffer_entry->address_pa = base_address_pa + REGISTER_OFFSET(*reg_prop);
255 next_command_buffer_entry->value = *(reg_prop + 1);
256 #if defined(__arm64__)
257 next_command_buffer_entry->delay_us = DELAY_US(*reg_prop);
258 next_command_buffer_entry->is_32bit = REGISTER_IS_32BIT(*reg_prop);
259 #else
260 next_command_buffer_entry->delay_us = 0;
261 next_command_buffer_entry->is_32bit = false;
262 #endif
263 if ((next_command_buffer_entry->cpu_selector_is_range = CPU_SELECTOR_IS_RANGE(*reg_prop))) {
264 next_command_buffer_entry->destination_cpu_selector.range.min_cpu = (uint8_t)CPU_SELECTOR_MIN_CPU(*reg_prop);
265 next_command_buffer_entry->destination_cpu_selector.range.max_cpu = (uint8_t)CPU_SELECTOR_MAX_CPU(*reg_prop);
266 } else {
267 next_command_buffer_entry->destination_cpu_selector.mask = (uint16_t)CPU_SELECTOR(*reg_prop);
268 }
269 next_command_buffer_entry++;
270 }
271 }
272
273 // null terminate the address field of the command to end it
274 (next_command_buffer_entry++)->address = 0;
275
276 // save pointer into table for this command
277 *command_buffer = command_starting_index;
278 }
279
280 static void
pe_run_debug_command(command_buffer_element_t * command_buffer)281 pe_run_debug_command(command_buffer_element_t *command_buffer)
282 {
283 if (!PE_arm_debug_and_trace_initialized()) {
284 /*
285 * In practice this can only happen if we panicked very early,
286 * when only the boot CPU is online and before it has finished
287 * initializing the debug and trace infrastructure. Avoid an
288 * unhelpful nested panic() here and instead resume execution
289 * to handle_debugger_trap(), which logs a user friendly error
290 * message before spinning forever.
291 */
292 return;
293 }
294
295 // When both the CPUs panic, one will get stuck on the lock and the other CPU will be halted when the first executes the debug command
296 simple_lock(&panic_hook_lock, LCK_GRP_NULL);
297
298 running_debug_command_on_cpu_number = cpu_number();
299
300 while (command_buffer && command_buffer->address) {
301 if (is_running_cpu_selected(command_buffer)) {
302 panic_trace_log("%s: cpu %d: reg write 0x%lx (VA 0x%lx):= 0x%lx",
303 __func__, running_debug_command_on_cpu_number, command_buffer->address_pa,
304 command_buffer->address, command_buffer->value);
305 if (command_buffer->is_32bit) {
306 *((volatile uint32_t*)(command_buffer->address)) = (uint32_t)(command_buffer->value);
307 } else {
308 *((volatile uintptr_t*)(command_buffer->address)) = command_buffer->value; // register = value;
309 }
310 if (command_buffer->delay_us != 0) {
311 uint64_t deadline;
312 nanoseconds_to_absolutetime(command_buffer->delay_us * NSEC_PER_USEC, &deadline);
313 deadline += ml_get_timebase();
314 while (ml_get_timebase() < deadline) {
315 os_compiler_barrier();
316 }
317 }
318 }
319 command_buffer++;
320 }
321
322 running_debug_command_on_cpu_number = -1;
323 simple_unlock(&panic_hook_lock);
324 }
325
326 #endif /* DEVELOPMENT || DEBUG */
327
328 /*****************
329 * Partial policy *
330 *****************/
331
332 /* Debug-only section. */
333 #if DEVELOPMENT || DEBUG
334
335 /* Util. */
336 #ifndef MIN
337 #define MIN(a, b) (((a) < (b)) ? (a) : (b))
338 #endif /* MIN */
339
340 /*
341 * The % of devices which will have panic_trace enabled when using a partial
342 * enablement policy.
343 */
344 static TUNABLE_DT(uint32_t, panic_trace_partial_percent,
345 "/arm-io/cpu-debug-interface", "panic-trace-partial-percent",
346 "panic_trace_partial_percent", 50, TUNABLE_DT_NONE);
347
348 /*
349 * Detect if we're running on stress-racks.
350 */
351 static boolean_t
_is_stress_racks(void)352 _is_stress_racks(void)
353 {
354 DTEntry ent = NULL;
355 const void *propP = NULL;
356 unsigned int size = 0;
357 if (SecureDTLookupEntry(NULL, "/chosen", &ent) == kSuccess &&
358 SecureDTGetProperty(ent, "stress-rack", &propP, &size) == kSuccess) {
359 return true;
360 }
361 return false;
362 }
363
364 /*
365 * Stress racks opt out of panic_trace, unless overridden by the panic_trace boot-arg.
366 */
367 static void
panic_trace_apply_stress_rack_policy(void)368 panic_trace_apply_stress_rack_policy(void)
369 {
370 if (PE_parse_boot_argn("panic_trace", NULL, 0)) {
371 // Prefer user specified boot-arg even when running on stress racks.
372 // Make an exception for devices with broken single-stepping.
373 } else {
374 panic_trace = 0;
375 }
376 }
377
378 /*
379 * When the `panic_trace_partial_policy` flag is set, not all devices will have
380 * the panic_trace settings applied. The actual % is determined by
381 * `panic_trace_partial_percent`.
382 * By using the ECID instead of a random number the process is made
383 * deterministic for any given device.
384 * This function disables panic trace if the device falls into the disabled %
385 * range. It otherwise leaves the panic_trace value unmodified.
386 * Called on the boot path, thus does not lock panic_trace_lock.
387 */
388 static void
panic_trace_apply_partial_policy(void)389 panic_trace_apply_partial_policy(void)
390 {
391 assert3u((panic_trace & panic_trace_partial_policy), !=, 0);
392
393 DTEntry ent = NULL;
394 unsigned int size = 0;
395 const void *ecid = NULL;
396
397 /* Grab the ECID. */
398 if (SecureDTLookupEntry(NULL, "/chosen", &ent) != kSuccess ||
399 SecureDTGetProperty(ent, "unique-chip-id", &ecid, &size) != kSuccess) {
400 panic_trace = panic_trace_disabled;
401 return;
402 }
403
404 /*
405 * Use os_hash_jenkins to convert the decidedly non-random ECID into
406 * something resembling a random number. Better (cryptographic) hash
407 * functions are not available at this point in boot.
408 */
409 const uint32_t rand = os_hash_jenkins(ecid, size);
410
411 /* Sanitize the percent value. */
412 const uint32_t percent = MIN(100, panic_trace_partial_percent);
413
414 /*
415 * Apply the ECID percent value. The bias here should be so tiny as to not
416 * matter for this purpose.
417 */
418 if ((rand % 100) >= percent) {
419 panic_trace = panic_trace_disabled;
420 }
421 }
422
423 #endif /* DEVELOPMENT || DEBUG */
424
425 /***************
426 * External API *
427 ***************/
428
429 #if DEVELOPMENT || DEBUG
430 void
PE_arm_debug_enable_trace(bool should_log)431 PE_arm_debug_enable_trace(bool should_log)
432 {
433 if (should_log) {
434 panic_trace_log("%s enter", __FUNCTION__);
435 }
436 if (should_log) {
437 panic_trace_log("%s exit", __FUNCTION__);
438 }
439 }
440 #endif /* DEVELOPMENT || DEBUG */
441
442 #if DEVELOPMENT || DEBUG
443 static void
PE_arm_panic_hook(const char * str __unused)444 PE_arm_panic_hook(const char *str __unused)
445 {
446 (void)str; // not used
447 #if defined(__arm64__) && !APPLEVIRTUALPLATFORM
448 /*
449 * For Fastsim support--inform the simulator that it can dump a
450 * panic trace now (so we don't capture all the panic handling).
451 * This constant is randomly chosen by agreement between xnu and
452 * Fastsim.
453 */
454 __asm__ volatile ("hint #0x4f");
455 #endif /* defined(__arm64__) && !APPLEVIRTUALPLATFORM */
456 if (bootarg_stop_clocks) {
457 pe_run_debug_command(stop_clocks);
458 }
459 // disable panic trace to snapshot its ringbuffer
460 // note: Not taking panic_trace_lock to avoid delaying cpu halt.
461 // This is known to be racy.
462 if (panic_trace) {
463 if (running_debug_command_on_cpu_number == cpu_number()) {
464 // This is going to end badly if we don't trap, since we'd be panic-ing during our own code
465 kprintf("## Panic Trace code caused the panic ##\n");
466 return; // allow the normal panic operation to occur.
467 }
468
469 // Stop tracing to freeze the buffer and return to normal panic processing.
470 }
471 }
472 #endif /* DEVELOPMENT || DEBUG */
473
474
475 #if DEVELOPMENT || DEBUG
476 void (*PE_arm_debug_panic_hook)(const char *str) = PE_arm_panic_hook;
477 #else
478 void(*const PE_arm_debug_panic_hook)(const char *str) = NULL;
479 #endif // DEVELOPMENT || DEBUG
480
481 void
PE_init_cpu(void)482 PE_init_cpu(void)
483 {
484 #if DEVELOPMENT || DEBUG
485 if (bootarg_stop_clocks) {
486 pe_run_debug_command(enable_stop_clocks);
487 }
488 #endif // DEVELOPMENT || DEBUG
489
490 pe_init_fiq();
491 }
492
493
494 void
PE_singlestep_hook(void)495 PE_singlestep_hook(void)
496 {
497 }
498
499 void
PE_panic_hook(const char * str __unused)500 PE_panic_hook(const char *str __unused)
501 {
502 if (PE_arm_debug_panic_hook != NULL) {
503 PE_arm_debug_panic_hook(str);
504 }
505 }
506
507 /*
508 * Early part of the debug system init.
509 * Ran on the boot CPU with VM system enabled, mappings to any region
510 * allowed, carveouts not yet enabled, SPR lockdown not applied.
511 */
512 void
pe_arm_debug_init_early(void * boot_cpu_data)513 pe_arm_debug_init_early(void *boot_cpu_data)
514 {
515 DTEntry entryP;
516 uintptr_t const *reg_prop;
517 uint32_t prop_size;
518
519 /* Require gSocPhys to be initialized. */
520 if (gSocPhys == 0) {
521 kprintf("pe_arm_init_debug: failed to initialize : gSocPhys == 0\n");
522 return;
523 }
524
525 #if DEVELOPMENT || DEBUG
526
527 /* Determine if we're enabled at 100% rate,
528 * report it globally. */
529 _panic_trace_always_enabled = (panic_trace & panic_trace_enabled) && !(panic_trace & panic_trace_partial_policy);
530
531 /* Determine if we're running on stress-racks,
532 * report it globally. */
533 _panic_trace_stress_racks = _is_stress_racks();
534
535 /* Update the panic_trace start policy depending on the execution environment. */
536 if ((panic_trace != 0) && (_panic_trace_stress_racks)) {
537 panic_trace_apply_stress_rack_policy();
538 }
539
540 if ((panic_trace & panic_trace_partial_policy) != 0) {
541 panic_trace_apply_partial_policy();
542 }
543 #endif /* DEVELOPMENT || DEBUG */
544
545 /* Lookup the cpu debug interface in the device tree. */
546 if (SecureDTFindEntry("device_type", "cpu-debug-interface", &entryP) == kSuccess) {
547 /* Initialize the arm debug interface. */
548 if (SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size) == kSuccess) {
549 ml_init_arm_debug_interface(boot_cpu_data, ml_io_map(gSocPhys + *reg_prop, *(reg_prop + 1)));
550 }
551
552 /* Initialze the stop-clocks infrastructure. */
553 #if DEVELOPMENT || DEBUG
554 if (bootarg_stop_clocks) {
555 pe_init_debug_command(entryP, &enable_stop_clocks, "enable_stop_clocks");
556 pe_init_debug_command(entryP, &stop_clocks, "stop_clocks");
557 }
558 #endif
559
560 /* Initialize panic-trace. */
561 #if DEVELOPMENT || DEBUG
562 simple_lock_init(&panic_hook_lock, 0); //assuming single threaded mode
563 #endif
564 } else {
565 #if DEVELOPMENT || DEBUG
566 const uint32_t dependent_modes = (panic_trace_enabled | panic_trace_alt_enabled);
567 if (bootarg_stop_clocks || (panic_trace & dependent_modes)) {
568 panic("failed to find cpu-debug-interface node in the EDT! "
569 "(required by `panic_trace={0x01, 0x10}` or `stop_clocks=1`)");
570 } else
571 #endif
572 {
573 kprintf("pe_arm_init_debug: failed to find cpu-debug-interface\n");
574 }
575 }
576
577
578 /* Report init. */
579 debug_and_trace_initialized = true;
580 }
581
582 /*
583 * Late part of the init of the debug system,
584 * when carveouts have been allocated.
585 */
586 void
pe_arm_debug_init_late(void)587 pe_arm_debug_init_late(void)
588 {
589 }
590
591
592 /*********************
593 * Panic-trace sysctl *
594 *********************/
595
596