1 /*
2 * Copyright (c) 2012-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39 #include <arm64/instructions.h>
40
41 #include <kern/debug.h>
42 #include <kern/restartable.h>
43 #include <kern/socd_client.h>
44 #include <kern/thread.h>
45 #include <kern/zalloc_internal.h>
46 #include <mach/exception.h>
47 #include <mach/arm/traps.h>
48 #include <mach/vm_types.h>
49 #include <mach/machine/thread_status.h>
50
51 #include <machine/atomic.h>
52 #include <machine/limits.h>
53
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/arm64/apple_arm64_regs.h>
56 #include <pexpert/arm64/board_config.h>
57
58 #include <vm/vm_page.h>
59 #include <vm/pmap.h>
60 #include <vm/vm_fault.h>
61 #include <vm/vm_kern.h>
62
63 #include <sys/errno.h>
64 #include <sys/kdebug.h>
65 #include <kperf/kperf.h>
66
67 #include <kern/policy_internal.h>
68 #if CONFIG_TELEMETRY
69 #include <kern/telemetry.h>
70 #endif
71
72 #include <prng/entropy.h>
73
74
75
76
77 #include <arm64/platform_error_handler.h>
78
79 #if KASAN_TBI
80 #include <san/kasan.h>
81 #endif /* KASAN_TBI */
82
83 #if CONFIG_UBSAN_MINIMAL
84 #include <san/ubsan_minimal.h>
85 #endif
86
87
88 #ifndef __arm64__
89 #error Should only be compiling for arm64.
90 #endif
91
92 #define TEST_CONTEXT32_SANITY(context) \
93 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
94 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
95
96 #define TEST_CONTEXT64_SANITY(context) \
97 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
98 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
99
100 #define ASSERT_CONTEXT_SANITY(context) \
101 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
102
103
104 #define COPYIN(src, dst, size) \
105 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
106 copyin_kern(src, dst, size) : \
107 copyin(src, dst, size)
108
109 #define COPYOUT(src, dst, size) \
110 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
111 copyout_kern(src, dst, size) : \
112 copyout(src, dst, size)
113
114 // Below is for concatenating a string param to a string literal
115 #define STR1(x) #x
116 #define STR(x) STR1(x)
117
118 #define ARM64_KDBG_CODE_KERNEL (0 << 8)
119 #define ARM64_KDBG_CODE_USER (1 << 8)
120 #define ARM64_KDBG_CODE_GUEST (2 << 8)
121
122 _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
123 _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
124
125 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
126
127 void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike;
128 void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t);
129
130
131
132 void sleh_irq(arm_saved_state_t *);
133 void sleh_fiq(arm_saved_state_t *);
134 void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
135 void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2;
136
137 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
138 static void sleh_interrupt_handler_epilogue(void);
139
140 static void handle_svc(arm_saved_state_t *);
141 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
142 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
143
144 static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr);
145 #if __has_feature(ptrauth_calls)
146 static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2;
147 #endif
148
149 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
150
151 static void handle_uncategorized(arm_saved_state_t *);
152
153 static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t);
154
155 static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2;
156
157 typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
158 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
159 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
160
161 static int is_vm_fault(fault_status_t);
162 static int is_translation_fault(fault_status_t);
163 static int is_alignment_fault(fault_status_t);
164
165 typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
166 static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
167 static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
168
169 static void handle_pc_align(arm_saved_state_t *ss) __dead2;
170 static void handle_sp_align(arm_saved_state_t *ss) __dead2;
171 static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
172 static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
173 static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2;
174
175 static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
176
177 static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
178
179 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2;
180
181 static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2;
182
183 extern void mach_kauth_cred_thread_update(void);
184 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
185
186 struct proc;
187
188 typedef uint32_t arm64_instr_t;
189
190 extern void
191 unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
192
193 extern void
194 mach_syscall(struct arm_saved_state*);
195
196 #if CONFIG_DTRACE
197 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
198 extern boolean_t dtrace_tally_fault(user_addr_t);
199
200 /*
201 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
202 * and paste the trap instructions
203 * over from that file. Need to keep these in sync!
204 */
205 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
206 #define FASTTRAP_THUMB32_INSTR 0xdefc
207 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
208
209 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
210 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
211 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
212
213 /* See <rdar://problem/4613924> */
214 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
215 #endif
216
217
218
219 extern void arm64_thread_exception_return(void) __dead2;
220
221 #if defined(APPLETYPHOON)
222 #define CPU_NAME "Typhoon"
223 #elif defined(APPLETWISTER)
224 #define CPU_NAME "Twister"
225 #elif defined(APPLEHURRICANE)
226 #define CPU_NAME "Hurricane"
227 #elif defined(APPLELIGHTNING)
228 #define CPU_NAME "Lightning"
229 #else
230 #define CPU_NAME "Unknown"
231 #endif
232
233 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
234 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
235 #define ESR_WT_REASON(esr) ((esr) & 0xff)
236
237 #define WT_REASON_NONE 0
238 #define WT_REASON_INTEGRITY_FAIL 1
239 #define WT_REASON_BAD_SYSCALL 2
240 #define WT_REASON_NOT_LOCKED 3
241 #define WT_REASON_ALREADY_LOCKED 4
242 #define WT_REASON_SW_REQ 5
243 #define WT_REASON_PT_INVALID 6
244 #define WT_REASON_PT_VIOLATION 7
245 #define WT_REASON_REG_VIOLATION 8
246 #endif
247
248 #if defined(HAS_IPI)
249 void cpu_signal_handler(void);
250 extern unsigned int gFastIPI;
251 #endif /* defined(HAS_IPI) */
252
253 static arm_saved_state64_t *original_faulting_state = NULL;
254
255
256 TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
257
258 extern vm_offset_t static_memory_end;
259
260 /*
261 * Fault copyio_recovery_entry in copyin/copyout routines.
262 *
263 * Offets are expressed in bytes from ©_recovery_table
264 */
265 struct copyio_recovery_entry {
266 ptrdiff_t cre_start;
267 ptrdiff_t cre_end;
268 ptrdiff_t cre_recovery;
269 };
270
271 extern struct copyio_recovery_entry copyio_recover_table[];
272 extern struct copyio_recovery_entry copyio_recover_table_end[];
273
274 static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)275 copyio_recovery_offset(uintptr_t addr)
276 {
277 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
278 }
279
280 static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)281 copyio_recovery_addr(ptrdiff_t offset)
282 {
283 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
284 }
285
286 static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(arm_saved_state_t * state)287 find_copyio_recovery_entry(arm_saved_state_t *state)
288 {
289 ptrdiff_t offset = copyio_recovery_offset(get_saved_state_pc(state));
290 struct copyio_recovery_entry *e;
291
292 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
293 if (offset >= e->cre_start && offset < e->cre_end) {
294 return e;
295 }
296 }
297
298 return NULL;
299 }
300
301 static inline uintptr_t
copyio_recovery_get_recover_addr(arm_saved_state_t * state)302 copyio_recovery_get_recover_addr(
303 arm_saved_state_t *state)
304 {
305 struct copyio_recovery_entry *e = find_copyio_recovery_entry(state);
306 if (e == NULL) {
307 panic("copyio recovery: couldn't find a range for %p",
308 (void *)get_saved_state_pc(state));
309 }
310 return copyio_recovery_addr(e->cre_recovery);
311 }
312
313 static inline int
is_vm_fault(fault_status_t status)314 is_vm_fault(fault_status_t status)
315 {
316 switch (status) {
317 case FSC_TRANSLATION_FAULT_L0:
318 case FSC_TRANSLATION_FAULT_L1:
319 case FSC_TRANSLATION_FAULT_L2:
320 case FSC_TRANSLATION_FAULT_L3:
321 case FSC_ACCESS_FLAG_FAULT_L1:
322 case FSC_ACCESS_FLAG_FAULT_L2:
323 case FSC_ACCESS_FLAG_FAULT_L3:
324 case FSC_PERMISSION_FAULT_L1:
325 case FSC_PERMISSION_FAULT_L2:
326 case FSC_PERMISSION_FAULT_L3:
327 return TRUE;
328 default:
329 return FALSE;
330 }
331 }
332
333 static inline int
is_translation_fault(fault_status_t status)334 is_translation_fault(fault_status_t status)
335 {
336 switch (status) {
337 case FSC_TRANSLATION_FAULT_L0:
338 case FSC_TRANSLATION_FAULT_L1:
339 case FSC_TRANSLATION_FAULT_L2:
340 case FSC_TRANSLATION_FAULT_L3:
341 return TRUE;
342 default:
343 return FALSE;
344 }
345 }
346
347 static inline int
is_permission_fault(fault_status_t status)348 is_permission_fault(fault_status_t status)
349 {
350 switch (status) {
351 case FSC_PERMISSION_FAULT_L1:
352 case FSC_PERMISSION_FAULT_L2:
353 case FSC_PERMISSION_FAULT_L3:
354 return TRUE;
355 default:
356 return FALSE;
357 }
358 }
359
360 static inline int
is_alignment_fault(fault_status_t status)361 is_alignment_fault(fault_status_t status)
362 {
363 return status == FSC_ALIGNMENT_FAULT;
364 }
365
366 static inline int
is_parity_error(fault_status_t status)367 is_parity_error(fault_status_t status)
368 {
369 switch (status) {
370 /*
371 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
372 * only applicable to AArch32 HSR register. Can this be removed?
373 */
374 case FSC_ASYNC_PARITY:
375 case FSC_SYNC_PARITY:
376 case FSC_SYNC_PARITY_TT_L1:
377 case FSC_SYNC_PARITY_TT_L2:
378 case FSC_SYNC_PARITY_TT_L3:
379 return TRUE;
380 default:
381 return FALSE;
382 }
383 }
384
385
386
387 __dead2 __unused
388 static void
arm64_implementation_specific_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far)389 arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
390 {
391 #pragma unused (state, esr, far)
392 panic_plain("Unhandled implementation specific error\n");
393 }
394
395 #if CONFIG_KERNEL_INTEGRITY
396 #pragma clang diagnostic push
397 #pragma clang diagnostic ignored "-Wunused-parameter"
398 static void
kernel_integrity_error_handler(uint32_t esr,vm_offset_t far)399 kernel_integrity_error_handler(uint32_t esr, vm_offset_t far)
400 {
401 #if defined(KERNEL_INTEGRITY_WT)
402 #if (DEVELOPMENT || DEBUG)
403 if (ESR_WT_SERROR(esr)) {
404 switch (ESR_WT_REASON(esr)) {
405 case WT_REASON_INTEGRITY_FAIL:
406 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
407 case WT_REASON_BAD_SYSCALL:
408 panic_plain("Kernel integrity, bad syscall.");
409 case WT_REASON_NOT_LOCKED:
410 panic_plain("Kernel integrity, not locked.");
411 case WT_REASON_ALREADY_LOCKED:
412 panic_plain("Kernel integrity, already locked.");
413 case WT_REASON_SW_REQ:
414 panic_plain("Kernel integrity, software request.");
415 case WT_REASON_PT_INVALID:
416 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
417 "walking 0x%016lx.", far);
418 case WT_REASON_PT_VIOLATION:
419 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
420 far);
421 case WT_REASON_REG_VIOLATION:
422 panic_plain("Kernel integrity, violation in system register %d.",
423 (unsigned) far);
424 default:
425 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
426 }
427 }
428 #else
429 if (ESR_WT_SERROR(esr)) {
430 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
431 }
432 #endif
433 #endif
434 }
435 #pragma clang diagnostic pop
436 #endif
437
438 static void
arm64_platform_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far,platform_error_source_t source)439 arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far, platform_error_source_t source)
440 {
441 #if CONFIG_KERNEL_INTEGRITY
442 kernel_integrity_error_handler(esr, far);
443 #endif
444
445 (void)source;
446 cpu_data_t *cdp = getCpuDatap();
447
448 if (PE_handle_platform_error(far)) {
449 return;
450 } else if (cdp->platform_error_handler != NULL) {
451 cdp->platform_error_handler(cdp->cpu_id, far);
452 } else {
453 arm64_implementation_specific_error(state, esr, far);
454 }
455 }
456
457 void
panic_with_thread_kernel_state(const char * msg,arm_saved_state_t * ss)458 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
459 {
460 boolean_t ss_valid;
461
462 ss_valid = is_saved_state64(ss);
463 arm_saved_state64_t *state = saved_state64(ss);
464
465 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
466
467 // rdar://80659177
468 // Read SoCD tracepoints up to twice — once the first time we call panic and
469 // another time if we encounter a nested panic after that.
470 static int twice = 2;
471 if (twice > 0) {
472 twice--;
473 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc),
474 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
475 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
476 VALUE(state->far));
477 }
478
479
480 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
481 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
482 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
483 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
484 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
485 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
486 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
487 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
488 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
489 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
490 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
491 state->x[0], state->x[1], state->x[2], state->x[3],
492 state->x[4], state->x[5], state->x[6], state->x[7],
493 state->x[8], state->x[9], state->x[10], state->x[11],
494 state->x[12], state->x[13], state->x[14], state->x[15],
495 state->x[16], state->x[17], state->x[18], state->x[19],
496 state->x[20], state->x[21], state->x[22], state->x[23],
497 state->x[24], state->x[25], state->x[26], state->x[27],
498 state->x[28], state->fp, state->lr, state->sp,
499 state->pc, state->cpsr, state->esr, state->far);
500 }
501
502 void
sleh_synchronous_sp1(arm_context_t * context,uint32_t esr,vm_offset_t far __unused)503 sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
504 {
505 esr_exception_class_t class = ESR_EC(esr);
506 arm_saved_state_t * state = &context->ss;
507
508 switch (class) {
509 case ESR_EC_UNCATEGORIZED:
510 {
511 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
512 if (IS_ARM_GDB_TRAP(instr)) {
513 DebuggerCall(EXC_BREAKPOINT, state);
514 }
515 }
516 OS_FALLTHROUGH; // panic if we return from the debugger
517 default:
518 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
519 }
520 }
521
522
523 __attribute__((noreturn))
524 void
thread_exception_return()525 thread_exception_return()
526 {
527 thread_t thread = current_thread();
528 if (thread->machine.exception_trace_code != 0) {
529 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
530 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
531 thread->machine.exception_trace_code = 0;
532 }
533
534 #if KASAN_TBI
535 kasan_unpoison_curstack(true);
536 #endif /* KASAN_TBI */
537 arm64_thread_exception_return();
538 __builtin_unreachable();
539 }
540
541 /*
542 * check whether task vtimers are running and set thread and CPU BSD AST
543 *
544 * must be called with interrupts masked so updates of fields are atomic
545 * must be emitted inline to avoid generating an FBT probe on the exception path
546 *
547 */
548 __attribute__((__always_inline__))
549 static inline void
task_vtimer_check(thread_t thread)550 task_vtimer_check(thread_t thread)
551 {
552 task_t task = get_threadtask_early(thread);
553
554 if (__improbable(task != NULL && task->vtimers)) {
555 thread_ast_set(thread, AST_BSD);
556 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
557 }
558 }
559
560 #if MACH_ASSERT
561 /**
562 * A version of get_preemption_level() that works in early boot.
563 *
564 * If an exception is raised in early boot before the initial thread has been
565 * set up, then calling get_preemption_level() in the SLEH will trigger an
566 * infinitely-recursing exception. This function handles this edge case.
567 */
568 static inline int
sleh_get_preemption_level(void)569 sleh_get_preemption_level(void)
570 {
571 if (__improbable(current_thread() == NULL)) {
572 return 0;
573 }
574 return get_preemption_level();
575 }
576 #endif // MACH_ASSERT
577
578 static inline bool
is_platform_error(uint32_t esr)579 is_platform_error(uint32_t esr)
580 {
581 esr_exception_class_t class = ESR_EC(esr);
582 uint32_t iss = ESR_ISS(esr);
583 fault_status_t fault_code;
584
585 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
586 fault_code = ISS_DA_FSC(iss);
587 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
588 fault_code = ISS_IA_FSC(iss);
589 } else {
590 return false;
591 }
592
593 return fault_code == FSC_SYNC_PARITY;
594 }
595
596 void
sleh_synchronous(arm_context_t * context,uint32_t esr,vm_offset_t far)597 sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far)
598 {
599 esr_exception_class_t class = ESR_EC(esr);
600 arm_saved_state_t * state = &context->ss;
601 thread_t thread = current_thread();
602 #if MACH_ASSERT
603 int preemption_level = sleh_get_preemption_level();
604 #endif
605 expected_fault_handler_t expected_fault_handler = NULL;
606 #ifdef CONFIG_XNUPOST
607 expected_fault_handler_t saved_expected_fault_handler = NULL;
608 uintptr_t saved_expected_fault_addr = 0;
609 #endif /* CONFIG_XNUPOST */
610
611 ASSERT_CONTEXT_SANITY(context);
612
613 task_vtimer_check(thread);
614
615 #if CONFIG_DTRACE
616 /*
617 * Handle kernel DTrace probes as early as possible to minimize the likelihood
618 * that this path will itself trigger a DTrace probe, which would lead to infinite
619 * probe recursion.
620 */
621 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
622 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
623 return;
624 }
625 #endif
626 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
627
628 /*
629 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
630 * that would disclose the behavior of PT_DENY_ATTACH processes.
631 */
632 if (is_user) {
633 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
634 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
635 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
636 esr, far, get_saved_state_pc(state), 0, 0);
637 } else {
638 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
639 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
640 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
641 }
642
643 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
644 /*
645 * We no longer support 32-bit, which means no 2-byte
646 * instructions.
647 */
648 if (is_user) {
649 panic("Exception on 2-byte instruction, "
650 "context=%p, esr=%#x, far=%p",
651 context, esr, (void *)far);
652 } else {
653 panic_with_thread_kernel_state("Exception on 2-byte instruction", state);
654 }
655 }
656
657 #ifdef CONFIG_XNUPOST
658 if (thread->machine.expected_fault_handler != NULL) {
659 saved_expected_fault_handler = thread->machine.expected_fault_handler;
660 saved_expected_fault_addr = thread->machine.expected_fault_addr;
661
662 thread->machine.expected_fault_handler = NULL;
663 thread->machine.expected_fault_addr = 0;
664
665 if (saved_expected_fault_addr == far) {
666 expected_fault_handler = saved_expected_fault_handler;
667 }
668 }
669 #endif /* CONFIG_XNUPOST */
670
671 if (__improbable(is_platform_error(esr))) {
672 /*
673 * Must gather error info in platform error handler before
674 * thread is preempted to another core/cluster to guarantee
675 * accurate error details
676 */
677
678 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_SYNC);
679 return;
680 }
681
682 if (is_user && class == ESR_EC_DABORT_EL0) {
683 thread_reset_pcs_will_fault(thread);
684 }
685
686 /* Inherit the interrupt masks from previous context */
687 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
688 ml_set_interrupts_enabled(TRUE);
689 }
690
691 switch (class) {
692 case ESR_EC_SVC_64:
693 if (!is_saved_state64(state) || !is_user) {
694 panic("Invalid SVC_64 context");
695 }
696
697 handle_svc(state);
698 break;
699
700 case ESR_EC_DABORT_EL0:
701 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
702 break;
703
704 case ESR_EC_MSR_TRAP:
705 handle_msr_trap(state, esr);
706 break;
707 /**
708 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for
709 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host
710 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC
711 * handler here needs to be built into the guest kernels to prevent the exception to fall
712 * through.
713 */
714 #if __has_feature(ptrauth_calls)
715 case ESR_EC_PAC_FAIL:
716 handle_pac_fail(state, esr);
717 __builtin_unreachable();
718
719 #endif /* __has_feature(ptrauth_calls) */
720
721
722 case ESR_EC_IABORT_EL0:
723 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
724 break;
725
726 case ESR_EC_IABORT_EL1:
727 #ifdef CONFIG_XNUPOST
728 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
729 break;
730 }
731 #endif /* CONFIG_XNUPOST */
732
733 panic_with_thread_kernel_state("Kernel instruction fetch abort", state);
734
735 case ESR_EC_PC_ALIGN:
736 handle_pc_align(state);
737 __builtin_unreachable();
738
739 case ESR_EC_DABORT_EL1:
740 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
741 break;
742
743 case ESR_EC_UNCATEGORIZED:
744 assert(!ESR_ISS(esr));
745
746 handle_uncategorized(&context->ss);
747 break;
748
749 case ESR_EC_SP_ALIGN:
750 handle_sp_align(state);
751 __builtin_unreachable();
752
753 case ESR_EC_BKPT_AARCH32:
754 handle_breakpoint(state, esr);
755 __builtin_unreachable();
756
757 case ESR_EC_BRK_AARCH64:
758 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
759 handle_kernel_breakpoint(state, esr);
760 break;
761 } else {
762 handle_breakpoint(state, esr);
763 __builtin_unreachable();
764 }
765
766 case ESR_EC_BKPT_REG_MATCH_EL0:
767 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
768 handle_breakpoint(state, esr);
769 }
770 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
771 class, state, class, esr, (void *)far);
772 __builtin_unreachable();
773
774 case ESR_EC_BKPT_REG_MATCH_EL1:
775 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state);
776 __builtin_unreachable();
777
778 case ESR_EC_SW_STEP_DEBUG_EL0:
779 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
780 handle_sw_step_debug(state);
781 }
782 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
783 class, state, class, esr, (void *)far);
784 __builtin_unreachable();
785
786 case ESR_EC_SW_STEP_DEBUG_EL1:
787 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state);
788 __builtin_unreachable();
789
790 case ESR_EC_WATCHPT_MATCH_EL0:
791 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
792 handle_watchpoint(far);
793 }
794 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
795 class, state, class, esr, (void *)far);
796 __builtin_unreachable();
797
798 case ESR_EC_WATCHPT_MATCH_EL1:
799 /*
800 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
801 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
802 */
803 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
804 arm_debug_set(NULL);
805 break; /* return to first level handler */
806 }
807 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
808 class, state, class, esr, (void *)far);
809 __builtin_unreachable();
810
811 case ESR_EC_TRAP_SIMD_FP:
812 handle_simd_trap(state, esr);
813 __builtin_unreachable();
814
815 case ESR_EC_ILLEGAL_INSTR_SET:
816 if (EXCB_ACTION_RERUN !=
817 ex_cb_invoke(EXCB_CLASS_ILLEGAL_INSTR_SET, far)) {
818 // instruction is not re-executed
819 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
820 state, class, esr, (void *)far, get_saved_state_cpsr(state));
821 }
822 // must clear this fault in PSR to re-run
823 mask_saved_state_cpsr(state, 0, PSR64_IL);
824 break;
825
826 case ESR_EC_MCR_MRC_CP15_TRAP:
827 case ESR_EC_MCRR_MRRC_CP15_TRAP:
828 case ESR_EC_MCR_MRC_CP14_TRAP:
829 case ESR_EC_LDC_STC_CP14_TRAP:
830 case ESR_EC_MCRR_MRRC_CP14_TRAP:
831 handle_user_trapped_instruction32(state, esr);
832 __builtin_unreachable();
833
834 case ESR_EC_WFI_WFE:
835 // Use of WFI or WFE instruction when they have been disabled for EL0
836 handle_wf_trap(state);
837 __builtin_unreachable();
838
839 case ESR_EC_FLOATING_POINT_64:
840 handle_fp_trap(state, esr);
841 __builtin_unreachable();
842
843 default:
844 handle_uncategorized(state);
845 }
846
847 #ifdef CONFIG_XNUPOST
848 if (saved_expected_fault_handler != NULL) {
849 thread->machine.expected_fault_handler = saved_expected_fault_handler;
850 thread->machine.expected_fault_addr = saved_expected_fault_addr;
851 }
852 #endif /* CONFIG_XNUPOST */
853
854 if (is_user) {
855 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
856 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
857 esr, far, get_saved_state_pc(state), 0, 0);
858 thread->machine.exception_trace_code = 0;
859 } else {
860 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
861 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
862 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
863 }
864 #if MACH_ASSERT
865 if (preemption_level != sleh_get_preemption_level()) {
866 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
867 }
868 #endif
869 }
870
871 /*
872 * Uncategorized exceptions are a catch-all for general execution errors.
873 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
874 */
875 static void
handle_uncategorized(arm_saved_state_t * state)876 handle_uncategorized(arm_saved_state_t *state)
877 {
878 exception_type_t exception = EXC_BAD_INSTRUCTION;
879 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
880 mach_msg_type_number_t numcodes = 2;
881 uint32_t instr = 0;
882
883 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
884
885 #if CONFIG_DTRACE
886
887 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
888 /*
889 * For a 64bit user process, we care about all 4 bytes of the
890 * instr.
891 */
892 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
893 if (dtrace_user_probe(state) == KERN_SUCCESS) {
894 return;
895 }
896 }
897 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
898 /*
899 * For a 32bit user process, we check for thumb mode, in
900 * which case we only care about a 2 byte instruction length.
901 * For non-thumb mode, we care about all 4 bytes of the instructin.
902 */
903 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
904 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
905 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
906 if (dtrace_user_probe(state) == KERN_SUCCESS) {
907 return;
908 }
909 }
910 } else {
911 if ((instr == FASTTRAP_ARM32_INSTR) ||
912 (instr == FASTTRAP_ARM32_RET_INSTR)) {
913 if (dtrace_user_probe(state) == KERN_SUCCESS) {
914 return;
915 }
916 }
917 }
918 }
919
920 #endif /* CONFIG_DTRACE */
921
922 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
923 if (IS_ARM_GDB_TRAP(instr)) {
924 boolean_t interrupt_state;
925 exception = EXC_BREAKPOINT;
926
927 interrupt_state = ml_set_interrupts_enabled(FALSE);
928
929 /* Save off the context here (so that the debug logic
930 * can see the original state of this thread).
931 */
932 current_thread()->machine.kpcb = state;
933
934 /* Hop into the debugger (typically either due to a
935 * fatal exception, an explicit panic, or a stackshot
936 * request.
937 */
938 DebuggerCall(exception, state);
939
940 current_thread()->machine.kpcb = NULL;
941 (void) ml_set_interrupts_enabled(interrupt_state);
942 return;
943 } else {
944 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
945 }
946 }
947
948 /*
949 * Check for GDB breakpoint via illegal opcode.
950 */
951 if (IS_ARM_GDB_TRAP(instr)) {
952 exception = EXC_BREAKPOINT;
953 codes[0] = EXC_ARM_BREAKPOINT;
954 codes[1] = instr;
955 } else {
956 codes[1] = instr;
957 }
958
959 exception_triage(exception, codes, numcodes);
960 __builtin_unreachable();
961 }
962
963 #if __has_feature(ptrauth_calls)
964 static const uint16_t PTRAUTH_TRAPS_START = 0xC470;
965 static inline bool
brk_comment_is_ptrauth(uint16_t comment)966 brk_comment_is_ptrauth(uint16_t comment)
967 {
968 return comment >= PTRAUTH_TRAPS_START &&
969 comment <= PTRAUTH_TRAPS_START + ptrauth_key_asdb;
970 }
971
972 static inline const char *
ptrauth_key_to_string(ptrauth_key key)973 ptrauth_key_to_string(ptrauth_key key)
974 {
975 switch (key) {
976 case ptrauth_key_asia:
977 return "IA";
978 case ptrauth_key_asib:
979 return "IB";
980 case ptrauth_key_asda:
981 return "DA";
982 case ptrauth_key_asdb:
983 return "DB";
984 default:
985 __builtin_unreachable();
986 }
987 }
988
989 static void __attribute__((noreturn))
ptrauth_handle_brk_trap(void * tstate,uint16_t comment)990 ptrauth_handle_brk_trap(void *tstate, uint16_t comment)
991 {
992 arm_saved_state_t *state = (arm_saved_state_t *)tstate;
993 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx"
994 char msg[strlen(MSG_FMT)
995 - strlen("0x%04X") + strlen("0xFFFF")
996 - strlen("%s") + strlen("IA")
997 - strlen("0x%016llx") + strlen("0xFFFFFFFFFFFFFFFF")
998 + 1];
999 ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAPS_START);
1000 const char *key_str = ptrauth_key_to_string(key);
1001 snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]);
1002 #undef MSG_FMT
1003
1004 panic_with_thread_kernel_state(msg, state);
1005 __builtin_unreachable();
1006 }
1007 #endif /* __has_feature(ptrauth_calls) */
1008
1009 static uint32_t bound_chk_violations_event;
1010
1011 static void
telemetry_handle_brk_trap(void * tstate,uint16_t comment)1012 telemetry_handle_brk_trap(
1013 void *tstate,
1014 uint16_t comment)
1015 {
1016 #if CONFIG_UBSAN_MINIMAL
1017 if (comment == UBSAN_SIGNED_OVERFLOW_TRAP) {
1018 ubsan_handle_brk_trap(tstate, comment);
1019 }
1020 #else
1021 (void)tstate;
1022 #endif
1023
1024 if (comment == CLANG_BOUND_CHK_SOFT_TRAP) {
1025 os_atomic_inc(&bound_chk_violations_event, relaxed);
1026 }
1027 }
1028
1029 #if __has_feature(ptrauth_calls)
1030 KERNEL_BRK_DESCRIPTOR_DEFINE(ptrauth_desc,
1031 .type = KERNEL_BRK_TYPE_PTRAUTH,
1032 .base = PTRAUTH_TRAPS_START,
1033 .max = PTRAUTH_TRAPS_START + ptrauth_key_asdb,
1034 .options = KERNEL_BRK_UNRECOVERABLE,
1035 .handle_breakpoint = ptrauth_handle_brk_trap);
1036 #endif
1037
1038 KERNEL_BRK_DESCRIPTOR_DEFINE(clang_desc,
1039 .type = KERNEL_BRK_TYPE_CLANG,
1040 .base = CLANG_TRAPS_ARM_START,
1041 .max = CLANG_TRAPS_ARM_END,
1042 .options = KERNEL_BRK_UNRECOVERABLE,
1043 .handle_breakpoint = NULL);
1044
1045 KERNEL_BRK_DESCRIPTOR_DEFINE(libcxx_desc,
1046 .type = KERNEL_BRK_TYPE_LIBCXX,
1047 .base = LIBCXX_TRAPS_START,
1048 .max = LIBCXX_TRAPS_END,
1049 .options = KERNEL_BRK_UNRECOVERABLE,
1050 .handle_breakpoint = NULL);
1051
1052 KERNEL_BRK_DESCRIPTOR_DEFINE(telemetry_desc,
1053 .type = KERNEL_BRK_TYPE_TELEMETRY,
1054 .base = TELEMETRY_TRAPS_START,
1055 .max = TELEMETRY_TRAPS_END,
1056 .options = KERNEL_BRK_RECOVERABLE | KERNEL_BRK_CORE_ANALYTICS,
1057 .handle_breakpoint = telemetry_handle_brk_trap);
1058
1059 static void
handle_kernel_breakpoint(arm_saved_state_t * state,uint32_t esr)1060 handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr)
1061 {
1062 uint16_t comment = ISS_BRK_COMMENT(esr);
1063 const struct kernel_brk_descriptor *desc;
1064
1065 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)"
1066 char msg[strlen(MSG_FMT) - strlen("0x%04X") + strlen("0xFFFF") + 1];
1067
1068 desc = find_brk_descriptor_by_comment(comment);
1069
1070 if (!desc) {
1071 goto brk_out;
1072 }
1073
1074 if (desc->options & KERNEL_BRK_TELEMETRY_OPTIONS) {
1075 telemetry_kernel_brk(desc->type, desc->options, (void *)state, comment);
1076 }
1077
1078 if (desc->handle_breakpoint) {
1079 desc->handle_breakpoint(state, comment); /* May trigger panic */
1080 }
1081
1082 /* Still alive? Check if we should recover. */
1083 if (desc->options & KERNEL_BRK_RECOVERABLE) {
1084 add_saved_state_pc(state, 4);
1085 return;
1086 }
1087
1088 brk_out:
1089 snprintf(msg, sizeof(msg), MSG_FMT, comment);
1090
1091 panic_with_thread_kernel_state(msg, state);
1092 __builtin_unreachable();
1093 #undef MSG_FMT
1094 }
1095
1096 static void
handle_breakpoint(arm_saved_state_t * state,uint32_t esr __unused)1097 handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused)
1098 {
1099 exception_type_t exception = EXC_BREAKPOINT;
1100 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1101 mach_msg_type_number_t numcodes = 2;
1102
1103 #if __has_feature(ptrauth_calls)
1104 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1105 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1106 exception |= EXC_PTRAUTH_BIT;
1107 }
1108 #endif /* __has_feature(ptrauth_calls) */
1109
1110 codes[1] = get_saved_state_pc(state);
1111 exception_triage(exception, codes, numcodes);
1112 __builtin_unreachable();
1113 }
1114
1115 static void
handle_watchpoint(vm_offset_t fault_addr)1116 handle_watchpoint(vm_offset_t fault_addr)
1117 {
1118 exception_type_t exception = EXC_BREAKPOINT;
1119 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1120 mach_msg_type_number_t numcodes = 2;
1121
1122 codes[1] = fault_addr;
1123 exception_triage(exception, codes, numcodes);
1124 __builtin_unreachable();
1125 }
1126
1127 static void
handle_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,abort_inspector_t inspect_abort,abort_handler_t handler,expected_fault_handler_t expected_fault_handler)1128 handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1129 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1130 {
1131 fault_status_t fault_code;
1132 vm_prot_t fault_type;
1133
1134 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1135 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1136 }
1137
1138 static void
inspect_instruction_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1139 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1140 {
1141 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1142 *fault_code = ISS_IA_FSC(iss);
1143 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1144 }
1145
1146 static void
inspect_data_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1147 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1148 {
1149 getCpuDatap()->cpu_stat.data_ex_cnt++;
1150 *fault_code = ISS_DA_FSC(iss);
1151
1152 /*
1153 * Cache maintenance operations always report faults as write access.
1154 * Change these to read access, unless they report a permission fault.
1155 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1156 * access to the mapping, but if a cache maintenance operation that only requires
1157 * read access generates a permission fault, then we will not be able to handle
1158 * the fault regardless of whether we treat it as a read or write fault.
1159 */
1160 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
1161 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1162 } else {
1163 *fault_type = (VM_PROT_READ);
1164 }
1165 }
1166
1167 #if __has_feature(ptrauth_calls)
1168 static inline uint64_t
fault_addr_bitmask(unsigned int bit_from,unsigned int bit_to)1169 fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1170 {
1171 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1172 }
1173
1174 static inline bool
fault_addr_bit(vm_offset_t fault_addr,unsigned int bit)1175 fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1176 {
1177 return (bool)((fault_addr >> bit) & 1);
1178 }
1179
1180 extern int gARM_FEAT_PAuth2;
1181
1182 /**
1183 * Determines whether a fault address taken at EL0 contains a PAC error code
1184 * corresponding to the specified kind of ptrauth key.
1185 */
1186 static bool
user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr,bool data_key)1187 user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key)
1188 {
1189 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1190 bool tbi = data_key || __improbable(instruction_tbi);
1191
1192 if (gARM_FEAT_PAuth2) {
1193 /*
1194 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1195 * treat all non-canonical address bits like potential poison bits.
1196 */
1197 uint64_t mask = fault_addr_bitmask(T0SZ_BOOT, 54);
1198 if (!tbi) {
1199 mask |= fault_addr_bitmask(56, 63);
1200 }
1201 return (fault_addr & mask) != 0;
1202 } else {
1203 unsigned int poison_shift;
1204 if (tbi) {
1205 poison_shift = 53;
1206 } else {
1207 poison_shift = 61;
1208 }
1209
1210 /* PAC error codes are always in the form key_number:NOT(key_number) */
1211 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1212 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1213 return poison_bit_1 != poison_bit_2;
1214 }
1215 }
1216 #endif /* __has_feature(ptrauth_calls) */
1217
1218 static void
handle_pc_align(arm_saved_state_t * ss)1219 handle_pc_align(arm_saved_state_t *ss)
1220 {
1221 exception_type_t exc;
1222 mach_exception_data_type_t codes[2];
1223 mach_msg_type_number_t numcodes = 2;
1224
1225 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1226 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
1227 }
1228
1229 exc = EXC_BAD_ACCESS;
1230 #if __has_feature(ptrauth_calls)
1231 if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) {
1232 exc |= EXC_PTRAUTH_BIT;
1233 }
1234 #endif /* __has_feature(ptrauth_calls) */
1235
1236 codes[0] = EXC_ARM_DA_ALIGN;
1237 codes[1] = get_saved_state_pc(ss);
1238
1239 exception_triage(exc, codes, numcodes);
1240 __builtin_unreachable();
1241 }
1242
1243 static void
handle_sp_align(arm_saved_state_t * ss)1244 handle_sp_align(arm_saved_state_t *ss)
1245 {
1246 exception_type_t exc;
1247 mach_exception_data_type_t codes[2];
1248 mach_msg_type_number_t numcodes = 2;
1249
1250 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1251 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
1252 }
1253
1254 exc = EXC_BAD_ACCESS;
1255 #if __has_feature(ptrauth_calls)
1256 if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) {
1257 exc |= EXC_PTRAUTH_BIT;
1258 }
1259 #endif /* __has_feature(ptrauth_calls) */
1260
1261 codes[0] = EXC_ARM_SP_ALIGN;
1262 codes[1] = get_saved_state_sp(ss);
1263
1264 exception_triage(exc, codes, numcodes);
1265 __builtin_unreachable();
1266 }
1267
1268 static void
handle_wf_trap(arm_saved_state_t * state)1269 handle_wf_trap(arm_saved_state_t *state)
1270 {
1271 exception_type_t exc;
1272 mach_exception_data_type_t codes[2];
1273 mach_msg_type_number_t numcodes = 2;
1274 uint32_t instr = 0;
1275
1276 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1277
1278 exc = EXC_BAD_INSTRUCTION;
1279 codes[0] = EXC_ARM_UNDEFINED;
1280 codes[1] = instr;
1281
1282 exception_triage(exc, codes, numcodes);
1283 __builtin_unreachable();
1284 }
1285
1286 static void
handle_fp_trap(arm_saved_state_t * state,uint32_t esr)1287 handle_fp_trap(arm_saved_state_t *state, uint32_t esr)
1288 {
1289 exception_type_t exc = EXC_ARITHMETIC;
1290 mach_exception_data_type_t codes[2];
1291 mach_msg_type_number_t numcodes = 2;
1292 uint32_t instr = 0;
1293
1294 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1295 panic_with_thread_kernel_state("Floating point exception from kernel", state);
1296 }
1297
1298 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1299 codes[1] = instr;
1300
1301 /* The floating point trap flags are only valid if TFV is set. */
1302 if (!fp_exceptions_enabled) {
1303 exc = EXC_BAD_INSTRUCTION;
1304 codes[0] = EXC_ARM_UNDEFINED;
1305 } else if (!(esr & ISS_FP_TFV)) {
1306 codes[0] = EXC_ARM_FP_UNDEFINED;
1307 } else if (esr & ISS_FP_UFF) {
1308 codes[0] = EXC_ARM_FP_UF;
1309 } else if (esr & ISS_FP_OFF) {
1310 codes[0] = EXC_ARM_FP_OF;
1311 } else if (esr & ISS_FP_IOF) {
1312 codes[0] = EXC_ARM_FP_IO;
1313 } else if (esr & ISS_FP_DZF) {
1314 codes[0] = EXC_ARM_FP_DZ;
1315 } else if (esr & ISS_FP_IDF) {
1316 codes[0] = EXC_ARM_FP_ID;
1317 } else if (esr & ISS_FP_IXF) {
1318 codes[0] = EXC_ARM_FP_IX;
1319 } else {
1320 panic("Unrecognized floating point exception, state=%p, esr=%#x", state, esr);
1321 }
1322
1323 exception_triage(exc, codes, numcodes);
1324 __builtin_unreachable();
1325 }
1326
1327
1328
1329 /*
1330 * handle_alignment_fault_from_user:
1331 * state: Saved state
1332 *
1333 * Attempts to deal with an alignment fault from userspace (possibly by
1334 * emulating the faulting instruction). If emulation failed due to an
1335 * unservicable fault, the ESR for that fault will be stored in the
1336 * recovery_esr field of the thread by the exception code.
1337 *
1338 * Returns:
1339 * -1: Emulation failed (emulation of state/instr not supported)
1340 * 0: Successfully emulated the instruction
1341 * EFAULT: Emulation failed (probably due to permissions)
1342 * EINVAL: Emulation failed (probably due to a bad address)
1343 */
1344
1345
1346 static int
handle_alignment_fault_from_user(arm_saved_state_t * state,kern_return_t * vmfr)1347 handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1348 {
1349 int ret = -1;
1350
1351 #pragma unused (state)
1352 #pragma unused (vmfr)
1353
1354 return ret;
1355 }
1356
1357
1358
1359
1360 static void
handle_sw_step_debug(arm_saved_state_t * state)1361 handle_sw_step_debug(arm_saved_state_t *state)
1362 {
1363 thread_t thread = current_thread();
1364 exception_type_t exc;
1365 mach_exception_data_type_t codes[2];
1366 mach_msg_type_number_t numcodes = 2;
1367
1368 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1369 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
1370 }
1371
1372 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1373 if (thread->machine.DebugData != NULL) {
1374 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1375 } else {
1376 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
1377 }
1378
1379 mask_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);
1380
1381 // Special encoding for gdb single step event on ARM
1382 exc = EXC_BREAKPOINT;
1383 codes[0] = 1;
1384 codes[1] = 0;
1385
1386 exception_triage(exc, codes, numcodes);
1387 __builtin_unreachable();
1388 }
1389
1390 static void
handle_user_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1391 handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1392 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1393 {
1394 exception_type_t exc = EXC_BAD_ACCESS;
1395 mach_exception_data_type_t codes[2];
1396 mach_msg_type_number_t numcodes = 2;
1397 thread_t thread = current_thread();
1398
1399 (void)esr;
1400 (void)expected_fault_handler;
1401
1402 if (ml_at_interrupt_context()) {
1403 panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state);
1404 }
1405
1406 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1407
1408 if (!is_vm_fault(fault_code) &&
1409 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1410 thread_reset_pcs_done_faulting(thread);
1411 }
1412
1413 if (is_vm_fault(fault_code)) {
1414 vm_map_t map = thread->map;
1415 vm_offset_t vm_fault_addr = fault_addr;
1416 kern_return_t result = KERN_FAILURE;
1417
1418 assert(map != kernel_map);
1419
1420 if (!(fault_type & VM_PROT_EXECUTE)) {
1421 vm_fault_addr = VM_USER_STRIP_TBI(fault_addr);
1422 }
1423
1424 /* check to see if it is just a pmap ref/modify fault */
1425 if (!is_translation_fault(fault_code)) {
1426 result = arm_fast_fault(map->pmap,
1427 vm_fault_addr,
1428 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1429 }
1430 if (result != KERN_SUCCESS) {
1431
1432 {
1433 /* We have to fault the page in */
1434 result = vm_fault(map, vm_fault_addr, fault_type,
1435 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1436 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1437 }
1438 }
1439 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1440 thread_reset_pcs_done_faulting(thread);
1441 }
1442 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1443 return;
1444 }
1445
1446 /*
1447 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1448 * If it does, we're leaking preemption disables somewhere in the kernel.
1449 */
1450 if (__improbable(result == KERN_FAILURE)) {
1451 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1452 }
1453
1454 codes[0] = result;
1455 } else if (is_alignment_fault(fault_code)) {
1456 kern_return_t vmfkr = KERN_SUCCESS;
1457 thread->machine.recover_esr = 0;
1458 thread->machine.recover_far = 0;
1459 int result = handle_alignment_fault_from_user(state, &vmfkr);
1460 if (result == 0) {
1461 /* Successfully emulated, or instruction
1462 * copyin() for decode/emulation failed.
1463 * Continue, or redrive instruction.
1464 */
1465 thread_exception_return();
1466 } else if (((result == EFAULT) || (result == EINVAL)) &&
1467 (thread->machine.recover_esr == 0)) {
1468 /*
1469 * If we didn't actually take a fault, but got one of
1470 * these errors, then we failed basic sanity checks of
1471 * the fault address. Treat this as an invalid
1472 * address.
1473 */
1474 codes[0] = KERN_INVALID_ADDRESS;
1475 } else if ((result == EFAULT) &&
1476 (thread->machine.recover_esr)) {
1477 /*
1478 * Since alignment aborts are prioritized
1479 * ahead of translation aborts, the misaligned
1480 * atomic emulation flow may have triggered a
1481 * VM pagefault, which the VM could not resolve.
1482 * Report the VM fault error in codes[]
1483 */
1484
1485 codes[0] = vmfkr;
1486 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1487 /* Cause ESR_EC to reflect an EL0 abort */
1488 thread->machine.recover_esr &= ~ESR_EC_MASK;
1489 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1490 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr);
1491 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far);
1492 fault_addr = thread->machine.recover_far;
1493 } else {
1494 /* This was just an unsupported alignment
1495 * exception. Misaligned atomic emulation
1496 * timeouts fall in this category.
1497 */
1498 codes[0] = EXC_ARM_DA_ALIGN;
1499 }
1500 } else if (is_parity_error(fault_code)) {
1501 #if defined(APPLE_ARM64_ARCH_FAMILY)
1502 /*
1503 * Platform errors are handled in sleh_sync before interrupts are enabled.
1504 */
1505 #else
1506 panic("User parity error.");
1507 #endif
1508 } else {
1509 codes[0] = KERN_FAILURE;
1510 }
1511
1512 codes[1] = fault_addr;
1513 #if __has_feature(ptrauth_calls)
1514 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1515 if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) {
1516 exc |= EXC_PTRAUTH_BIT;
1517 }
1518 #endif /* __has_feature(ptrauth_calls) */
1519 exception_triage(exc, codes, numcodes);
1520 __builtin_unreachable();
1521 }
1522
1523 static void
handle_kernel_abort_recover(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,thread_t thread)1524 handle_kernel_abort_recover(
1525 arm_saved_state_t *state,
1526 uint32_t esr,
1527 vm_offset_t fault_addr,
1528 thread_t thread)
1529 {
1530 thread->machine.recover_esr = esr;
1531 thread->machine.recover_far = fault_addr;
1532 #if defined(HAS_APPLE_PAC)
1533 MANIPULATE_SIGNED_THREAD_STATE(state,
1534 "mov x1, %[pc] \n"
1535 "str x1, [x0, %[SS64_PC]] \n",
1536 [pc] "r"(copyio_recovery_get_recover_addr(state))
1537 );
1538 #else
1539 saved_state64(state)->pc = copyio_recovery_get_recover_addr(state);
1540 #endif
1541 }
1542
1543 static void
handle_kernel_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1544 handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1545 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1546 {
1547 thread_t thread = current_thread();
1548 bool recover = find_copyio_recovery_entry(state) != 0;
1549
1550 #ifdef CONFIG_KERNEL_TAGGING
1551 /*
1552 * If a read/write access to a tagged address faults over pageable kernel memory
1553 * vm_fault() will need to find the right vm entry and offset. Canonicalize the
1554 * address here so that the correct comparisons can happen later in the VM code.
1555 */
1556 if (!(fault_type & VM_PROT_EXECUTE) && VM_KERNEL_ADDRESS(fault_addr)) {
1557 fault_addr = vm_memtag_canonicalize_address(fault_addr);
1558 }
1559 #endif /* CONFIG_KERNEL_TAGGING */
1560
1561 #ifndef CONFIG_XNUPOST
1562 (void)expected_fault_handler;
1563 #endif /* CONFIG_XNUPOST */
1564
1565 #if CONFIG_DTRACE
1566 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
1567 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1568 /*
1569 * Point to next instruction, or recovery handler if set.
1570 */
1571 if (recover) {
1572 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1573 } else {
1574 add_saved_state_pc(state, 4);
1575 }
1576 return;
1577 } else {
1578 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
1579 }
1580 }
1581 #endif
1582
1583 if (ml_at_interrupt_context()) {
1584 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1585 }
1586
1587 if (is_vm_fault(fault_code)) {
1588 kern_return_t result = KERN_FAILURE;
1589 vm_map_t map;
1590 int interruptible;
1591
1592 /*
1593 * Ensure no faults in the physical aperture. This could happen if
1594 * a page table is incorrectly allocated from the read only region
1595 * when running with KTRR.
1596 */
1597
1598 #ifdef CONFIG_XNUPOST
1599 if (expected_fault_handler && expected_fault_handler(state)) {
1600 return;
1601 }
1602 #endif /* CONFIG_XNUPOST */
1603
1604 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1605 panic_with_thread_kernel_state("Unexpected fault in kernel static region\n", state);
1606 }
1607
1608 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) {
1609 /*
1610 * If no recovery handler is supplied, always drive the fault against
1611 * the kernel map. If the fault was taken against a userspace VA, indicating
1612 * an unprotected access to user address space, vm_fault() should fail and
1613 * ultimately lead to a panic here.
1614 */
1615 map = kernel_map;
1616 interruptible = THREAD_UNINT;
1617 } else {
1618 map = thread->map;
1619
1620 /**
1621 * In the case that the recovery handler is set (e.g., during copyio
1622 * and dtrace probes), we don't want the vm_fault() operation to be
1623 * aborted early. Those code paths can't handle restarting the
1624 * vm_fault() operation so don't allow it to return early without
1625 * creating the wanted mapping.
1626 */
1627 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
1628 }
1629
1630 /* check to see if it is just a pmap ref/modify fault */
1631 if (!is_translation_fault(fault_code)) {
1632 result = arm_fast_fault(map->pmap,
1633 fault_addr,
1634 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
1635 if (result == KERN_SUCCESS) {
1636 return;
1637 }
1638 }
1639
1640 if (result != KERN_PROTECTION_FAILURE) {
1641 /*
1642 * We have to "fault" the page in.
1643 */
1644 result = vm_fault(map, fault_addr, fault_type,
1645 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1646 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1647 }
1648
1649 if (result == KERN_SUCCESS) {
1650 return;
1651 }
1652
1653 /*
1654 * If we have a recover handler, invoke it now.
1655 */
1656 if (recover) {
1657 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1658 return;
1659 }
1660
1661 panic_fault_address = fault_addr;
1662 } else if (is_alignment_fault(fault_code)) {
1663 if (recover) {
1664 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1665 return;
1666 }
1667 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
1668 } else if (is_parity_error(fault_code)) {
1669 #if defined(APPLE_ARM64_ARCH_FAMILY)
1670 /*
1671 * Platform errors are handled in sleh_sync before interrupts are enabled.
1672 */
1673 #else
1674 panic_with_thread_kernel_state("Kernel parity error.", state);
1675 #endif
1676 } else {
1677 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1678 }
1679
1680 panic_with_thread_kernel_state("Kernel data abort.", state);
1681 }
1682
1683 extern void syscall_trace(struct arm_saved_state * regs);
1684
1685 static void
handle_svc(arm_saved_state_t * state)1686 handle_svc(arm_saved_state_t *state)
1687 {
1688 int trap_no = get_saved_state_svc_number(state);
1689 thread_t thread = current_thread();
1690 struct proc *p;
1691
1692 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1693
1694 #define TRACE_SYSCALL 1
1695 #if TRACE_SYSCALL
1696 syscall_trace(state);
1697 #endif
1698
1699 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1700
1701 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1702 platform_syscall(state);
1703 panic("Returned from platform_syscall()?");
1704 }
1705
1706 mach_kauth_cred_thread_update();
1707
1708 if (trap_no < 0) {
1709 switch (trap_no) {
1710 case MACH_ARM_TRAP_ABSTIME:
1711 handle_mach_absolute_time_trap(state);
1712 return;
1713 case MACH_ARM_TRAP_CONTTIME:
1714 handle_mach_continuous_time_trap(state);
1715 return;
1716 }
1717
1718 /* Counting perhaps better in the handler, but this is how it's been done */
1719 thread->syscalls_mach++;
1720 mach_syscall(state);
1721 } else {
1722 /* Counting perhaps better in the handler, but this is how it's been done */
1723 thread->syscalls_unix++;
1724 p = get_bsdthreadtask_info(thread);
1725
1726 assert(p);
1727
1728 unix_syscall(state, thread, p);
1729 }
1730 }
1731
1732 static void
handle_mach_absolute_time_trap(arm_saved_state_t * state)1733 handle_mach_absolute_time_trap(arm_saved_state_t *state)
1734 {
1735 uint64_t now = mach_absolute_time();
1736 saved_state64(state)->x[0] = now;
1737 }
1738
1739 static void
handle_mach_continuous_time_trap(arm_saved_state_t * state)1740 handle_mach_continuous_time_trap(arm_saved_state_t *state)
1741 {
1742 uint64_t now = mach_continuous_time();
1743 saved_state64(state)->x[0] = now;
1744 }
1745
1746
1747 __attribute__((noreturn))
1748 static void
handle_msr_trap(arm_saved_state_t * state,uint32_t esr)1749 handle_msr_trap(arm_saved_state_t *state, uint32_t esr)
1750 {
1751 exception_type_t exception = EXC_BAD_INSTRUCTION;
1752 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1753 mach_msg_type_number_t numcodes = 2;
1754 uint32_t instr = 0;
1755
1756 if (!is_saved_state64(state)) {
1757 panic("MSR/MRS trap (ESR 0x%x) from 32-bit state", esr);
1758 }
1759
1760 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1761 panic("MSR/MRS trap (ESR 0x%x) from kernel", esr);
1762 }
1763
1764 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1765 codes[1] = instr;
1766
1767 exception_triage(exception, codes, numcodes);
1768 __builtin_unreachable();
1769 }
1770
1771 #if __has_feature(ptrauth_calls)
1772 static void
stringify_gpr(unsigned int r,char reg[4])1773 stringify_gpr(unsigned int r, char reg[4])
1774 {
1775 switch (r) {
1776 case 29:
1777 strncpy(reg, "fp", 4);
1778 return;
1779
1780 case 30:
1781 strncpy(reg, "lr", 4);
1782 return;
1783
1784 case 31:
1785 strncpy(reg, "xzr", 4);
1786 return;
1787
1788 default:
1789 snprintf(reg, 4, "x%u", r);
1790 return;
1791 }
1792 }
1793
1794 static void
autxx_instruction_extract_reg(uint32_t instr,char reg[4])1795 autxx_instruction_extract_reg(uint32_t instr, char reg[4])
1796 {
1797 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
1798 stringify_gpr(rd, reg);
1799 }
1800
1801 static const char *
autix_system_instruction_extract_reg(uint32_t instr)1802 autix_system_instruction_extract_reg(uint32_t instr)
1803 {
1804 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
1805 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
1806 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
1807 return "x17";
1808 } else {
1809 return "lr";
1810 }
1811 }
1812
1813
1814 static void
handle_pac_fail(arm_saved_state_t * state,uint32_t esr)1815 handle_pac_fail(arm_saved_state_t *state, uint32_t esr)
1816 {
1817 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
1818 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
1819 mach_msg_type_number_t numcodes = 2;
1820 uint32_t instr = 0;
1821
1822 if (!is_saved_state64(state)) {
1823 panic("PAC failure (ESR 0x%x) from 32-bit state", esr);
1824 }
1825
1826 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1827
1828 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1829 #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
1830 #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
1831 #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
1832 #define MAX_PAC_MSG_FMT AUTXX_MSG_FMT
1833
1834 char msg[strlen(MAX_PAC_MSG_FMT)
1835 - strlen("%s") + strlen("IA")
1836 - strlen("%s") + strlen("xzr")
1837 + 1];
1838 ptrauth_key key = (ptrauth_key)(esr & 0x3);
1839 const char *key_str = ptrauth_key_to_string(key);
1840
1841 if (ARM64_INSTR_IS_AUTxx(instr)) {
1842 char reg[4];
1843 autxx_instruction_extract_reg(instr, reg);
1844 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1845 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
1846 const char *reg = autix_system_instruction_extract_reg(instr);
1847 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1848 } else {
1849 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
1850 }
1851 panic_with_thread_kernel_state(msg, state);
1852 }
1853
1854 codes[1] = instr;
1855
1856 exception_triage(exception, codes, numcodes);
1857 __builtin_unreachable();
1858 }
1859 #endif /* __has_feature(ptrauth_calls) */
1860
1861 static void
handle_user_trapped_instruction32(arm_saved_state_t * state,uint32_t esr)1862 handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
1863 {
1864 exception_type_t exception = EXC_BAD_INSTRUCTION;
1865 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1866 mach_msg_type_number_t numcodes = 2;
1867 uint32_t instr;
1868
1869 if (is_saved_state64(state)) {
1870 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
1871 }
1872
1873 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1874 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
1875 }
1876
1877 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1878 codes[1] = instr;
1879
1880 exception_triage(exception, codes, numcodes);
1881 __builtin_unreachable();
1882 }
1883
1884 static void
handle_simd_trap(arm_saved_state_t * state,uint32_t esr)1885 handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
1886 {
1887 exception_type_t exception = EXC_BAD_INSTRUCTION;
1888 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1889 mach_msg_type_number_t numcodes = 2;
1890 uint32_t instr = 0;
1891
1892 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1893 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
1894 }
1895
1896 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1897 codes[1] = instr;
1898
1899 exception_triage(exception, codes, numcodes);
1900 __builtin_unreachable();
1901 }
1902
1903 void
sleh_irq(arm_saved_state_t * state)1904 sleh_irq(arm_saved_state_t *state)
1905 {
1906 cpu_data_t * cdp __unused = getCpuDatap();
1907 #if MACH_ASSERT
1908 int preemption_level = sleh_get_preemption_level();
1909 #endif
1910
1911
1912 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
1913
1914 #if USE_APPLEARMSMP
1915 PE_handle_ext_interrupt();
1916 #else
1917 /* Run the registered interrupt handler. */
1918 cdp->interrupt_handler(cdp->interrupt_target,
1919 cdp->interrupt_refCon,
1920 cdp->interrupt_nub,
1921 cdp->interrupt_source);
1922 #endif
1923
1924 entropy_collect();
1925
1926
1927 sleh_interrupt_handler_epilogue();
1928 #if MACH_ASSERT
1929 if (preemption_level != sleh_get_preemption_level()) {
1930 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
1931 }
1932 #endif
1933 }
1934
1935 void
sleh_fiq(arm_saved_state_t * state)1936 sleh_fiq(arm_saved_state_t *state)
1937 {
1938 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
1939 #if MACH_ASSERT
1940 int preemption_level = sleh_get_preemption_level();
1941 #endif
1942
1943 #if MONOTONIC_FIQ
1944 uint64_t pmcr0 = 0, upmsr = 0;
1945 #endif /* MONOTONIC_FIQ */
1946
1947 #if defined(HAS_IPI)
1948 boolean_t is_ipi = FALSE;
1949 uint64_t ipi_sr = 0;
1950
1951 if (gFastIPI) {
1952 MRS(ipi_sr, "S3_5_C15_C1_1");
1953
1954 if (ipi_sr & ARM64_IPISR_IPI_PENDING) {
1955 is_ipi = TRUE;
1956 }
1957 }
1958
1959 if (is_ipi) {
1960 type = DBG_INTR_TYPE_IPI;
1961 } else
1962 #endif /* defined(HAS_IPI) */
1963 if (ml_get_timer_pending()) {
1964 type = DBG_INTR_TYPE_TIMER;
1965 }
1966 #if MONOTONIC_FIQ
1967 /* Consult the PMI sysregs last, after IPI/timer
1968 * classification.
1969 */
1970 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
1971 type = DBG_INTR_TYPE_PMI;
1972 }
1973 #endif /* MONOTONIC_FIQ */
1974
1975 sleh_interrupt_handler_prologue(state, type);
1976
1977 #if APPLEVIRTUALPLATFORM
1978 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
1979 #endif
1980
1981 #if defined(HAS_IPI)
1982 if (type == DBG_INTR_TYPE_IPI) {
1983 /*
1984 * Order is important here: we must ack the IPI by writing IPI_SR
1985 * before we call cpu_signal_handler(). Otherwise, there will be
1986 * a window between the completion of pending-signal processing in
1987 * cpu_signal_handler() and the ack during which a newly-issued
1988 * IPI to this CPU may be lost. ISB is required to ensure the msr
1989 * is retired before execution of cpu_signal_handler().
1990 */
1991 MSR("S3_5_C15_C1_1", ARM64_IPISR_IPI_PENDING);
1992 __builtin_arm_isb(ISB_SY);
1993 cpu_signal_handler();
1994 } else
1995 #endif /* defined(HAS_IPI) */
1996 #if MONOTONIC_FIQ
1997 if (type == DBG_INTR_TYPE_PMI) {
1998 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
1999 mt_fiq(getCpuDatap(), pmcr0, upmsr);
2000 INTERRUPT_MASKED_DEBUG_END();
2001 } else
2002 #endif /* MONOTONIC_FIQ */
2003 {
2004 /*
2005 * We don't know that this is a timer, but we don't have insight into
2006 * the other interrupts that go down this path.
2007 */
2008
2009 cpu_data_t *cdp = getCpuDatap();
2010
2011 cdp->cpu_decrementer = -1; /* Large */
2012
2013 /*
2014 * ARM64_TODO: whether we're coming from userland is ignored right now.
2015 * We can easily thread it through, but not bothering for the
2016 * moment (AArch32 doesn't either).
2017 */
2018 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
2019 rtclock_intr(TRUE);
2020 INTERRUPT_MASKED_DEBUG_END();
2021 }
2022
2023 #if APPLEVIRTUALPLATFORM
2024 if (iar != GIC_SPURIOUS_IRQ) {
2025 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
2026 __builtin_arm_isb(ISB_SY);
2027 }
2028 #endif
2029
2030 sleh_interrupt_handler_epilogue();
2031 #if MACH_ASSERT
2032 if (preemption_level != sleh_get_preemption_level()) {
2033 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
2034 }
2035 #endif
2036 }
2037
2038 void
sleh_serror(arm_context_t * context,uint32_t esr,vm_offset_t far)2039 sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
2040 {
2041 task_vtimer_check(current_thread());
2042
2043 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
2044 esr, VM_KERNEL_ADDRHIDE(far));
2045 arm_saved_state_t *state = &context->ss;
2046 #if MACH_ASSERT
2047 int preemption_level = sleh_get_preemption_level();
2048 #endif
2049
2050
2051 ASSERT_CONTEXT_SANITY(context);
2052 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC);
2053 #if MACH_ASSERT
2054 if (preemption_level != sleh_get_preemption_level()) {
2055 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
2056 }
2057 #endif
2058 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
2059 esr, VM_KERNEL_ADDRHIDE(far));
2060 }
2061
2062 void
mach_syscall_trace_exit(unsigned int retval,unsigned int call_number)2063 mach_syscall_trace_exit(unsigned int retval,
2064 unsigned int call_number)
2065 {
2066 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2067 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
2068 DBG_FUNC_END, retval, 0, 0, 0, 0);
2069 }
2070
2071 __attribute__((noreturn))
2072 void
thread_syscall_return(kern_return_t error)2073 thread_syscall_return(kern_return_t error)
2074 {
2075 thread_t thread;
2076 struct arm_saved_state *state;
2077
2078 thread = current_thread();
2079 state = get_user_regs(thread);
2080
2081 assert(is_saved_state64(state));
2082 saved_state64(state)->x[0] = error;
2083
2084 #if MACH_ASSERT
2085 kern_allocation_name_t
2086 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
2087 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2088 #endif /* MACH_ASSERT */
2089
2090 if (kdebug_enable) {
2091 /* Invert syscall number (negative for a mach syscall) */
2092 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
2093 }
2094
2095 thread_exception_return();
2096 }
2097
2098 void
syscall_trace(struct arm_saved_state * regs __unused)2099 syscall_trace(
2100 struct arm_saved_state * regs __unused)
2101 {
2102 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2103 }
2104
2105 static void
sleh_interrupt_handler_prologue(arm_saved_state_t * state,unsigned int type)2106 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2107 {
2108 boolean_t is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2109
2110 task_vtimer_check(current_thread());
2111
2112 uint64_t pc = is_user ? get_saved_state_pc(state) :
2113 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2114
2115 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2116 0, pc, is_user, type);
2117
2118 #if CONFIG_TELEMETRY
2119 if (telemetry_needs_record) {
2120 telemetry_mark_curthread(is_user, FALSE);
2121 }
2122 #endif /* CONFIG_TELEMETRY */
2123 }
2124
2125 static void
sleh_interrupt_handler_epilogue(void)2126 sleh_interrupt_handler_epilogue(void)
2127 {
2128 #if KPERF
2129 kperf_interrupt();
2130 #endif /* KPERF */
2131 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2132 }
2133
2134 void
sleh_invalid_stack(arm_context_t * context,uint32_t esr __unused,vm_offset_t far __unused)2135 sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
2136 {
2137 thread_t thread = current_thread();
2138 vm_offset_t kernel_stack_bottom, sp;
2139
2140 sp = get_saved_state_sp(&context->ss);
2141 vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr;
2142 kernel_stack_bottom = round_page(kstackptr) - KERNEL_STACK_SIZE;
2143
2144 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2145 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
2146 }
2147
2148 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
2149 }
2150
2151
2152 #if DEVELOPMENT || DEBUG
2153 static int trap_handled;
2154 static void
handle_recoverable_kernel_trap(__unused void * tstate,uint16_t comment)2155 handle_recoverable_kernel_trap(
2156 __unused void *tstate,
2157 uint16_t comment)
2158 {
2159 assert(comment == TEST_RECOVERABLE_SOFT_TRAP);
2160
2161 printf("Recoverable trap handled.\n");
2162 trap_handled = 1;
2163 }
2164
2165 KERNEL_BRK_DESCRIPTOR_DEFINE(test_desc,
2166 .type = KERNEL_BRK_TYPE_TEST,
2167 .base = TEST_RECOVERABLE_SOFT_TRAP,
2168 .max = TEST_RECOVERABLE_SOFT_TRAP,
2169 .options = KERNEL_BRK_RECOVERABLE,
2170 .handle_breakpoint = handle_recoverable_kernel_trap);
2171
2172 static int
recoverable_kernel_trap_test(__unused int64_t in,int64_t * out)2173 recoverable_kernel_trap_test(__unused int64_t in, int64_t *out)
2174 {
2175 ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP);
2176
2177 *out = trap_handled;
2178 return 0;
2179 }
2180
2181 SYSCTL_TEST_REGISTER(recoverable_kernel_trap, recoverable_kernel_trap_test);
2182
2183 #endif
2184