1 /*
2 * Copyright (c) 2012-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39 #include <arm64/instructions.h>
40
41 #include <kern/debug.h>
42 #include <kern/restartable.h>
43 #include <kern/socd_client.h>
44 #include <kern/thread.h>
45 #include <kern/zalloc_internal.h>
46 #include <mach/exception.h>
47 #include <mach/arm/traps.h>
48 #include <mach/vm_types.h>
49 #include <mach/machine/thread_status.h>
50
51 #include <machine/atomic.h>
52 #include <machine/limits.h>
53
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/arm64/apple_arm64_regs.h>
56 #include <pexpert/arm64/board_config.h>
57
58 #include <vm/vm_page.h>
59 #include <vm/pmap.h>
60 #include <vm/vm_fault.h>
61 #include <vm/vm_kern.h>
62
63 #include <sys/errno.h>
64 #include <sys/kdebug.h>
65 #include <kperf/kperf.h>
66
67 #include <kern/policy_internal.h>
68 #if CONFIG_TELEMETRY
69 #include <kern/telemetry.h>
70 #endif
71
72 #include <prng/entropy.h>
73
74
75
76
77 #include <arm64/platform_error_handler.h>
78
79 #if KASAN_TBI
80 #include <san/kasan.h>
81 #endif /* KASAN_TBI */
82
83 #if CONFIG_UBSAN_MINIMAL
84 #include <san/ubsan_minimal.h>
85 #endif
86
87
88 #ifndef __arm64__
89 #error Should only be compiling for arm64.
90 #endif
91
92 #if DEBUG || DEVELOPMENT
93 #define HAS_TELEMETRY_KERNEL_BRK 1
94 #endif
95
96
97 #define TEST_CONTEXT32_SANITY(context) \
98 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
99 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
100
101 #define TEST_CONTEXT64_SANITY(context) \
102 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
103 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
104
105 #define ASSERT_CONTEXT_SANITY(context) \
106 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
107
108
109 #define COPYIN(src, dst, size) \
110 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
111 copyin_kern(src, dst, size) : \
112 copyin(src, dst, size)
113
114 #define COPYOUT(src, dst, size) \
115 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
116 copyout_kern(src, dst, size) : \
117 copyout(src, dst, size)
118
119 // Below is for concatenating a string param to a string literal
120 #define STR1(x) #x
121 #define STR(x) STR1(x)
122
123 #define ARM64_KDBG_CODE_KERNEL (0 << 8)
124 #define ARM64_KDBG_CODE_USER (1 << 8)
125 #define ARM64_KDBG_CODE_GUEST (2 << 8)
126
127 _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
128 _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
129
130 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
131
132 void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike;
133 void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t, bool);
134
135
136
137 void sleh_irq(arm_saved_state_t *);
138 void sleh_fiq(arm_saved_state_t *);
139 void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
140 void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2;
141
142 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
143 static void sleh_interrupt_handler_epilogue(void);
144
145 static void handle_svc(arm_saved_state_t *);
146 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
147 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
148
149 static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr);
150 #if __has_feature(ptrauth_calls)
151 static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2;
152 #endif
153
154 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
155
156 static void handle_uncategorized(arm_saved_state_t *);
157
158 static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t);
159
160 static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2;
161
162 typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
163 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
164 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
165
166 static int is_vm_fault(fault_status_t);
167 static int is_translation_fault(fault_status_t);
168 static int is_alignment_fault(fault_status_t);
169
170 typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
171 static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
172 static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
173
174 static void handle_pc_align(arm_saved_state_t *ss) __dead2;
175 static void handle_sp_align(arm_saved_state_t *ss) __dead2;
176 static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
177 static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
178 static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2;
179
180 static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
181
182 static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
183
184 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2;
185
186 static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2;
187
188 extern void mach_kauth_cred_thread_update(void);
189 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
190
191 struct proc;
192
193 typedef uint32_t arm64_instr_t;
194
195 extern void
196 unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
197
198 extern void
199 mach_syscall(struct arm_saved_state*);
200
201 #if CONFIG_DTRACE
202 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
203 extern boolean_t dtrace_tally_fault(user_addr_t);
204
205 /*
206 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
207 * and paste the trap instructions
208 * over from that file. Need to keep these in sync!
209 */
210 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
211 #define FASTTRAP_THUMB32_INSTR 0xdefc
212 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
213
214 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
215 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
216 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
217
218 /* See <rdar://problem/4613924> */
219 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
220 #endif
221
222
223
224 extern void arm64_thread_exception_return(void) __dead2;
225
226 #if defined(APPLETYPHOON)
227 #define CPU_NAME "Typhoon"
228 #elif defined(APPLETWISTER)
229 #define CPU_NAME "Twister"
230 #elif defined(APPLEHURRICANE)
231 #define CPU_NAME "Hurricane"
232 #elif defined(APPLELIGHTNING)
233 #define CPU_NAME "Lightning"
234 #else
235 #define CPU_NAME "Unknown"
236 #endif
237
238 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
239 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
240 #define ESR_WT_REASON(esr) ((esr) & 0xff)
241
242 #define WT_REASON_NONE 0
243 #define WT_REASON_INTEGRITY_FAIL 1
244 #define WT_REASON_BAD_SYSCALL 2
245 #define WT_REASON_NOT_LOCKED 3
246 #define WT_REASON_ALREADY_LOCKED 4
247 #define WT_REASON_SW_REQ 5
248 #define WT_REASON_PT_INVALID 6
249 #define WT_REASON_PT_VIOLATION 7
250 #define WT_REASON_REG_VIOLATION 8
251 #endif
252
253 #if defined(HAS_IPI)
254 void cpu_signal_handler(void);
255 extern unsigned int gFastIPI;
256 #endif /* defined(HAS_IPI) */
257
258 static arm_saved_state64_t *original_faulting_state = NULL;
259
260
261 TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
262
263 extern vm_offset_t static_memory_end;
264
265 /*
266 * Fault copyio_recovery_entry in copyin/copyout routines.
267 *
268 * Offets are expressed in bytes from ©_recovery_table
269 */
270 struct copyio_recovery_entry {
271 ptrdiff_t cre_start;
272 ptrdiff_t cre_end;
273 ptrdiff_t cre_recovery;
274 };
275
276 extern struct copyio_recovery_entry copyio_recover_table[];
277 extern struct copyio_recovery_entry copyio_recover_table_end[];
278
279 static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)280 copyio_recovery_offset(uintptr_t addr)
281 {
282 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
283 }
284
285 #if !HAS_APPLE_PAC
286 static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)287 copyio_recovery_addr(ptrdiff_t offset)
288 {
289 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
290 }
291 #endif
292
293 static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(arm_saved_state_t * state)294 find_copyio_recovery_entry(arm_saved_state_t *state)
295 {
296 ptrdiff_t offset = copyio_recovery_offset(get_saved_state_pc(state));
297 struct copyio_recovery_entry *e;
298
299 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
300 if (offset >= e->cre_start && offset < e->cre_end) {
301 return e;
302 }
303 }
304
305 return NULL;
306 }
307
308 static inline int
is_vm_fault(fault_status_t status)309 is_vm_fault(fault_status_t status)
310 {
311 switch (status) {
312 case FSC_TRANSLATION_FAULT_L0:
313 case FSC_TRANSLATION_FAULT_L1:
314 case FSC_TRANSLATION_FAULT_L2:
315 case FSC_TRANSLATION_FAULT_L3:
316 case FSC_ACCESS_FLAG_FAULT_L1:
317 case FSC_ACCESS_FLAG_FAULT_L2:
318 case FSC_ACCESS_FLAG_FAULT_L3:
319 case FSC_PERMISSION_FAULT_L1:
320 case FSC_PERMISSION_FAULT_L2:
321 case FSC_PERMISSION_FAULT_L3:
322 return TRUE;
323 default:
324 return FALSE;
325 }
326 }
327
328 static inline int
is_translation_fault(fault_status_t status)329 is_translation_fault(fault_status_t status)
330 {
331 switch (status) {
332 case FSC_TRANSLATION_FAULT_L0:
333 case FSC_TRANSLATION_FAULT_L1:
334 case FSC_TRANSLATION_FAULT_L2:
335 case FSC_TRANSLATION_FAULT_L3:
336 return TRUE;
337 default:
338 return FALSE;
339 }
340 }
341
342 static inline int
is_permission_fault(fault_status_t status)343 is_permission_fault(fault_status_t status)
344 {
345 switch (status) {
346 case FSC_PERMISSION_FAULT_L1:
347 case FSC_PERMISSION_FAULT_L2:
348 case FSC_PERMISSION_FAULT_L3:
349 return TRUE;
350 default:
351 return FALSE;
352 }
353 }
354
355 static inline int
is_alignment_fault(fault_status_t status)356 is_alignment_fault(fault_status_t status)
357 {
358 return status == FSC_ALIGNMENT_FAULT;
359 }
360
361 static inline int
is_parity_error(fault_status_t status)362 is_parity_error(fault_status_t status)
363 {
364 switch (status) {
365 /*
366 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
367 * only applicable to AArch32 HSR register. Can this be removed?
368 */
369 case FSC_ASYNC_PARITY:
370 case FSC_SYNC_PARITY:
371 case FSC_SYNC_PARITY_TT_L1:
372 case FSC_SYNC_PARITY_TT_L2:
373 case FSC_SYNC_PARITY_TT_L3:
374 return TRUE;
375 default:
376 return FALSE;
377 }
378 }
379
380
381
382 __dead2 __unused
383 static void
arm64_implementation_specific_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far)384 arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
385 {
386 #pragma unused (state, esr, far)
387 panic_plain("Unhandled implementation specific error\n");
388 }
389
390 #if CONFIG_KERNEL_INTEGRITY
391 #pragma clang diagnostic push
392 #pragma clang diagnostic ignored "-Wunused-parameter"
393 static void
kernel_integrity_error_handler(uint32_t esr,vm_offset_t far)394 kernel_integrity_error_handler(uint32_t esr, vm_offset_t far)
395 {
396 #if defined(KERNEL_INTEGRITY_WT)
397 #if (DEVELOPMENT || DEBUG)
398 if (ESR_WT_SERROR(esr)) {
399 switch (ESR_WT_REASON(esr)) {
400 case WT_REASON_INTEGRITY_FAIL:
401 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
402 case WT_REASON_BAD_SYSCALL:
403 panic_plain("Kernel integrity, bad syscall.");
404 case WT_REASON_NOT_LOCKED:
405 panic_plain("Kernel integrity, not locked.");
406 case WT_REASON_ALREADY_LOCKED:
407 panic_plain("Kernel integrity, already locked.");
408 case WT_REASON_SW_REQ:
409 panic_plain("Kernel integrity, software request.");
410 case WT_REASON_PT_INVALID:
411 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
412 "walking 0x%016lx.", far);
413 case WT_REASON_PT_VIOLATION:
414 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
415 far);
416 case WT_REASON_REG_VIOLATION:
417 panic_plain("Kernel integrity, violation in system register %d.",
418 (unsigned) far);
419 default:
420 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
421 }
422 }
423 #else
424 if (ESR_WT_SERROR(esr)) {
425 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
426 }
427 #endif
428 #endif
429 }
430 #pragma clang diagnostic pop
431 #endif
432
433 static void
arm64_platform_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far,platform_error_source_t source)434 arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far, platform_error_source_t source)
435 {
436 #if CONFIG_KERNEL_INTEGRITY
437 kernel_integrity_error_handler(esr, far);
438 #endif
439
440 (void)source;
441 cpu_data_t *cdp = getCpuDatap();
442
443 if (PE_handle_platform_error(far)) {
444 return;
445 } else if (cdp->platform_error_handler != NULL) {
446 cdp->platform_error_handler(cdp->cpu_id, far);
447 } else {
448 arm64_implementation_specific_error(state, esr, far);
449 }
450 }
451
452 void
panic_with_thread_kernel_state(const char * msg,arm_saved_state_t * ss)453 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
454 {
455 boolean_t ss_valid;
456
457 ss_valid = is_saved_state64(ss);
458 arm_saved_state64_t *state = saved_state64(ss);
459
460 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
461
462 // rdar://80659177
463 // Read SoCD tracepoints up to twice — once the first time we call panic and
464 // another time if we encounter a nested panic after that.
465 static int twice = 2;
466 if (twice > 0) {
467 twice--;
468 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc),
469 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
470 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
471 VALUE(state->far));
472 }
473
474
475 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
476 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
477 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
478 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
479 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
480 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
481 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
482 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
483 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
484 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
485 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
486 state->x[0], state->x[1], state->x[2], state->x[3],
487 state->x[4], state->x[5], state->x[6], state->x[7],
488 state->x[8], state->x[9], state->x[10], state->x[11],
489 state->x[12], state->x[13], state->x[14], state->x[15],
490 state->x[16], state->x[17], state->x[18], state->x[19],
491 state->x[20], state->x[21], state->x[22], state->x[23],
492 state->x[24], state->x[25], state->x[26], state->x[27],
493 state->x[28], state->fp, state->lr, state->sp,
494 state->pc, state->cpsr, state->esr, state->far);
495 }
496
497 void
sleh_synchronous_sp1(arm_context_t * context,uint32_t esr,vm_offset_t far __unused)498 sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
499 {
500 esr_exception_class_t class = ESR_EC(esr);
501 arm_saved_state_t * state = &context->ss;
502
503 switch (class) {
504 case ESR_EC_UNCATEGORIZED:
505 {
506 #if (DEVELOPMENT || DEBUG)
507 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
508 if (IS_ARM_GDB_TRAP(instr)) {
509 DebuggerCall(EXC_BREAKPOINT, state);
510 }
511 OS_FALLTHROUGH; // panic if we return from the debugger
512 #else
513 panic_with_thread_kernel_state("Unexpected debugger trap while SP1 selected", state);
514 #endif /* (DEVELOPMENT || DEBUG) */
515 }
516 default:
517 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
518 }
519 }
520
521
522 __attribute__((noreturn))
523 void
thread_exception_return()524 thread_exception_return()
525 {
526 thread_t thread = current_thread();
527 if (thread->machine.exception_trace_code != 0) {
528 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
529 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
530 thread->machine.exception_trace_code = 0;
531 }
532
533 #if KASAN_TBI
534 kasan_unpoison_curstack(true);
535 #endif /* KASAN_TBI */
536 arm64_thread_exception_return();
537 __builtin_unreachable();
538 }
539
540 /*
541 * check whether task vtimers are running and set thread and CPU BSD AST
542 *
543 * must be called with interrupts masked so updates of fields are atomic
544 * must be emitted inline to avoid generating an FBT probe on the exception path
545 *
546 */
547 __attribute__((__always_inline__))
548 static inline void
task_vtimer_check(thread_t thread)549 task_vtimer_check(thread_t thread)
550 {
551 task_t task = get_threadtask_early(thread);
552
553 if (__improbable(task != NULL && task->vtimers)) {
554 thread_ast_set(thread, AST_BSD);
555 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
556 }
557 }
558
559 #if MACH_ASSERT
560 /**
561 * A version of get_preemption_level() that works in early boot.
562 *
563 * If an exception is raised in early boot before the initial thread has been
564 * set up, then calling get_preemption_level() in the SLEH will trigger an
565 * infinitely-recursing exception. This function handles this edge case.
566 */
567 static inline int
sleh_get_preemption_level(void)568 sleh_get_preemption_level(void)
569 {
570 if (__improbable(current_thread() == NULL)) {
571 return 0;
572 }
573 return get_preemption_level();
574 }
575 #endif // MACH_ASSERT
576
577 static inline bool
is_platform_error(uint32_t esr)578 is_platform_error(uint32_t esr)
579 {
580 esr_exception_class_t class = ESR_EC(esr);
581 uint32_t iss = ESR_ISS(esr);
582 fault_status_t fault_code;
583
584 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
585 fault_code = ISS_DA_FSC(iss);
586 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
587 fault_code = ISS_IA_FSC(iss);
588 } else {
589 return false;
590 }
591
592 return fault_code == FSC_SYNC_PARITY;
593 }
594
595 void
sleh_synchronous(arm_context_t * context,uint32_t esr,vm_offset_t far,__unused bool did_initiate_panic_lockdown)596 sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far, __unused bool did_initiate_panic_lockdown)
597 {
598 esr_exception_class_t class = ESR_EC(esr);
599 arm_saved_state_t * state = &context->ss;
600 thread_t thread = current_thread();
601 #if MACH_ASSERT
602 int preemption_level = sleh_get_preemption_level();
603 #endif
604 expected_fault_handler_t expected_fault_handler = NULL;
605 #ifdef CONFIG_XNUPOST
606 expected_fault_handler_t saved_expected_fault_handler = NULL;
607 uintptr_t saved_expected_fault_addr = 0;
608 uintptr_t saved_expected_fault_pc = 0;
609 #endif /* CONFIG_XNUPOST */
610
611 ASSERT_CONTEXT_SANITY(context);
612
613 task_vtimer_check(thread);
614
615 #if CONFIG_DTRACE
616 /*
617 * Handle kernel DTrace probes as early as possible to minimize the likelihood
618 * that this path will itself trigger a DTrace probe, which would lead to infinite
619 * probe recursion.
620 */
621 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
622 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
623 return;
624 }
625 #endif
626 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
627
628
629 /*
630 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
631 * that would disclose the behavior of PT_DENY_ATTACH processes.
632 */
633 if (is_user) {
634 /* Sanitize FAR (but only if the exception was taken from userspace) */
635 switch (class) {
636 case ESR_EC_IABORT_EL1:
637 case ESR_EC_IABORT_EL0:
638 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */
639 if (ISS_IA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) {
640 saved_state64(state)->far = 0;
641 }
642 break;
643 case ESR_EC_DABORT_EL1:
644 case ESR_EC_DABORT_EL0:
645 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */
646 if (ISS_DA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) {
647 saved_state64(state)->far = 0;
648 }
649 break;
650 case ESR_EC_WATCHPT_MATCH_EL1:
651 case ESR_EC_WATCHPT_MATCH_EL0:
652 case ESR_EC_PC_ALIGN:
653 break; /* FAR_ELx is valid */
654 default:
655 saved_state64(state)->far = 0;
656 break;
657 }
658
659 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
660 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
661 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
662 esr, far, get_saved_state_pc(state), 0, 0);
663 } else {
664 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
665 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
666 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
667 }
668
669 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
670 /*
671 * We no longer support 32-bit, which means no 2-byte
672 * instructions.
673 */
674 if (is_user) {
675 panic("Exception on 2-byte instruction, "
676 "context=%p, esr=%#x, far=%p",
677 context, esr, (void *)far);
678 } else {
679 panic_with_thread_kernel_state("Exception on 2-byte instruction", state);
680 }
681 }
682
683 #ifdef CONFIG_XNUPOST
684 if (thread->machine.expected_fault_handler != NULL) {
685 saved_expected_fault_handler = thread->machine.expected_fault_handler;
686 saved_expected_fault_addr = thread->machine.expected_fault_addr;
687 saved_expected_fault_pc = thread->machine.expected_fault_pc;
688
689 thread->machine.expected_fault_handler = NULL;
690 thread->machine.expected_fault_addr = 0;
691 thread->machine.expected_fault_pc = 0;
692
693 if (saved_expected_fault_addr == far ||
694 saved_expected_fault_pc == get_saved_state_pc(state)) {
695 expected_fault_handler = saved_expected_fault_handler;
696 }
697 }
698 #endif /* CONFIG_XNUPOST */
699
700 if (__improbable(is_platform_error(esr))) {
701 /*
702 * Must gather error info in platform error handler before
703 * thread is preempted to another core/cluster to guarantee
704 * accurate error details
705 */
706
707 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_SYNC);
708 return;
709 }
710
711 if (is_user && class == ESR_EC_DABORT_EL0) {
712 thread_reset_pcs_will_fault(thread);
713 }
714
715 /* Inherit the interrupt masks from previous context */
716 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
717 ml_set_interrupts_enabled(TRUE);
718 }
719
720 switch (class) {
721 case ESR_EC_SVC_64:
722 if (!is_saved_state64(state) || !is_user) {
723 panic("Invalid SVC_64 context");
724 }
725
726 handle_svc(state);
727 break;
728
729 case ESR_EC_DABORT_EL0:
730 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
731 break;
732
733 case ESR_EC_MSR_TRAP:
734 handle_msr_trap(state, esr);
735 break;
736 /**
737 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for
738 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host
739 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC
740 * handler here needs to be built into the guest kernels to prevent the exception to fall
741 * through.
742 */
743 #if __has_feature(ptrauth_calls)
744 case ESR_EC_PAC_FAIL:
745 #ifdef CONFIG_XNUPOST
746 if (expected_fault_handler != NULL && expected_fault_handler(state)) {
747 break;
748 }
749 #endif /* CONFIG_XNUPOST */
750 handle_pac_fail(state, esr);
751 __builtin_unreachable();
752
753 #endif /* __has_feature(ptrauth_calls) */
754
755
756 case ESR_EC_IABORT_EL0:
757 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
758 break;
759
760 case ESR_EC_IABORT_EL1:
761 #ifdef CONFIG_XNUPOST
762 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
763 break;
764 }
765 #endif /* CONFIG_XNUPOST */
766
767 panic_with_thread_kernel_state("Kernel instruction fetch abort", state);
768
769 case ESR_EC_PC_ALIGN:
770 handle_pc_align(state);
771 __builtin_unreachable();
772
773 case ESR_EC_DABORT_EL1:
774 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
775 break;
776
777 case ESR_EC_UNCATEGORIZED:
778 assert(!ESR_ISS(esr));
779
780 #if CONFIG_XNUPOST
781 if (!is_user && (expected_fault_handler != NULL) && expected_fault_handler(state)) {
782 /*
783 * The fault handler accepted the exception and handled it on its
784 * own. Don't trap to the debugger/panic.
785 */
786 break;
787 }
788 #endif /* CONFIG_XNUPOST */
789 handle_uncategorized(&context->ss);
790 break;
791
792 case ESR_EC_SP_ALIGN:
793 handle_sp_align(state);
794 __builtin_unreachable();
795
796 case ESR_EC_BKPT_AARCH32:
797 handle_breakpoint(state, esr);
798 __builtin_unreachable();
799
800 case ESR_EC_BRK_AARCH64:
801 #ifdef CONFIG_XNUPOST
802 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
803 break;
804 }
805 #endif /* CONFIG_XNUPOST */
806 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
807 handle_kernel_breakpoint(state, esr);
808 break;
809 } else {
810 handle_breakpoint(state, esr);
811 __builtin_unreachable();
812 }
813
814 case ESR_EC_BKPT_REG_MATCH_EL0:
815 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
816 handle_breakpoint(state, esr);
817 }
818 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
819 class, state, class, esr, (void *)far);
820 __builtin_unreachable();
821
822 case ESR_EC_BKPT_REG_MATCH_EL1:
823 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state);
824 __builtin_unreachable();
825
826 case ESR_EC_SW_STEP_DEBUG_EL0:
827 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
828 handle_sw_step_debug(state);
829 }
830 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
831 class, state, class, esr, (void *)far);
832 __builtin_unreachable();
833
834 case ESR_EC_SW_STEP_DEBUG_EL1:
835 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state);
836 __builtin_unreachable();
837
838 case ESR_EC_WATCHPT_MATCH_EL0:
839 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
840 handle_watchpoint(far);
841 }
842 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
843 class, state, class, esr, (void *)far);
844 __builtin_unreachable();
845
846 case ESR_EC_WATCHPT_MATCH_EL1:
847 /*
848 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
849 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
850 */
851 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
852 arm_debug_set(NULL);
853 break; /* return to first level handler */
854 }
855 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
856 class, state, class, esr, (void *)far);
857 __builtin_unreachable();
858
859 case ESR_EC_TRAP_SIMD_FP:
860 handle_simd_trap(state, esr);
861 __builtin_unreachable();
862
863 case ESR_EC_ILLEGAL_INSTR_SET:
864 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
865 state, class, esr, (void *)far, get_saved_state_cpsr(state));
866 __builtin_unreachable();
867
868 case ESR_EC_MCR_MRC_CP15_TRAP:
869 case ESR_EC_MCRR_MRRC_CP15_TRAP:
870 case ESR_EC_MCR_MRC_CP14_TRAP:
871 case ESR_EC_LDC_STC_CP14_TRAP:
872 case ESR_EC_MCRR_MRRC_CP14_TRAP:
873 handle_user_trapped_instruction32(state, esr);
874 __builtin_unreachable();
875
876 case ESR_EC_WFI_WFE:
877 // Use of WFI or WFE instruction when they have been disabled for EL0
878 handle_wf_trap(state);
879 __builtin_unreachable();
880
881 case ESR_EC_FLOATING_POINT_64:
882 handle_fp_trap(state, esr);
883 __builtin_unreachable();
884
885 default:
886 handle_uncategorized(state);
887 }
888
889 #ifdef CONFIG_XNUPOST
890 if (saved_expected_fault_handler != NULL) {
891 thread->machine.expected_fault_handler = saved_expected_fault_handler;
892 thread->machine.expected_fault_addr = saved_expected_fault_addr;
893 thread->machine.expected_fault_pc = saved_expected_fault_pc;
894 }
895 #endif /* CONFIG_XNUPOST */
896
897 if (is_user) {
898 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
899 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
900 esr, far, get_saved_state_pc(state), 0, 0);
901 thread->machine.exception_trace_code = 0;
902 } else {
903 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
904 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
905 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
906 }
907 #if MACH_ASSERT
908 if (preemption_level != sleh_get_preemption_level()) {
909 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
910 }
911 #endif
912
913 }
914
915 /*
916 * Uncategorized exceptions are a catch-all for general execution errors.
917 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
918 */
919 static void
handle_uncategorized(arm_saved_state_t * state)920 handle_uncategorized(arm_saved_state_t *state)
921 {
922 exception_type_t exception = EXC_BAD_INSTRUCTION;
923 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
924 mach_msg_type_number_t numcodes = 2;
925 uint32_t instr = 0;
926
927 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
928
929 #if CONFIG_DTRACE
930
931 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
932 /*
933 * For a 64bit user process, we care about all 4 bytes of the
934 * instr.
935 */
936 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
937 if (dtrace_user_probe(state) == KERN_SUCCESS) {
938 return;
939 }
940 }
941 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
942 /*
943 * For a 32bit user process, we check for thumb mode, in
944 * which case we only care about a 2 byte instruction length.
945 * For non-thumb mode, we care about all 4 bytes of the instructin.
946 */
947 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
948 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
949 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
950 if (dtrace_user_probe(state) == KERN_SUCCESS) {
951 return;
952 }
953 }
954 } else {
955 if ((instr == FASTTRAP_ARM32_INSTR) ||
956 (instr == FASTTRAP_ARM32_RET_INSTR)) {
957 if (dtrace_user_probe(state) == KERN_SUCCESS) {
958 return;
959 }
960 }
961 }
962 }
963
964 #endif /* CONFIG_DTRACE */
965
966 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
967 if (IS_ARM_GDB_TRAP(instr)) {
968 boolean_t interrupt_state;
969 exception = EXC_BREAKPOINT;
970
971 interrupt_state = ml_set_interrupts_enabled(FALSE);
972
973 /* Save off the context here (so that the debug logic
974 * can see the original state of this thread).
975 */
976 current_thread()->machine.kpcb = state;
977
978 /* Hop into the debugger (typically either due to a
979 * fatal exception, an explicit panic, or a stackshot
980 * request.
981 */
982 DebuggerCall(exception, state);
983
984 current_thread()->machine.kpcb = NULL;
985 (void) ml_set_interrupts_enabled(interrupt_state);
986 return;
987 } else {
988 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
989 }
990 }
991
992 /*
993 * Check for GDB breakpoint via illegal opcode.
994 */
995 if (IS_ARM_GDB_TRAP(instr)) {
996 exception = EXC_BREAKPOINT;
997 codes[0] = EXC_ARM_BREAKPOINT;
998 codes[1] = instr;
999 } else {
1000 codes[1] = instr;
1001 }
1002
1003 exception_triage(exception, codes, numcodes);
1004 __builtin_unreachable();
1005 }
1006
1007 #if __has_feature(ptrauth_calls)
1008 static const uint16_t PTRAUTH_TRAPS_START = 0xC470;
1009 static inline bool
brk_comment_is_ptrauth(uint16_t comment)1010 brk_comment_is_ptrauth(uint16_t comment)
1011 {
1012 return comment >= PTRAUTH_TRAPS_START &&
1013 comment <= PTRAUTH_TRAPS_START + ptrauth_key_asdb;
1014 }
1015
1016 static inline const char *
ptrauth_key_to_string(ptrauth_key key)1017 ptrauth_key_to_string(ptrauth_key key)
1018 {
1019 switch (key) {
1020 case ptrauth_key_asia:
1021 return "IA";
1022 case ptrauth_key_asib:
1023 return "IB";
1024 case ptrauth_key_asda:
1025 return "DA";
1026 case ptrauth_key_asdb:
1027 return "DB";
1028 default:
1029 __builtin_unreachable();
1030 }
1031 }
1032
1033 static void __attribute__((noreturn))
ptrauth_handle_brk_trap(void * tstate,uint16_t comment)1034 ptrauth_handle_brk_trap(void *tstate, uint16_t comment)
1035 {
1036 arm_saved_state_t *state = (arm_saved_state_t *)tstate;
1037 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx"
1038 char msg[strlen(MSG_FMT)
1039 - strlen("0x%04X") + strlen("0xFFFF")
1040 - strlen("%s") + strlen("IA")
1041 - strlen("0x%016llx") + strlen("0xFFFFFFFFFFFFFFFF")
1042 + 1];
1043 ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAPS_START);
1044 const char *key_str = ptrauth_key_to_string(key);
1045 snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]);
1046 #undef MSG_FMT
1047
1048 panic_with_thread_kernel_state(msg, state);
1049 __builtin_unreachable();
1050 }
1051 #endif /* __has_feature(ptrauth_calls) */
1052
1053 #if HAS_TELEMETRY_KERNEL_BRK
1054 static uint32_t bound_chk_violations_event;
1055
1056 static void
telemetry_handle_brk_trap(void * tstate,uint16_t comment)1057 telemetry_handle_brk_trap(
1058 void *tstate,
1059 uint16_t comment)
1060 {
1061 #if CONFIG_UBSAN_MINIMAL
1062 if (comment == UBSAN_SIGNED_OVERFLOW_TRAP) {
1063 ubsan_handle_brk_trap(tstate, comment);
1064 }
1065 #else
1066 (void)tstate;
1067 #endif
1068
1069 if (comment == CLANG_BOUND_CHK_SOFT_TRAP) {
1070 os_atomic_inc(&bound_chk_violations_event, relaxed);
1071 }
1072 }
1073 #endif /* HAS_TELEMETRY_KERNEL_BRK */
1074
1075 #if __has_feature(ptrauth_calls)
1076 KERNEL_BRK_DESCRIPTOR_DEFINE(ptrauth_desc,
1077 .type = KERNEL_BRK_TYPE_PTRAUTH,
1078 .base = PTRAUTH_TRAPS_START,
1079 .max = PTRAUTH_TRAPS_START + ptrauth_key_asdb,
1080 .options = KERNEL_BRK_UNRECOVERABLE,
1081 .handle_breakpoint = ptrauth_handle_brk_trap);
1082 #endif
1083
1084 KERNEL_BRK_DESCRIPTOR_DEFINE(clang_desc,
1085 .type = KERNEL_BRK_TYPE_CLANG,
1086 .base = CLANG_TRAPS_ARM_START,
1087 .max = CLANG_TRAPS_ARM_END,
1088 .options = KERNEL_BRK_UNRECOVERABLE,
1089 .handle_breakpoint = NULL);
1090
1091 KERNEL_BRK_DESCRIPTOR_DEFINE(libcxx_desc,
1092 .type = KERNEL_BRK_TYPE_LIBCXX,
1093 .base = LIBCXX_TRAPS_START,
1094 .max = LIBCXX_TRAPS_END,
1095 .options = KERNEL_BRK_UNRECOVERABLE,
1096 .handle_breakpoint = NULL);
1097
1098 #if HAS_TELEMETRY_KERNEL_BRK
1099 KERNEL_BRK_DESCRIPTOR_DEFINE(telemetry_desc,
1100 .type = KERNEL_BRK_TYPE_TELEMETRY,
1101 .base = TELEMETRY_TRAPS_START,
1102 .max = TELEMETRY_TRAPS_END,
1103 .options = KERNEL_BRK_RECOVERABLE | KERNEL_BRK_CORE_ANALYTICS,
1104 .handle_breakpoint = telemetry_handle_brk_trap);
1105 #endif /* HAS_TELEMETRY_KERNEL_BRK */
1106
1107 static void
1108 #if !HAS_TELEMETRY_KERNEL_BRK
1109 __attribute__((noreturn))
1110 #endif
handle_kernel_breakpoint(arm_saved_state_t * state,uint32_t esr)1111 handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr)
1112 {
1113 uint16_t comment = ISS_BRK_COMMENT(esr);
1114 const struct kernel_brk_descriptor *desc;
1115
1116 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)"
1117 char msg[strlen(MSG_FMT) - strlen("0x%04X") + strlen("0xFFFF") + 1];
1118
1119 desc = find_brk_descriptor_by_comment(comment);
1120
1121 if (!desc) {
1122 goto brk_out;
1123 }
1124
1125 #if HAS_TELEMETRY_KERNEL_BRK
1126 if (desc->options & KERNEL_BRK_TELEMETRY_OPTIONS) {
1127 telemetry_kernel_brk(desc->type, desc->options, (void *)state, comment);
1128 }
1129 #endif
1130
1131 if (desc->handle_breakpoint) {
1132 desc->handle_breakpoint(state, comment); /* May trigger panic */
1133 }
1134
1135 #if HAS_TELEMETRY_KERNEL_BRK
1136 /* Still alive? Check if we should recover. */
1137 if (desc->options & KERNEL_BRK_RECOVERABLE) {
1138 add_saved_state_pc(state, 4);
1139 return;
1140 }
1141 #endif
1142
1143 brk_out:
1144 snprintf(msg, sizeof(msg), MSG_FMT, comment);
1145
1146 panic_with_thread_kernel_state(msg, state);
1147 __builtin_unreachable();
1148 #undef MSG_FMT
1149 }
1150
1151 static void
handle_breakpoint(arm_saved_state_t * state,uint32_t esr __unused)1152 handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused)
1153 {
1154 exception_type_t exception = EXC_BREAKPOINT;
1155 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1156 mach_msg_type_number_t numcodes = 2;
1157
1158 #if __has_feature(ptrauth_calls)
1159 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1160 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1161 exception |= EXC_PTRAUTH_BIT;
1162 }
1163 #endif /* __has_feature(ptrauth_calls) */
1164
1165 codes[1] = get_saved_state_pc(state);
1166 exception_triage(exception, codes, numcodes);
1167 __builtin_unreachable();
1168 }
1169
1170 static void
handle_watchpoint(vm_offset_t fault_addr)1171 handle_watchpoint(vm_offset_t fault_addr)
1172 {
1173 exception_type_t exception = EXC_BREAKPOINT;
1174 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1175 mach_msg_type_number_t numcodes = 2;
1176
1177 codes[1] = fault_addr;
1178 exception_triage(exception, codes, numcodes);
1179 __builtin_unreachable();
1180 }
1181
1182 static void
handle_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,abort_inspector_t inspect_abort,abort_handler_t handler,expected_fault_handler_t expected_fault_handler)1183 handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1184 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1185 {
1186 fault_status_t fault_code;
1187 vm_prot_t fault_type;
1188
1189 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1190 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1191 }
1192
1193 static void
inspect_instruction_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1194 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1195 {
1196 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1197 *fault_code = ISS_IA_FSC(iss);
1198 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1199 }
1200
1201 static void
inspect_data_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1202 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1203 {
1204 getCpuDatap()->cpu_stat.data_ex_cnt++;
1205 *fault_code = ISS_DA_FSC(iss);
1206
1207 /*
1208 * Cache maintenance operations always report faults as write access.
1209 * Change these to read access, unless they report a permission fault.
1210 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1211 * access to the mapping, but if a cache maintenance operation that only requires
1212 * read access generates a permission fault, then we will not be able to handle
1213 * the fault regardless of whether we treat it as a read or write fault.
1214 */
1215 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
1216 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1217 } else {
1218 *fault_type = (VM_PROT_READ);
1219 }
1220 }
1221
1222 #if __has_feature(ptrauth_calls)
1223 static inline uint64_t
fault_addr_bitmask(unsigned int bit_from,unsigned int bit_to)1224 fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1225 {
1226 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1227 }
1228
1229 static inline bool
fault_addr_bit(vm_offset_t fault_addr,unsigned int bit)1230 fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1231 {
1232 return (bool)((fault_addr >> bit) & 1);
1233 }
1234
1235 extern int gARM_FEAT_PAuth2;
1236
1237 /**
1238 * Determines whether a fault address taken at EL0 contains a PAC error code
1239 * corresponding to the specified kind of ptrauth key.
1240 */
1241 static bool
user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr,bool data_key)1242 user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key)
1243 {
1244 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1245 bool tbi = data_key || __improbable(instruction_tbi);
1246
1247 if (gARM_FEAT_PAuth2) {
1248 /*
1249 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1250 * treat all non-canonical address bits like potential poison bits.
1251 */
1252 uint64_t mask = fault_addr_bitmask(T0SZ_BOOT, 54);
1253 if (!tbi) {
1254 mask |= fault_addr_bitmask(56, 63);
1255 }
1256 return (fault_addr & mask) != 0;
1257 } else {
1258 unsigned int poison_shift;
1259 if (tbi) {
1260 poison_shift = 53;
1261 } else {
1262 poison_shift = 61;
1263 }
1264
1265 /* PAC error codes are always in the form key_number:NOT(key_number) */
1266 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1267 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1268 return poison_bit_1 != poison_bit_2;
1269 }
1270 }
1271 #endif /* __has_feature(ptrauth_calls) */
1272
1273 static void
handle_pc_align(arm_saved_state_t * ss)1274 handle_pc_align(arm_saved_state_t *ss)
1275 {
1276 exception_type_t exc;
1277 mach_exception_data_type_t codes[2];
1278 mach_msg_type_number_t numcodes = 2;
1279
1280 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1281 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
1282 }
1283
1284 exc = EXC_BAD_ACCESS;
1285 #if __has_feature(ptrauth_calls)
1286 if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) {
1287 exc |= EXC_PTRAUTH_BIT;
1288 }
1289 #endif /* __has_feature(ptrauth_calls) */
1290
1291 codes[0] = EXC_ARM_DA_ALIGN;
1292 codes[1] = get_saved_state_pc(ss);
1293
1294 exception_triage(exc, codes, numcodes);
1295 __builtin_unreachable();
1296 }
1297
1298 static void
handle_sp_align(arm_saved_state_t * ss)1299 handle_sp_align(arm_saved_state_t *ss)
1300 {
1301 exception_type_t exc;
1302 mach_exception_data_type_t codes[2];
1303 mach_msg_type_number_t numcodes = 2;
1304
1305 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1306 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
1307 }
1308
1309 exc = EXC_BAD_ACCESS;
1310 #if __has_feature(ptrauth_calls)
1311 if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) {
1312 exc |= EXC_PTRAUTH_BIT;
1313 }
1314 #endif /* __has_feature(ptrauth_calls) */
1315
1316 codes[0] = EXC_ARM_SP_ALIGN;
1317 codes[1] = get_saved_state_sp(ss);
1318
1319 exception_triage(exc, codes, numcodes);
1320 __builtin_unreachable();
1321 }
1322
1323 static void
handle_wf_trap(arm_saved_state_t * state)1324 handle_wf_trap(arm_saved_state_t *state)
1325 {
1326 exception_type_t exc;
1327 mach_exception_data_type_t codes[2];
1328 mach_msg_type_number_t numcodes = 2;
1329 uint32_t instr = 0;
1330
1331 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1332
1333 exc = EXC_BAD_INSTRUCTION;
1334 codes[0] = EXC_ARM_UNDEFINED;
1335 codes[1] = instr;
1336
1337 exception_triage(exc, codes, numcodes);
1338 __builtin_unreachable();
1339 }
1340
1341 static void
handle_fp_trap(arm_saved_state_t * state,uint32_t esr)1342 handle_fp_trap(arm_saved_state_t *state, uint32_t esr)
1343 {
1344 exception_type_t exc = EXC_ARITHMETIC;
1345 mach_exception_data_type_t codes[2];
1346 mach_msg_type_number_t numcodes = 2;
1347 uint32_t instr = 0;
1348
1349 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1350 panic_with_thread_kernel_state("Floating point exception from kernel", state);
1351 }
1352
1353 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1354 codes[1] = instr;
1355
1356 /* The floating point trap flags are only valid if TFV is set. */
1357 if (!fp_exceptions_enabled) {
1358 exc = EXC_BAD_INSTRUCTION;
1359 codes[0] = EXC_ARM_UNDEFINED;
1360 } else if (!(esr & ISS_FP_TFV)) {
1361 codes[0] = EXC_ARM_FP_UNDEFINED;
1362 } else if (esr & ISS_FP_UFF) {
1363 codes[0] = EXC_ARM_FP_UF;
1364 } else if (esr & ISS_FP_OFF) {
1365 codes[0] = EXC_ARM_FP_OF;
1366 } else if (esr & ISS_FP_IOF) {
1367 codes[0] = EXC_ARM_FP_IO;
1368 } else if (esr & ISS_FP_DZF) {
1369 codes[0] = EXC_ARM_FP_DZ;
1370 } else if (esr & ISS_FP_IDF) {
1371 codes[0] = EXC_ARM_FP_ID;
1372 } else if (esr & ISS_FP_IXF) {
1373 codes[0] = EXC_ARM_FP_IX;
1374 } else {
1375 panic("Unrecognized floating point exception, state=%p, esr=%#x", state, esr);
1376 }
1377
1378 exception_triage(exc, codes, numcodes);
1379 __builtin_unreachable();
1380 }
1381
1382
1383
1384 /*
1385 * handle_alignment_fault_from_user:
1386 * state: Saved state
1387 *
1388 * Attempts to deal with an alignment fault from userspace (possibly by
1389 * emulating the faulting instruction). If emulation failed due to an
1390 * unservicable fault, the ESR for that fault will be stored in the
1391 * recovery_esr field of the thread by the exception code.
1392 *
1393 * Returns:
1394 * -1: Emulation failed (emulation of state/instr not supported)
1395 * 0: Successfully emulated the instruction
1396 * EFAULT: Emulation failed (probably due to permissions)
1397 * EINVAL: Emulation failed (probably due to a bad address)
1398 */
1399
1400
1401 static int
handle_alignment_fault_from_user(arm_saved_state_t * state,kern_return_t * vmfr)1402 handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1403 {
1404 int ret = -1;
1405
1406 #pragma unused (state)
1407 #pragma unused (vmfr)
1408
1409 return ret;
1410 }
1411
1412
1413
1414
1415 static void
handle_sw_step_debug(arm_saved_state_t * state)1416 handle_sw_step_debug(arm_saved_state_t *state)
1417 {
1418 thread_t thread = current_thread();
1419 exception_type_t exc;
1420 mach_exception_data_type_t codes[2];
1421 mach_msg_type_number_t numcodes = 2;
1422
1423 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1424 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
1425 }
1426
1427 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1428 if (thread->machine.DebugData != NULL) {
1429 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1430 } else {
1431 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
1432 }
1433
1434 mask_user_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);
1435
1436 // Special encoding for gdb single step event on ARM
1437 exc = EXC_BREAKPOINT;
1438 codes[0] = 1;
1439 codes[1] = 0;
1440
1441 exception_triage(exc, codes, numcodes);
1442 __builtin_unreachable();
1443 }
1444
1445 static void
handle_user_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1446 handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1447 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1448 {
1449 exception_type_t exc = EXC_BAD_ACCESS;
1450 mach_exception_data_type_t codes[2];
1451 mach_msg_type_number_t numcodes = 2;
1452 thread_t thread = current_thread();
1453
1454 (void)esr;
1455 (void)expected_fault_handler;
1456
1457 if (ml_at_interrupt_context()) {
1458 panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state);
1459 }
1460
1461 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1462
1463 if (!is_vm_fault(fault_code) &&
1464 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1465 thread_reset_pcs_done_faulting(thread);
1466 }
1467
1468 if (is_vm_fault(fault_code)) {
1469 vm_map_t map = thread->map;
1470 vm_offset_t vm_fault_addr = fault_addr;
1471 kern_return_t result = KERN_FAILURE;
1472
1473 assert(map != kernel_map);
1474
1475 if (!(fault_type & VM_PROT_EXECUTE)) {
1476 vm_fault_addr = VM_USER_STRIP_TBI(fault_addr);
1477 }
1478
1479 /* check to see if it is just a pmap ref/modify fault */
1480 if (!is_translation_fault(fault_code)) {
1481 result = arm_fast_fault(map->pmap,
1482 vm_fault_addr,
1483 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1484 }
1485 if (result != KERN_SUCCESS) {
1486
1487 {
1488 /* We have to fault the page in */
1489 result = vm_fault(map, vm_fault_addr, fault_type,
1490 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1491 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1492 }
1493 }
1494 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1495 thread_reset_pcs_done_faulting(thread);
1496 }
1497 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1498 return;
1499 }
1500
1501 /*
1502 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1503 * If it does, we're leaking preemption disables somewhere in the kernel.
1504 */
1505 if (__improbable(result == KERN_FAILURE)) {
1506 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1507 }
1508
1509 codes[0] = result;
1510 } else if (is_alignment_fault(fault_code)) {
1511 kern_return_t vmfkr = KERN_SUCCESS;
1512 thread->machine.recover_esr = 0;
1513 thread->machine.recover_far = 0;
1514 int result = handle_alignment_fault_from_user(state, &vmfkr);
1515 if (result == 0) {
1516 /* Successfully emulated, or instruction
1517 * copyin() for decode/emulation failed.
1518 * Continue, or redrive instruction.
1519 */
1520 thread_exception_return();
1521 } else if (((result == EFAULT) || (result == EINVAL)) &&
1522 (thread->machine.recover_esr == 0)) {
1523 /*
1524 * If we didn't actually take a fault, but got one of
1525 * these errors, then we failed basic sanity checks of
1526 * the fault address. Treat this as an invalid
1527 * address.
1528 */
1529 codes[0] = KERN_INVALID_ADDRESS;
1530 } else if ((result == EFAULT) &&
1531 (thread->machine.recover_esr)) {
1532 /*
1533 * Since alignment aborts are prioritized
1534 * ahead of translation aborts, the misaligned
1535 * atomic emulation flow may have triggered a
1536 * VM pagefault, which the VM could not resolve.
1537 * Report the VM fault error in codes[]
1538 */
1539
1540 codes[0] = vmfkr;
1541 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1542 /* Cause ESR_EC to reflect an EL0 abort */
1543 thread->machine.recover_esr &= ~ESR_EC_MASK;
1544 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1545 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr);
1546 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far);
1547 fault_addr = thread->machine.recover_far;
1548 } else {
1549 /* This was just an unsupported alignment
1550 * exception. Misaligned atomic emulation
1551 * timeouts fall in this category.
1552 */
1553 codes[0] = EXC_ARM_DA_ALIGN;
1554 }
1555 } else if (is_parity_error(fault_code)) {
1556 #if defined(APPLE_ARM64_ARCH_FAMILY)
1557 /*
1558 * Platform errors are handled in sleh_sync before interrupts are enabled.
1559 */
1560 #else
1561 panic("User parity error.");
1562 #endif
1563 } else {
1564 codes[0] = KERN_FAILURE;
1565 }
1566
1567 codes[1] = fault_addr;
1568 #if __has_feature(ptrauth_calls)
1569 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1570 if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) {
1571 exc |= EXC_PTRAUTH_BIT;
1572 }
1573 #endif /* __has_feature(ptrauth_calls) */
1574 exception_triage(exc, codes, numcodes);
1575 __builtin_unreachable();
1576 }
1577
1578 /**
1579 * Panic because the kernel abort handler tried to apply a recovery handler that
1580 * isn't inside copyio_recover_table[].
1581 *
1582 * @param state original saved-state
1583 * @param recover invalid recovery handler
1584 */
1585 __attribute__((noreturn, used))
1586 static void
panic_on_invalid_recovery_handler(arm_saved_state_t * state,struct copyio_recovery_entry * recover)1587 panic_on_invalid_recovery_handler(arm_saved_state_t *state, struct copyio_recovery_entry *recover)
1588 {
1589 panic("attempt to set invalid recovery handler %p on kernel saved-state %p", recover, state);
1590 }
1591
1592 static void
handle_kernel_abort_recover(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,thread_t thread,struct copyio_recovery_entry * _Nonnull recover)1593 handle_kernel_abort_recover(
1594 arm_saved_state_t *state,
1595 uint32_t esr,
1596 vm_offset_t fault_addr,
1597 thread_t thread,
1598 struct copyio_recovery_entry *_Nonnull recover)
1599 {
1600 thread->machine.recover_esr = esr;
1601 thread->machine.recover_far = fault_addr;
1602 #if defined(HAS_APPLE_PAC)
1603 MANIPULATE_SIGNED_THREAD_STATE(state,
1604 "adrp x6, _copyio_recover_table_end@page \n"
1605 "add x6, x6, _copyio_recover_table_end@pageoff \n"
1606 "cmp %[recover], x6 \n"
1607 "b.lt 1f \n"
1608 "bl _panic_on_invalid_recovery_handler \n"
1609 "brk #0 \n"
1610 "1: \n"
1611 "adrp x6, _copyio_recover_table@page \n"
1612 "add x6, x6, _copyio_recover_table@pageoff \n"
1613 "cmp %[recover], x6 \n"
1614 "b.ge 1f \n"
1615 "bl _panic_on_invalid_recovery_handler \n"
1616 "brk #0 \n"
1617 "1: \n"
1618 "ldr x1, [%[recover], %[CRE_RECOVERY]] \n"
1619 "add x1, x1, x6 \n"
1620 "str x1, [x0, %[SS64_PC]] \n",
1621 [recover] "r"(recover),
1622 [CRE_RECOVERY] "i"(offsetof(struct copyio_recovery_entry, cre_recovery))
1623 );
1624 #else
1625 if ((uintptr_t)recover < (uintptr_t)copyio_recover_table ||
1626 (uintptr_t)recover >= (uintptr_t)copyio_recover_table_end) {
1627 panic_on_invalid_recovery_handler(state, recover);
1628 }
1629 saved_state64(state)->pc = copyio_recovery_addr(recover->cre_recovery);
1630 #endif
1631 }
1632
1633 static void
handle_kernel_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1634 handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1635 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1636 {
1637 thread_t thread = current_thread();
1638 struct copyio_recovery_entry *recover = find_copyio_recovery_entry(state);
1639
1640 #ifdef CONFIG_KERNEL_TAGGING
1641 /*
1642 * If a read/write access to a tagged address faults over pageable kernel memory
1643 * vm_fault() will need to find the right vm entry and offset. Canonicalize the
1644 * address here so that the correct comparisons can happen later in the VM code.
1645 */
1646 if (!(fault_type & VM_PROT_EXECUTE) && VM_KERNEL_ADDRESS(fault_addr)) {
1647 fault_addr = vm_memtag_canonicalize_address(fault_addr);
1648 }
1649 #endif /* CONFIG_KERNEL_TAGGING */
1650
1651 #ifndef CONFIG_XNUPOST
1652 (void)expected_fault_handler;
1653 #endif /* CONFIG_XNUPOST */
1654
1655 #if CONFIG_DTRACE
1656 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
1657 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1658 /*
1659 * Point to next instruction, or recovery handler if set.
1660 */
1661 if (recover) {
1662 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
1663 } else {
1664 add_saved_state_pc(state, 4);
1665 }
1666 return;
1667 } else {
1668 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
1669 }
1670 }
1671 #endif
1672
1673 if (ml_at_interrupt_context()) {
1674 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1675 }
1676
1677 if (is_vm_fault(fault_code)) {
1678 kern_return_t result = KERN_FAILURE;
1679 vm_map_t map;
1680 int interruptible;
1681
1682 /*
1683 * Ensure no faults in the physical aperture. This could happen if
1684 * a page table is incorrectly allocated from the read only region
1685 * when running with KTRR.
1686 */
1687
1688 #ifdef CONFIG_XNUPOST
1689 if (expected_fault_handler && expected_fault_handler(state)) {
1690 return;
1691 }
1692 #endif /* CONFIG_XNUPOST */
1693
1694 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1695 panic_with_thread_kernel_state("Unexpected fault in kernel static region\n", state);
1696 }
1697
1698 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) {
1699 /*
1700 * If no recovery handler is supplied, always drive the fault against
1701 * the kernel map. If the fault was taken against a userspace VA, indicating
1702 * an unprotected access to user address space, vm_fault() should fail and
1703 * ultimately lead to a panic here.
1704 */
1705 map = kernel_map;
1706 interruptible = THREAD_UNINT;
1707 } else {
1708 map = thread->map;
1709
1710 /**
1711 * In the case that the recovery handler is set (e.g., during copyio
1712 * and dtrace probes), we don't want the vm_fault() operation to be
1713 * aborted early. Those code paths can't handle restarting the
1714 * vm_fault() operation so don't allow it to return early without
1715 * creating the wanted mapping.
1716 */
1717 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
1718 }
1719
1720 /* check to see if it is just a pmap ref/modify fault */
1721 if (!is_translation_fault(fault_code)) {
1722 result = arm_fast_fault(map->pmap,
1723 fault_addr,
1724 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
1725 if (result == KERN_SUCCESS) {
1726 return;
1727 }
1728 }
1729
1730 if (result != KERN_PROTECTION_FAILURE) {
1731 /*
1732 * We have to "fault" the page in.
1733 */
1734 result = vm_fault(map, fault_addr, fault_type,
1735 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1736 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1737 }
1738
1739 if (result == KERN_SUCCESS) {
1740 return;
1741 }
1742
1743 /*
1744 * If we have a recover handler, invoke it now.
1745 */
1746 if (recover) {
1747 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
1748 return;
1749 }
1750
1751 panic_fault_address = fault_addr;
1752 } else if (is_alignment_fault(fault_code)) {
1753 if (recover) {
1754 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
1755 return;
1756 }
1757 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
1758 } else if (is_parity_error(fault_code)) {
1759 #if defined(APPLE_ARM64_ARCH_FAMILY)
1760 /*
1761 * Platform errors are handled in sleh_sync before interrupts are enabled.
1762 */
1763 #else
1764 panic_with_thread_kernel_state("Kernel parity error.", state);
1765 #endif
1766 } else {
1767 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1768 }
1769
1770 panic_with_thread_kernel_state("Kernel data abort.", state);
1771 }
1772
1773 extern void syscall_trace(struct arm_saved_state * regs);
1774
1775 static void
handle_svc(arm_saved_state_t * state)1776 handle_svc(arm_saved_state_t *state)
1777 {
1778 int trap_no = get_saved_state_svc_number(state);
1779 thread_t thread = current_thread();
1780 struct proc *p;
1781
1782 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1783
1784 #define TRACE_SYSCALL 1
1785 #if TRACE_SYSCALL
1786 syscall_trace(state);
1787 #endif
1788
1789 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1790
1791 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1792 platform_syscall(state);
1793 panic("Returned from platform_syscall()?");
1794 }
1795
1796 mach_kauth_cred_thread_update();
1797
1798 if (trap_no < 0) {
1799 switch (trap_no) {
1800 case MACH_ARM_TRAP_ABSTIME:
1801 handle_mach_absolute_time_trap(state);
1802 return;
1803 case MACH_ARM_TRAP_CONTTIME:
1804 handle_mach_continuous_time_trap(state);
1805 return;
1806 }
1807
1808 /* Counting perhaps better in the handler, but this is how it's been done */
1809 thread->syscalls_mach++;
1810 mach_syscall(state);
1811 } else {
1812 /* Counting perhaps better in the handler, but this is how it's been done */
1813 thread->syscalls_unix++;
1814 p = get_bsdthreadtask_info(thread);
1815
1816 assert(p);
1817
1818 unix_syscall(state, thread, p);
1819 }
1820 }
1821
1822 static void
handle_mach_absolute_time_trap(arm_saved_state_t * state)1823 handle_mach_absolute_time_trap(arm_saved_state_t *state)
1824 {
1825 uint64_t now = mach_absolute_time();
1826 saved_state64(state)->x[0] = now;
1827 }
1828
1829 static void
handle_mach_continuous_time_trap(arm_saved_state_t * state)1830 handle_mach_continuous_time_trap(arm_saved_state_t *state)
1831 {
1832 uint64_t now = mach_continuous_time();
1833 saved_state64(state)->x[0] = now;
1834 }
1835
1836
1837 __attribute__((noreturn))
1838 static void
handle_msr_trap(arm_saved_state_t * state,uint32_t esr)1839 handle_msr_trap(arm_saved_state_t *state, uint32_t esr)
1840 {
1841 exception_type_t exception = EXC_BAD_INSTRUCTION;
1842 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1843 mach_msg_type_number_t numcodes = 2;
1844 uint32_t instr = 0;
1845
1846 if (!is_saved_state64(state)) {
1847 panic("MSR/MRS trap (ESR 0x%x) from 32-bit state", esr);
1848 }
1849
1850 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1851 panic("MSR/MRS trap (ESR 0x%x) from kernel", esr);
1852 }
1853
1854 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1855 codes[1] = instr;
1856
1857 exception_triage(exception, codes, numcodes);
1858 __builtin_unreachable();
1859 }
1860
1861 #if __has_feature(ptrauth_calls)
1862 static void
stringify_gpr(unsigned int r,char reg[4])1863 stringify_gpr(unsigned int r, char reg[4])
1864 {
1865 switch (r) {
1866 case 29:
1867 strncpy(reg, "fp", 4);
1868 return;
1869
1870 case 30:
1871 strncpy(reg, "lr", 4);
1872 return;
1873
1874 case 31:
1875 strncpy(reg, "xzr", 4);
1876 return;
1877
1878 default:
1879 snprintf(reg, 4, "x%u", r);
1880 return;
1881 }
1882 }
1883
1884 static void
autxx_instruction_extract_reg(uint32_t instr,char reg[4])1885 autxx_instruction_extract_reg(uint32_t instr, char reg[4])
1886 {
1887 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
1888 stringify_gpr(rd, reg);
1889 }
1890
1891 static const char *
autix_system_instruction_extract_reg(uint32_t instr)1892 autix_system_instruction_extract_reg(uint32_t instr)
1893 {
1894 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
1895 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
1896 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
1897 return "x17";
1898 } else {
1899 return "lr";
1900 }
1901 }
1902
1903
1904 static void
handle_pac_fail(arm_saved_state_t * state,uint32_t esr)1905 handle_pac_fail(arm_saved_state_t *state, uint32_t esr)
1906 {
1907 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
1908 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
1909 mach_msg_type_number_t numcodes = 2;
1910 uint32_t instr = 0;
1911
1912 if (!is_saved_state64(state)) {
1913 panic("PAC failure (ESR 0x%x) from 32-bit state", esr);
1914 }
1915
1916 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1917
1918 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1919 #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
1920 #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
1921 #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
1922 #define MAX_PAC_MSG_FMT AUTXX_MSG_FMT
1923
1924 char msg[strlen(MAX_PAC_MSG_FMT)
1925 - strlen("%s") + strlen("IA")
1926 - strlen("%s") + strlen("xzr")
1927 + 1];
1928 ptrauth_key key = (ptrauth_key)(esr & 0x3);
1929 const char *key_str = ptrauth_key_to_string(key);
1930
1931 if (ARM64_INSTR_IS_AUTxx(instr)) {
1932 char reg[4];
1933 autxx_instruction_extract_reg(instr, reg);
1934 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1935 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
1936 const char *reg = autix_system_instruction_extract_reg(instr);
1937 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1938 } else {
1939 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
1940 }
1941 panic_with_thread_kernel_state(msg, state);
1942 }
1943
1944 codes[1] = instr;
1945
1946 exception_triage(exception, codes, numcodes);
1947 __builtin_unreachable();
1948 }
1949 #endif /* __has_feature(ptrauth_calls) */
1950
1951 static void
handle_user_trapped_instruction32(arm_saved_state_t * state,uint32_t esr)1952 handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
1953 {
1954 exception_type_t exception = EXC_BAD_INSTRUCTION;
1955 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1956 mach_msg_type_number_t numcodes = 2;
1957 uint32_t instr;
1958
1959 if (is_saved_state64(state)) {
1960 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
1961 }
1962
1963 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1964 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
1965 }
1966
1967 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1968 codes[1] = instr;
1969
1970 exception_triage(exception, codes, numcodes);
1971 __builtin_unreachable();
1972 }
1973
1974 static void
handle_simd_trap(arm_saved_state_t * state,uint32_t esr)1975 handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
1976 {
1977 exception_type_t exception = EXC_BAD_INSTRUCTION;
1978 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1979 mach_msg_type_number_t numcodes = 2;
1980 uint32_t instr = 0;
1981
1982 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1983 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
1984 }
1985
1986 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1987 codes[1] = instr;
1988
1989 exception_triage(exception, codes, numcodes);
1990 __builtin_unreachable();
1991 }
1992
1993 void
sleh_irq(arm_saved_state_t * state)1994 sleh_irq(arm_saved_state_t *state)
1995 {
1996 cpu_data_t * cdp __unused = getCpuDatap();
1997 #if MACH_ASSERT
1998 int preemption_level = sleh_get_preemption_level();
1999 #endif
2000
2001
2002 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
2003
2004 #if USE_APPLEARMSMP
2005 PE_handle_ext_interrupt();
2006 #else
2007 /* Run the registered interrupt handler. */
2008 cdp->interrupt_handler(cdp->interrupt_target,
2009 cdp->interrupt_refCon,
2010 cdp->interrupt_nub,
2011 cdp->interrupt_source);
2012 #endif
2013
2014 entropy_collect();
2015
2016
2017 sleh_interrupt_handler_epilogue();
2018 #if MACH_ASSERT
2019 if (preemption_level != sleh_get_preemption_level()) {
2020 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
2021 }
2022 #endif
2023 }
2024
2025 void
sleh_fiq(arm_saved_state_t * state)2026 sleh_fiq(arm_saved_state_t *state)
2027 {
2028 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
2029 #if MACH_ASSERT
2030 int preemption_level = sleh_get_preemption_level();
2031 #endif
2032
2033 #if MONOTONIC_FIQ
2034 uint64_t pmcr0 = 0, upmsr = 0;
2035 #endif /* MONOTONIC_FIQ */
2036
2037 #if defined(HAS_IPI)
2038 boolean_t is_ipi = FALSE;
2039 uint64_t ipi_sr = 0;
2040
2041 if (gFastIPI) {
2042 MRS(ipi_sr, "S3_5_C15_C1_1");
2043
2044 if (ipi_sr & ARM64_IPISR_IPI_PENDING) {
2045 is_ipi = TRUE;
2046 }
2047 }
2048
2049 if (is_ipi) {
2050 type = DBG_INTR_TYPE_IPI;
2051 } else
2052 #endif /* defined(HAS_IPI) */
2053 if (ml_get_timer_pending()) {
2054 type = DBG_INTR_TYPE_TIMER;
2055 }
2056 #if MONOTONIC_FIQ
2057 /* Consult the PMI sysregs last, after IPI/timer
2058 * classification.
2059 */
2060 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
2061 type = DBG_INTR_TYPE_PMI;
2062 }
2063 #endif /* MONOTONIC_FIQ */
2064
2065 sleh_interrupt_handler_prologue(state, type);
2066
2067 #if APPLEVIRTUALPLATFORM
2068 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
2069 #endif
2070
2071 #if defined(HAS_IPI)
2072 if (type == DBG_INTR_TYPE_IPI) {
2073 /*
2074 * Order is important here: we must ack the IPI by writing IPI_SR
2075 * before we call cpu_signal_handler(). Otherwise, there will be
2076 * a window between the completion of pending-signal processing in
2077 * cpu_signal_handler() and the ack during which a newly-issued
2078 * IPI to this CPU may be lost. ISB is required to ensure the msr
2079 * is retired before execution of cpu_signal_handler().
2080 */
2081 MSR("S3_5_C15_C1_1", ARM64_IPISR_IPI_PENDING);
2082 __builtin_arm_isb(ISB_SY);
2083 cpu_signal_handler();
2084 } else
2085 #endif /* defined(HAS_IPI) */
2086 #if MONOTONIC_FIQ
2087 if (type == DBG_INTR_TYPE_PMI) {
2088 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
2089 mt_fiq(getCpuDatap(), pmcr0, upmsr);
2090 INTERRUPT_MASKED_DEBUG_END();
2091 } else
2092 #endif /* MONOTONIC_FIQ */
2093 {
2094 /*
2095 * We don't know that this is a timer, but we don't have insight into
2096 * the other interrupts that go down this path.
2097 */
2098
2099 cpu_data_t *cdp = getCpuDatap();
2100
2101 cdp->cpu_decrementer = -1; /* Large */
2102
2103 /*
2104 * ARM64_TODO: whether we're coming from userland is ignored right now.
2105 * We can easily thread it through, but not bothering for the
2106 * moment (AArch32 doesn't either).
2107 */
2108 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
2109 rtclock_intr(TRUE);
2110 INTERRUPT_MASKED_DEBUG_END();
2111 }
2112
2113 #if APPLEVIRTUALPLATFORM
2114 if (iar != GIC_SPURIOUS_IRQ) {
2115 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
2116 __builtin_arm_isb(ISB_SY);
2117 }
2118 #endif
2119
2120 sleh_interrupt_handler_epilogue();
2121 #if MACH_ASSERT
2122 if (preemption_level != sleh_get_preemption_level()) {
2123 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
2124 }
2125 #endif
2126 }
2127
2128 void
sleh_serror(arm_context_t * context,uint32_t esr,vm_offset_t far)2129 sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
2130 {
2131 task_vtimer_check(current_thread());
2132
2133 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
2134 esr, VM_KERNEL_ADDRHIDE(far));
2135 arm_saved_state_t *state = &context->ss;
2136 #if MACH_ASSERT
2137 int preemption_level = sleh_get_preemption_level();
2138 #endif
2139
2140 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
2141 /* Sanitize FAR (only if we came from userspace) */
2142 saved_state64(state)->far = 0;
2143 }
2144
2145 ASSERT_CONTEXT_SANITY(context);
2146 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC);
2147 #if MACH_ASSERT
2148 if (preemption_level != sleh_get_preemption_level()) {
2149 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
2150 }
2151 #endif
2152 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
2153 esr, VM_KERNEL_ADDRHIDE(far));
2154 }
2155
2156 void
mach_syscall_trace_exit(unsigned int retval,unsigned int call_number)2157 mach_syscall_trace_exit(unsigned int retval,
2158 unsigned int call_number)
2159 {
2160 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2161 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
2162 DBG_FUNC_END, retval, 0, 0, 0, 0);
2163 }
2164
2165 __attribute__((noreturn))
2166 void
thread_syscall_return(kern_return_t error)2167 thread_syscall_return(kern_return_t error)
2168 {
2169 thread_t thread;
2170 struct arm_saved_state *state;
2171
2172 thread = current_thread();
2173 state = get_user_regs(thread);
2174
2175 assert(is_saved_state64(state));
2176 saved_state64(state)->x[0] = error;
2177
2178 #if MACH_ASSERT
2179 kern_allocation_name_t
2180 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
2181 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2182 #endif /* MACH_ASSERT */
2183
2184 if (kdebug_enable) {
2185 /* Invert syscall number (negative for a mach syscall) */
2186 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
2187 }
2188
2189 thread_exception_return();
2190 }
2191
2192 void
syscall_trace(struct arm_saved_state * regs __unused)2193 syscall_trace(
2194 struct arm_saved_state * regs __unused)
2195 {
2196 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2197 }
2198
2199 static void
sleh_interrupt_handler_prologue(arm_saved_state_t * state,unsigned int type)2200 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2201 {
2202 const bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2203
2204 if (is_user == true) {
2205 /* Sanitize FAR (only if the interrupt occurred while the CPU was in usermode) */
2206 saved_state64(state)->far = 0;
2207 }
2208
2209 recount_enter_interrupt();
2210
2211 task_vtimer_check(current_thread());
2212
2213 uint64_t pc = is_user ? get_saved_state_pc(state) :
2214 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2215
2216 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2217 0, pc, is_user, type);
2218
2219 #if CONFIG_TELEMETRY
2220 if (telemetry_needs_record) {
2221 telemetry_mark_curthread(is_user, FALSE);
2222 }
2223 #endif /* CONFIG_TELEMETRY */
2224 }
2225
2226 static void
sleh_interrupt_handler_epilogue(void)2227 sleh_interrupt_handler_epilogue(void)
2228 {
2229 #if KPERF
2230 kperf_interrupt();
2231 #endif /* KPERF */
2232 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2233 recount_leave_interrupt();
2234 }
2235
2236 void
sleh_invalid_stack(arm_context_t * context,uint32_t esr __unused,vm_offset_t far __unused)2237 sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
2238 {
2239 thread_t thread = current_thread();
2240 vm_offset_t kernel_stack_bottom, sp;
2241
2242 sp = get_saved_state_sp(&context->ss);
2243 vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr;
2244 kernel_stack_bottom = round_page(kstackptr) - KERNEL_STACK_SIZE;
2245
2246 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2247 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
2248 }
2249
2250 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
2251 }
2252
2253
2254 #if DEVELOPMENT || DEBUG
2255 static int trap_handled;
2256 static void
handle_recoverable_kernel_trap(__unused void * tstate,uint16_t comment)2257 handle_recoverable_kernel_trap(
2258 __unused void *tstate,
2259 uint16_t comment)
2260 {
2261 assert(comment == TEST_RECOVERABLE_SOFT_TRAP);
2262
2263 printf("Recoverable trap handled.\n");
2264 trap_handled = 1;
2265 }
2266
2267 KERNEL_BRK_DESCRIPTOR_DEFINE(test_desc,
2268 .type = KERNEL_BRK_TYPE_TEST,
2269 .base = TEST_RECOVERABLE_SOFT_TRAP,
2270 .max = TEST_RECOVERABLE_SOFT_TRAP,
2271 .options = KERNEL_BRK_RECOVERABLE,
2272 .handle_breakpoint = handle_recoverable_kernel_trap);
2273
2274 static int
recoverable_kernel_trap_test(__unused int64_t in,int64_t * out)2275 recoverable_kernel_trap_test(__unused int64_t in, int64_t *out)
2276 {
2277 ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP);
2278
2279 *out = trap_handled;
2280 return 0;
2281 }
2282
2283 SYSCTL_TEST_REGISTER(recoverable_kernel_trap, recoverable_kernel_trap_test);
2284
2285 #endif
2286