1 /*
2 * Copyright (c) 2012-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39 #include <arm64/instructions.h>
40
41 #include <kern/debug.h>
42 #include <kern/socd_client.h>
43 #include <kern/thread.h>
44 #include <mach/exception.h>
45 #include <mach/arm/traps.h>
46 #include <mach/vm_types.h>
47 #include <mach/machine/thread_status.h>
48
49 #include <machine/atomic.h>
50 #include <machine/limits.h>
51
52 #include <pexpert/arm/protos.h>
53
54 #include <vm/vm_page.h>
55 #include <vm/pmap.h>
56 #include <vm/vm_fault.h>
57 #include <vm/vm_kern.h>
58
59 #include <sys/errno.h>
60 #include <sys/kdebug.h>
61 #include <kperf/kperf.h>
62
63 #include <kern/policy_internal.h>
64 #if CONFIG_TELEMETRY
65 #include <kern/telemetry.h>
66 #endif
67
68 #include <prng/entropy.h>
69
70
71
72
73
74 #if CONFIG_KERNEL_TBI && KASAN_TBI
75 #include <san/kasan.h>
76 #endif /* CONFIG_KERNEL_TBI && KASAN_TBI */
77
78 #if CONFIG_UBSAN_MINIMAL
79 #include <san/ubsan_minimal.h>
80 #endif /* CONFIG_UBSAN_MINIMAL */
81
82 #ifndef __arm64__
83 #error Should only be compiling for arm64.
84 #endif
85
86 #define TEST_CONTEXT32_SANITY(context) \
87 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
88 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
89
90 #define TEST_CONTEXT64_SANITY(context) \
91 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
92 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
93
94 #define ASSERT_CONTEXT_SANITY(context) \
95 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
96
97
98 #define COPYIN(src, dst, size) \
99 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
100 copyin_kern(src, dst, size) : \
101 copyin(src, dst, size)
102
103 #define COPYOUT(src, dst, size) \
104 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
105 copyout_kern(src, dst, size) : \
106 copyout(src, dst, size)
107
108 // Below is for concatenating a string param to a string literal
109 #define STR1(x) #x
110 #define STR(x) STR1(x)
111
112 #define ARM64_KDBG_CODE_KERNEL (0 << 8)
113 #define ARM64_KDBG_CODE_USER (1 << 8)
114 #define ARM64_KDBG_CODE_GUEST (2 << 8)
115
116 _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
117 _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
118
119 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
120
121 void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike;
122 void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t);
123
124
125
126 void sleh_irq(arm_saved_state_t *);
127 void sleh_fiq(arm_saved_state_t *);
128 void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
129 void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2;
130
131 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
132 static void sleh_interrupt_handler_epilogue(void);
133
134 static void handle_svc(arm_saved_state_t *);
135 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
136 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
137
138 static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr);
139 #ifdef __ARM_ARCH_8_6__
140 static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2;
141 #endif
142
143 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
144
145 static void handle_uncategorized(arm_saved_state_t *);
146
147 /*
148 * For UBSan trap and continue handling, we must be able to recover
149 * from handle_kernel_breakpoint().
150 */
151 #if !CONFIG_UBSAN_MINIMAL
152 __dead2
153 #endif /* CONFIG_UBSAN_MINIMAL */
154 static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t);
155
156 static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2;
157
158 typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
159 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
160 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
161
162 static int is_vm_fault(fault_status_t);
163 static int is_translation_fault(fault_status_t);
164 static int is_alignment_fault(fault_status_t);
165
166 typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
167 static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
168 static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
169
170 static void handle_pc_align(arm_saved_state_t *ss) __dead2;
171 static void handle_sp_align(arm_saved_state_t *ss) __dead2;
172 static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
173 static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
174 static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2;
175
176 static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
177
178 static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
179
180 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2;
181
182 static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2;
183
184 extern void mach_kauth_cred_thread_update(void);
185 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
186
187 struct proc;
188
189 typedef uint32_t arm64_instr_t;
190
191 extern void
192 unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
193
194 extern void
195 mach_syscall(struct arm_saved_state*);
196
197 #if CONFIG_DTRACE
198 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
199 extern boolean_t dtrace_tally_fault(user_addr_t);
200
201 /*
202 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
203 * and paste the trap instructions
204 * over from that file. Need to keep these in sync!
205 */
206 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
207 #define FASTTRAP_THUMB32_INSTR 0xdefc
208 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
209
210 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
211 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
212 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
213
214 /* See <rdar://problem/4613924> */
215 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
216 #endif
217
218
219
220 extern void arm64_thread_exception_return(void) __dead2;
221
222 #if defined(APPLETYPHOON)
223 #define CPU_NAME "Typhoon"
224 #elif defined(APPLETWISTER)
225 #define CPU_NAME "Twister"
226 #elif defined(APPLEHURRICANE)
227 #define CPU_NAME "Hurricane"
228 #elif defined(APPLELIGHTNING)
229 #define CPU_NAME "Lightning"
230 #else
231 #define CPU_NAME "Unknown"
232 #endif
233
234 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
235 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
236 #define ESR_WT_REASON(esr) ((esr) & 0xff)
237
238 #define WT_REASON_NONE 0
239 #define WT_REASON_INTEGRITY_FAIL 1
240 #define WT_REASON_BAD_SYSCALL 2
241 #define WT_REASON_NOT_LOCKED 3
242 #define WT_REASON_ALREADY_LOCKED 4
243 #define WT_REASON_SW_REQ 5
244 #define WT_REASON_PT_INVALID 6
245 #define WT_REASON_PT_VIOLATION 7
246 #define WT_REASON_REG_VIOLATION 8
247 #endif
248
249 #if defined(HAS_IPI)
250 void cpu_signal_handler(void);
251 extern unsigned int gFastIPI;
252 #endif /* defined(HAS_IPI) */
253
254 static arm_saved_state64_t *original_faulting_state = NULL;
255
256 TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
257
258 extern vm_offset_t static_memory_end;
259
260 /*
261 * Fault copyio_recovery_entry in copyin/copyout routines.
262 *
263 * Offets are expressed in bytes from ©_recovery_table
264 */
265 struct copyio_recovery_entry {
266 ptrdiff_t cre_start;
267 ptrdiff_t cre_end;
268 ptrdiff_t cre_recovery;
269 };
270
271 extern struct copyio_recovery_entry copyio_recover_table[];
272 extern struct copyio_recovery_entry copyio_recover_table_end[];
273
274 static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)275 copyio_recovery_offset(uintptr_t addr)
276 {
277 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
278 }
279
280 static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)281 copyio_recovery_addr(ptrdiff_t offset)
282 {
283 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
284 }
285
286 static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(arm_saved_state_t * state)287 find_copyio_recovery_entry(arm_saved_state_t *state)
288 {
289 ptrdiff_t offset = copyio_recovery_offset(get_saved_state_pc(state));
290 struct copyio_recovery_entry *e;
291
292 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
293 if (offset >= e->cre_start && offset < e->cre_end) {
294 return e;
295 }
296 }
297
298 return NULL;
299 }
300
301 static inline uintptr_t
copyio_recovery_get_recover_addr(arm_saved_state_t * state)302 copyio_recovery_get_recover_addr(
303 arm_saved_state_t *state)
304 {
305 struct copyio_recovery_entry *e = find_copyio_recovery_entry(state);
306 if (e == NULL) {
307 panic("copyio recovery: couldn't find a range for %p",
308 (void *)get_saved_state_pc(state));
309 }
310 return copyio_recovery_addr(e->cre_recovery);
311 }
312
313 static inline int
is_vm_fault(fault_status_t status)314 is_vm_fault(fault_status_t status)
315 {
316 switch (status) {
317 case FSC_TRANSLATION_FAULT_L0:
318 case FSC_TRANSLATION_FAULT_L1:
319 case FSC_TRANSLATION_FAULT_L2:
320 case FSC_TRANSLATION_FAULT_L3:
321 case FSC_ACCESS_FLAG_FAULT_L1:
322 case FSC_ACCESS_FLAG_FAULT_L2:
323 case FSC_ACCESS_FLAG_FAULT_L3:
324 case FSC_PERMISSION_FAULT_L1:
325 case FSC_PERMISSION_FAULT_L2:
326 case FSC_PERMISSION_FAULT_L3:
327 return TRUE;
328 default:
329 return FALSE;
330 }
331 }
332
333 static inline int
is_translation_fault(fault_status_t status)334 is_translation_fault(fault_status_t status)
335 {
336 switch (status) {
337 case FSC_TRANSLATION_FAULT_L0:
338 case FSC_TRANSLATION_FAULT_L1:
339 case FSC_TRANSLATION_FAULT_L2:
340 case FSC_TRANSLATION_FAULT_L3:
341 return TRUE;
342 default:
343 return FALSE;
344 }
345 }
346
347 static inline int
is_permission_fault(fault_status_t status)348 is_permission_fault(fault_status_t status)
349 {
350 switch (status) {
351 case FSC_PERMISSION_FAULT_L1:
352 case FSC_PERMISSION_FAULT_L2:
353 case FSC_PERMISSION_FAULT_L3:
354 return TRUE;
355 default:
356 return FALSE;
357 }
358 }
359
360 static inline int
is_alignment_fault(fault_status_t status)361 is_alignment_fault(fault_status_t status)
362 {
363 return status == FSC_ALIGNMENT_FAULT;
364 }
365
366 static inline int
is_parity_error(fault_status_t status)367 is_parity_error(fault_status_t status)
368 {
369 switch (status) {
370 case FSC_SYNC_PARITY:
371 case FSC_ASYNC_PARITY:
372 case FSC_SYNC_PARITY_TT_L1:
373 case FSC_SYNC_PARITY_TT_L2:
374 case FSC_SYNC_PARITY_TT_L3:
375 return TRUE;
376 default:
377 return FALSE;
378 }
379 }
380
381 __dead2 __unused
382 static void
arm64_implementation_specific_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far)383 arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
384 {
385 #pragma unused (state, esr, far)
386 panic_plain("Unhandled implementation specific error\n");
387 }
388
389 #if CONFIG_KERNEL_INTEGRITY
390 #pragma clang diagnostic push
391 #pragma clang diagnostic ignored "-Wunused-parameter"
392 static void
kernel_integrity_error_handler(uint32_t esr,vm_offset_t far)393 kernel_integrity_error_handler(uint32_t esr, vm_offset_t far)
394 {
395 #if defined(KERNEL_INTEGRITY_WT)
396 #if (DEVELOPMENT || DEBUG)
397 if (ESR_WT_SERROR(esr)) {
398 switch (ESR_WT_REASON(esr)) {
399 case WT_REASON_INTEGRITY_FAIL:
400 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
401 case WT_REASON_BAD_SYSCALL:
402 panic_plain("Kernel integrity, bad syscall.");
403 case WT_REASON_NOT_LOCKED:
404 panic_plain("Kernel integrity, not locked.");
405 case WT_REASON_ALREADY_LOCKED:
406 panic_plain("Kernel integrity, already locked.");
407 case WT_REASON_SW_REQ:
408 panic_plain("Kernel integrity, software request.");
409 case WT_REASON_PT_INVALID:
410 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
411 "walking 0x%016lx.", far);
412 case WT_REASON_PT_VIOLATION:
413 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
414 far);
415 case WT_REASON_REG_VIOLATION:
416 panic_plain("Kernel integrity, violation in system register %d.",
417 (unsigned) far);
418 default:
419 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
420 }
421 }
422 #else
423 if (ESR_WT_SERROR(esr)) {
424 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
425 }
426 #endif
427 #endif
428 }
429 #pragma clang diagnostic pop
430 #endif
431
432 static void
arm64_platform_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far)433 arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
434 {
435 #if CONFIG_KERNEL_INTEGRITY
436 kernel_integrity_error_handler(esr, far);
437 #endif
438
439 cpu_data_t *cdp = getCpuDatap();
440
441 if (PE_handle_platform_error(far)) {
442 return;
443 } else if (cdp->platform_error_handler != NULL) {
444 cdp->platform_error_handler(cdp->cpu_id, far);
445 } else {
446 arm64_implementation_specific_error(state, esr, far);
447 }
448 }
449
450 void
panic_with_thread_kernel_state(const char * msg,arm_saved_state_t * ss)451 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
452 {
453 boolean_t ss_valid;
454
455 ss_valid = is_saved_state64(ss);
456 arm_saved_state64_t *state = saved_state64(ss);
457
458 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
459
460 // rdar://80659177
461 // Read SoCD tracepoints up to twice — once the first time we call panic and
462 // another time if we encounter a nested panic after that.
463 static int twice = 2;
464 if (twice > 0) {
465 twice--;
466 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc),
467 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
468 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
469 VALUE(state->far));
470 }
471
472 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
473 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
474 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
475 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
476 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
477 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
478 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
479 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
480 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
481 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
482 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
483 state->x[0], state->x[1], state->x[2], state->x[3],
484 state->x[4], state->x[5], state->x[6], state->x[7],
485 state->x[8], state->x[9], state->x[10], state->x[11],
486 state->x[12], state->x[13], state->x[14], state->x[15],
487 state->x[16], state->x[17], state->x[18], state->x[19],
488 state->x[20], state->x[21], state->x[22], state->x[23],
489 state->x[24], state->x[25], state->x[26], state->x[27],
490 state->x[28], state->fp, state->lr, state->sp,
491 state->pc, state->cpsr, state->esr, state->far);
492 }
493
494 void
sleh_synchronous_sp1(arm_context_t * context,uint32_t esr,vm_offset_t far __unused)495 sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
496 {
497 esr_exception_class_t class = ESR_EC(esr);
498 arm_saved_state_t * state = &context->ss;
499
500 switch (class) {
501 case ESR_EC_UNCATEGORIZED:
502 {
503 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
504 if (IS_ARM_GDB_TRAP(instr)) {
505 DebuggerCall(EXC_BREAKPOINT, state);
506 }
507 }
508 OS_FALLTHROUGH; // panic if we return from the debugger
509 default:
510 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
511 }
512 }
513
514
515 __attribute__((noreturn))
516 void
thread_exception_return()517 thread_exception_return()
518 {
519 thread_t thread = current_thread();
520 if (thread->machine.exception_trace_code != 0) {
521 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
522 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
523 thread->machine.exception_trace_code = 0;
524 }
525
526 arm64_thread_exception_return();
527 __builtin_unreachable();
528 }
529
530 /*
531 * check whether task vtimers are running and set thread and CPU BSD AST
532 *
533 * must be called with interrupts masked so updates of fields are atomic
534 * must be emitted inline to avoid generating an FBT probe on the exception path
535 *
536 */
537 __attribute__((__always_inline__))
538 static inline void
task_vtimer_check(thread_t thread)539 task_vtimer_check(thread_t thread)
540 {
541 task_t task = get_threadtask_early(thread);
542
543 if (__improbable(task != NULL && task->vtimers)) {
544 thread->ast |= AST_BSD;
545 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
546 }
547 }
548
549 #if MACH_ASSERT
550 /**
551 * A version of get_preemption_level() that works in early boot.
552 *
553 * If an exception is raised in early boot before the initial thread has been
554 * set up, then calling get_preemption_level() in the SLEH will trigger an
555 * infinitely-recursing exception. This function handles this edge case.
556 */
557 static inline int
sleh_get_preemption_level(void)558 sleh_get_preemption_level(void)
559 {
560 if (__improbable(current_thread() == NULL)) {
561 return 0;
562 }
563 return get_preemption_level();
564 }
565 #endif // MACH_ASSERT
566
567 void
sleh_synchronous(arm_context_t * context,uint32_t esr,vm_offset_t far)568 sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far)
569 {
570 esr_exception_class_t class = ESR_EC(esr);
571 arm_saved_state_t * state = &context->ss;
572 thread_t thread = current_thread();
573 #if MACH_ASSERT
574 int preemption_level = sleh_get_preemption_level();
575 #endif
576 expected_fault_handler_t expected_fault_handler = NULL;
577 #ifdef CONFIG_XNUPOST
578 expected_fault_handler_t saved_expected_fault_handler = NULL;
579 uintptr_t saved_expected_fault_addr = 0;
580 #endif /* CONFIG_XNUPOST */
581
582 ASSERT_CONTEXT_SANITY(context);
583
584 task_vtimer_check(thread);
585
586 #if CONFIG_DTRACE
587 /*
588 * Handle kernel DTrace probes as early as possible to minimize the likelihood
589 * that this path will itself trigger a DTrace probe, which would lead to infinite
590 * probe recursion.
591 */
592 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
593 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
594 return;
595 }
596 #endif
597 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
598
599 /*
600 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
601 * that would disclose the behavior of PT_DENY_ATTACH processes.
602 */
603 if (is_user) {
604 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
605 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
606 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
607 esr, far, get_saved_state_pc(state), 0, 0);
608 } else {
609 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
610 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
611 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
612 }
613
614 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
615 /*
616 * We no longer support 32-bit, which means no 2-byte
617 * instructions.
618 */
619 if (is_user) {
620 panic("Exception on 2-byte instruction, "
621 "context=%p, esr=%#x, far=%p",
622 context, esr, (void *)far);
623 } else {
624 panic_with_thread_kernel_state("Exception on 2-byte instruction", state);
625 }
626 }
627
628 #ifdef CONFIG_XNUPOST
629 if (thread->machine.expected_fault_handler != NULL) {
630 saved_expected_fault_handler = thread->machine.expected_fault_handler;
631 saved_expected_fault_addr = thread->machine.expected_fault_addr;
632
633 thread->machine.expected_fault_handler = NULL;
634 thread->machine.expected_fault_addr = 0;
635
636 if (saved_expected_fault_addr == far) {
637 expected_fault_handler = saved_expected_fault_handler;
638 }
639 }
640 #endif /* CONFIG_XNUPOST */
641
642 /* Inherit the interrupt masks from previous context */
643 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
644 ml_set_interrupts_enabled(TRUE);
645 }
646
647 switch (class) {
648 case ESR_EC_SVC_64:
649 if (!is_saved_state64(state) || !is_user) {
650 panic("Invalid SVC_64 context");
651 }
652
653 handle_svc(state);
654 break;
655
656 case ESR_EC_DABORT_EL0:
657 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
658 break;
659
660 case ESR_EC_MSR_TRAP:
661 handle_msr_trap(state, esr);
662 break;
663
664 #ifdef __ARM_ARCH_8_6__
665 case ESR_EC_PAC_FAIL:
666 handle_pac_fail(state, esr);
667 __builtin_unreachable();
668
669 #endif /* __ARM_ARCH_8_6__ */
670
671 case ESR_EC_IABORT_EL0:
672 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
673 break;
674
675 case ESR_EC_IABORT_EL1:
676 #ifdef CONFIG_XNUPOST
677 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
678 break;
679 }
680 #endif /* CONFIG_XNUPOST */
681
682 panic_with_thread_kernel_state("Kernel instruction fetch abort", state);
683
684 case ESR_EC_PC_ALIGN:
685 handle_pc_align(state);
686 __builtin_unreachable();
687
688 case ESR_EC_DABORT_EL1:
689 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
690 break;
691
692 case ESR_EC_UNCATEGORIZED:
693 assert(!ESR_ISS(esr));
694
695 handle_uncategorized(&context->ss);
696 break;
697
698 case ESR_EC_SP_ALIGN:
699 handle_sp_align(state);
700 __builtin_unreachable();
701
702 case ESR_EC_BKPT_AARCH32:
703 handle_breakpoint(state, esr);
704 __builtin_unreachable();
705
706 case ESR_EC_BRK_AARCH64:
707 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
708 handle_kernel_breakpoint(state, esr);
709 #if CONFIG_UBSAN_MINIMAL
710 /* UBSan breakpoints are recoverable */
711 break;
712 #endif /* CONFIG_UBSAN_MINIMAL */
713 } else {
714 handle_breakpoint(state, esr);
715 __builtin_unreachable();
716 }
717
718 case ESR_EC_BKPT_REG_MATCH_EL0:
719 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
720 handle_breakpoint(state, esr);
721 }
722 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
723 class, state, class, esr, (void *)far);
724 __builtin_unreachable();
725
726 case ESR_EC_BKPT_REG_MATCH_EL1:
727 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state);
728 __builtin_unreachable();
729
730 case ESR_EC_SW_STEP_DEBUG_EL0:
731 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
732 handle_sw_step_debug(state);
733 }
734 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
735 class, state, class, esr, (void *)far);
736 __builtin_unreachable();
737
738 case ESR_EC_SW_STEP_DEBUG_EL1:
739 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state);
740 __builtin_unreachable();
741
742 case ESR_EC_WATCHPT_MATCH_EL0:
743 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
744 handle_watchpoint(far);
745 }
746 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
747 class, state, class, esr, (void *)far);
748 __builtin_unreachable();
749
750 case ESR_EC_WATCHPT_MATCH_EL1:
751 /*
752 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
753 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
754 */
755 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
756 arm_debug_set(NULL);
757 break; /* return to first level handler */
758 }
759 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
760 class, state, class, esr, (void *)far);
761 __builtin_unreachable();
762
763 case ESR_EC_TRAP_SIMD_FP:
764 handle_simd_trap(state, esr);
765 __builtin_unreachable();
766
767 case ESR_EC_ILLEGAL_INSTR_SET:
768 if (EXCB_ACTION_RERUN !=
769 ex_cb_invoke(EXCB_CLASS_ILLEGAL_INSTR_SET, far)) {
770 // instruction is not re-executed
771 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
772 state, class, esr, (void *)far, get_saved_state_cpsr(state));
773 }
774 // must clear this fault in PSR to re-run
775 mask_saved_state_cpsr(state, 0, PSR64_IL);
776 break;
777
778 case ESR_EC_MCR_MRC_CP15_TRAP:
779 case ESR_EC_MCRR_MRRC_CP15_TRAP:
780 case ESR_EC_MCR_MRC_CP14_TRAP:
781 case ESR_EC_LDC_STC_CP14_TRAP:
782 case ESR_EC_MCRR_MRRC_CP14_TRAP:
783 handle_user_trapped_instruction32(state, esr);
784 __builtin_unreachable();
785
786 case ESR_EC_WFI_WFE:
787 // Use of WFI or WFE instruction when they have been disabled for EL0
788 handle_wf_trap(state);
789 __builtin_unreachable();
790
791 case ESR_EC_FLOATING_POINT_64:
792 handle_fp_trap(state, esr);
793 __builtin_unreachable();
794
795 default:
796 handle_uncategorized(state);
797 }
798
799 #ifdef CONFIG_XNUPOST
800 if (saved_expected_fault_handler != NULL) {
801 thread->machine.expected_fault_handler = saved_expected_fault_handler;
802 thread->machine.expected_fault_addr = saved_expected_fault_addr;
803 }
804 #endif /* CONFIG_XNUPOST */
805
806 if (is_user) {
807 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
808 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
809 esr, far, get_saved_state_pc(state), 0, 0);
810 thread->machine.exception_trace_code = 0;
811 } else {
812 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
813 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
814 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
815 }
816 #if MACH_ASSERT
817 if (preemption_level != sleh_get_preemption_level()) {
818 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
819 }
820 #endif
821 }
822
823 /*
824 * Uncategorized exceptions are a catch-all for general execution errors.
825 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
826 */
827 static void
handle_uncategorized(arm_saved_state_t * state)828 handle_uncategorized(arm_saved_state_t *state)
829 {
830 exception_type_t exception = EXC_BAD_INSTRUCTION;
831 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
832 mach_msg_type_number_t numcodes = 2;
833 uint32_t instr = 0;
834
835 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
836
837 #if CONFIG_DTRACE
838
839 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
840 /*
841 * For a 64bit user process, we care about all 4 bytes of the
842 * instr.
843 */
844 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
845 if (dtrace_user_probe(state) == KERN_SUCCESS) {
846 return;
847 }
848 }
849 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
850 /*
851 * For a 32bit user process, we check for thumb mode, in
852 * which case we only care about a 2 byte instruction length.
853 * For non-thumb mode, we care about all 4 bytes of the instructin.
854 */
855 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
856 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
857 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
858 if (dtrace_user_probe(state) == KERN_SUCCESS) {
859 return;
860 }
861 }
862 } else {
863 if ((instr == FASTTRAP_ARM32_INSTR) ||
864 (instr == FASTTRAP_ARM32_RET_INSTR)) {
865 if (dtrace_user_probe(state) == KERN_SUCCESS) {
866 return;
867 }
868 }
869 }
870 }
871
872 #endif /* CONFIG_DTRACE */
873
874 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
875 if (IS_ARM_GDB_TRAP(instr)) {
876 boolean_t interrupt_state;
877 exception = EXC_BREAKPOINT;
878
879 interrupt_state = ml_set_interrupts_enabled(FALSE);
880
881 /* Save off the context here (so that the debug logic
882 * can see the original state of this thread).
883 */
884 current_thread()->machine.kpcb = state;
885
886 /* Hop into the debugger (typically either due to a
887 * fatal exception, an explicit panic, or a stackshot
888 * request.
889 */
890 DebuggerCall(exception, state);
891
892 current_thread()->machine.kpcb = NULL;
893 (void) ml_set_interrupts_enabled(interrupt_state);
894 return;
895 } else {
896 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
897 }
898 }
899
900 /*
901 * Check for GDB breakpoint via illegal opcode.
902 */
903 if (IS_ARM_GDB_TRAP(instr)) {
904 exception = EXC_BREAKPOINT;
905 codes[0] = EXC_ARM_BREAKPOINT;
906 codes[1] = instr;
907 } else {
908 codes[1] = instr;
909 }
910
911 exception_triage(exception, codes, numcodes);
912 __builtin_unreachable();
913 }
914
915 #if __has_feature(ptrauth_calls)
916 static const uint16_t ptrauth_brk_comment_base = 0xc470;
917
918 static inline bool
brk_comment_is_ptrauth(uint16_t comment)919 brk_comment_is_ptrauth(uint16_t comment)
920 {
921 return comment >= ptrauth_brk_comment_base &&
922 comment <= ptrauth_brk_comment_base + ptrauth_key_asdb;
923 }
924
925 static inline const char *
ptrauth_key_to_string(ptrauth_key key)926 ptrauth_key_to_string(ptrauth_key key)
927 {
928 switch (key) {
929 case ptrauth_key_asia:
930 return "IA";
931 case ptrauth_key_asib:
932 return "IB";
933 case ptrauth_key_asda:
934 return "DA";
935 case ptrauth_key_asdb:
936 return "DB";
937 default:
938 __builtin_unreachable();
939 }
940 }
941 #endif /* __has_feature(ptrauth_calls) */
942
943 #if CONFIG_KERNEL_TBI && KASAN_TBI
944 static inline bool
brk_comment_is_kasan_failure(uint16_t comment)945 brk_comment_is_kasan_failure(uint16_t comment)
946 {
947 return comment >= KASAN_TBI_ESR_BASE &&
948 comment <= KASAN_TBI_ESR_TOP;
949 }
950 #endif /* CONFIG_KERNEL_TBI && KASAN_TBI */
951
952 #if CONFIG_UBSAN_MINIMAL
953 static inline bool
brk_comment_is_ubsan(uint16_t comment)954 brk_comment_is_ubsan(uint16_t comment)
955 {
956 return comment >= UBSAN_MINIMAL_TRAPS_START &&
957 comment < UBSAN_MINIMAL_TRAPS_END;
958 }
959 #endif /* CONFIG_UBSAN_MINIMAL */
960
961 static void
handle_kernel_breakpoint(arm_saved_state_t * state,uint32_t esr)962 handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr)
963 {
964 uint16_t comment = ISS_BRK_COMMENT(esr);
965
966 #if __has_feature(ptrauth_calls)
967 if (brk_comment_is_ptrauth(comment)) {
968 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx"
969 char msg[strlen(MSG_FMT)
970 - strlen("0x%04X") + strlen("0xFFFF")
971 - strlen("%s") + strlen("IA")
972 - strlen("0x%016llx") + strlen("0xFFFFFFFFFFFFFFFF")
973 + 1];
974 ptrauth_key key = (ptrauth_key)(comment - ptrauth_brk_comment_base);
975 const char *key_str = ptrauth_key_to_string(key);
976 snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]);
977
978 panic_with_thread_kernel_state(msg, state);
979 __builtin_unreachable();
980 #undef MSG_FMT
981 }
982 #endif /* __has_feature(ptrauth_calls) */
983
984 #if CONFIG_KERNEL_TBI && KASAN_TBI
985 if (brk_comment_is_kasan_failure(comment)) {
986 kasan_handle_brk_failure(saved_state64(state)->x[0], comment);
987 __builtin_unreachable();
988 }
989 #endif /* CONFIG_KERNEL_TBI && KASAN_TBI */
990
991 #if CONFIG_UBSAN_MINIMAL
992 if (brk_comment_is_ubsan(comment)) {
993 ubsan_handle_brk_trap(comment, get_saved_state_pc(state),
994 get_saved_state_fp(state));
995 add_saved_state_pc(state, 4);
996 return;
997 }
998 #endif /* CONFIG_UBSAN_MINIMAL */
999
1000 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)"
1001 char msg[strlen(MSG_FMT) - strlen("0x%04X") + strlen("0xFFFF") + 1];
1002 snprintf(msg, sizeof(msg), MSG_FMT, comment);
1003 #undef MSG_FMT
1004
1005 panic_with_thread_kernel_state(msg, state);
1006 __builtin_unreachable();
1007 }
1008
1009 static void
handle_breakpoint(arm_saved_state_t * state,uint32_t esr __unused)1010 handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused)
1011 {
1012 exception_type_t exception = EXC_BREAKPOINT;
1013 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1014 mach_msg_type_number_t numcodes = 2;
1015
1016 #if __has_feature(ptrauth_calls) && !__ARM_ARCH_8_6__
1017 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1018 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1019 exception |= EXC_PTRAUTH_BIT;
1020 }
1021 #endif /* __has_feature(ptrauth_calls) && !__ARM_ARCH_8_6__ */
1022
1023 codes[1] = get_saved_state_pc(state);
1024 exception_triage(exception, codes, numcodes);
1025 __builtin_unreachable();
1026 }
1027
1028 static void
handle_watchpoint(vm_offset_t fault_addr)1029 handle_watchpoint(vm_offset_t fault_addr)
1030 {
1031 exception_type_t exception = EXC_BREAKPOINT;
1032 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1033 mach_msg_type_number_t numcodes = 2;
1034
1035 codes[1] = fault_addr;
1036 exception_triage(exception, codes, numcodes);
1037 __builtin_unreachable();
1038 }
1039
1040 static void
handle_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,abort_inspector_t inspect_abort,abort_handler_t handler,expected_fault_handler_t expected_fault_handler)1041 handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1042 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1043 {
1044 fault_status_t fault_code;
1045 vm_prot_t fault_type;
1046
1047 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1048 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1049 }
1050
1051 static void
inspect_instruction_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1052 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1053 {
1054 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1055 *fault_code = ISS_IA_FSC(iss);
1056 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1057 }
1058
1059 static void
inspect_data_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1060 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1061 {
1062 getCpuDatap()->cpu_stat.data_ex_cnt++;
1063 *fault_code = ISS_DA_FSC(iss);
1064
1065 /*
1066 * Cache maintenance operations always report faults as write access.
1067 * Change these to read access, unless they report a permission fault.
1068 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1069 * access to the mapping, but if a cache maintenance operation that only requires
1070 * read access generates a permission fault, then we will not be able to handle
1071 * the fault regardless of whether we treat it as a read or write fault.
1072 */
1073 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
1074 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1075 } else {
1076 *fault_type = (VM_PROT_READ);
1077 }
1078 }
1079
1080 #if __has_feature(ptrauth_calls)
1081 #ifdef __ARM_ARCH_8_6__
1082 static inline uint64_t
fault_addr_bitmask(unsigned int bit_from,unsigned int bit_to)1083 fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1084 {
1085 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1086 }
1087 #else
1088 static inline bool
fault_addr_bit(vm_offset_t fault_addr,unsigned int bit)1089 fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1090 {
1091 return (bool)((fault_addr >> bit) & 1);
1092 }
1093 #endif /* __ARM_ARCH_8_6__ */
1094
1095 /**
1096 * Determines whether a fault address taken at EL0 contains a PAC error code
1097 * corresponding to the specified kind of ptrauth key.
1098 */
1099 static bool
user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr,bool data_key)1100 user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key)
1101 {
1102 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1103 bool tbi = data_key || __improbable(instruction_tbi);
1104 #ifdef __ARM_ARCH_8_6__
1105 /*
1106 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1107 * treat all non-canonical address bits like potential poison bits.
1108 */
1109 uint64_t mask = fault_addr_bitmask(T0SZ_BOOT, 54);
1110 if (!tbi) {
1111 mask |= fault_addr_bitmask(56, 63);
1112 }
1113 return (fault_addr & mask) != 0;
1114 #else /* !__ARM_ARCH_8_6__ */
1115 unsigned int poison_shift;
1116 if (tbi) {
1117 poison_shift = 53;
1118 } else {
1119 poison_shift = 61;
1120 }
1121
1122 /* PAC error codes are always in the form key_number:NOT(key_number) */
1123 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1124 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1125 return poison_bit_1 != poison_bit_2;
1126 #endif /* __ARM_ARCH_8_6__ */
1127 }
1128 #endif /* __has_feature(ptrauth_calls) */
1129
1130 static void
handle_pc_align(arm_saved_state_t * ss)1131 handle_pc_align(arm_saved_state_t *ss)
1132 {
1133 exception_type_t exc;
1134 mach_exception_data_type_t codes[2];
1135 mach_msg_type_number_t numcodes = 2;
1136
1137 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1138 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
1139 }
1140
1141 exc = EXC_BAD_ACCESS;
1142 #if __has_feature(ptrauth_calls)
1143 if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) {
1144 exc |= EXC_PTRAUTH_BIT;
1145 }
1146 #endif /* __has_feature(ptrauth_calls) */
1147
1148 codes[0] = EXC_ARM_DA_ALIGN;
1149 codes[1] = get_saved_state_pc(ss);
1150
1151 exception_triage(exc, codes, numcodes);
1152 __builtin_unreachable();
1153 }
1154
1155 static void
handle_sp_align(arm_saved_state_t * ss)1156 handle_sp_align(arm_saved_state_t *ss)
1157 {
1158 exception_type_t exc;
1159 mach_exception_data_type_t codes[2];
1160 mach_msg_type_number_t numcodes = 2;
1161
1162 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1163 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
1164 }
1165
1166 exc = EXC_BAD_ACCESS;
1167 #if __has_feature(ptrauth_calls)
1168 if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) {
1169 exc |= EXC_PTRAUTH_BIT;
1170 }
1171 #endif /* __has_feature(ptrauth_calls) */
1172
1173 codes[0] = EXC_ARM_SP_ALIGN;
1174 codes[1] = get_saved_state_sp(ss);
1175
1176 exception_triage(exc, codes, numcodes);
1177 __builtin_unreachable();
1178 }
1179
1180 static void
handle_wf_trap(arm_saved_state_t * state)1181 handle_wf_trap(arm_saved_state_t *state)
1182 {
1183 exception_type_t exc;
1184 mach_exception_data_type_t codes[2];
1185 mach_msg_type_number_t numcodes = 2;
1186 uint32_t instr = 0;
1187
1188 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1189
1190 exc = EXC_BAD_INSTRUCTION;
1191 codes[0] = EXC_ARM_UNDEFINED;
1192 codes[1] = instr;
1193
1194 exception_triage(exc, codes, numcodes);
1195 __builtin_unreachable();
1196 }
1197
1198 static void
handle_fp_trap(arm_saved_state_t * state,uint32_t esr)1199 handle_fp_trap(arm_saved_state_t *state, uint32_t esr)
1200 {
1201 exception_type_t exc = EXC_ARITHMETIC;
1202 mach_exception_data_type_t codes[2];
1203 mach_msg_type_number_t numcodes = 2;
1204 uint32_t instr = 0;
1205
1206 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1207 panic_with_thread_kernel_state("Floating point exception from kernel", state);
1208 }
1209
1210 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1211 codes[1] = instr;
1212
1213 /* The floating point trap flags are only valid if TFV is set. */
1214 if (!fp_exceptions_enabled) {
1215 exc = EXC_BAD_INSTRUCTION;
1216 codes[0] = EXC_ARM_UNDEFINED;
1217 } else if (!(esr & ISS_FP_TFV)) {
1218 codes[0] = EXC_ARM_FP_UNDEFINED;
1219 } else if (esr & ISS_FP_UFF) {
1220 codes[0] = EXC_ARM_FP_UF;
1221 } else if (esr & ISS_FP_OFF) {
1222 codes[0] = EXC_ARM_FP_OF;
1223 } else if (esr & ISS_FP_IOF) {
1224 codes[0] = EXC_ARM_FP_IO;
1225 } else if (esr & ISS_FP_DZF) {
1226 codes[0] = EXC_ARM_FP_DZ;
1227 } else if (esr & ISS_FP_IDF) {
1228 codes[0] = EXC_ARM_FP_ID;
1229 } else if (esr & ISS_FP_IXF) {
1230 codes[0] = EXC_ARM_FP_IX;
1231 } else {
1232 panic("Unrecognized floating point exception, state=%p, esr=%#x", state, esr);
1233 }
1234
1235 exception_triage(exc, codes, numcodes);
1236 __builtin_unreachable();
1237 }
1238
1239
1240
1241 /*
1242 * handle_alignment_fault_from_user:
1243 * state: Saved state
1244 *
1245 * Attempts to deal with an alignment fault from userspace (possibly by
1246 * emulating the faulting instruction). If emulation failed due to an
1247 * unservicable fault, the ESR for that fault will be stored in the
1248 * recovery_esr field of the thread by the exception code.
1249 *
1250 * Returns:
1251 * -1: Emulation failed (emulation of state/instr not supported)
1252 * 0: Successfully emulated the instruction
1253 * EFAULT: Emulation failed (probably due to permissions)
1254 * EINVAL: Emulation failed (probably due to a bad address)
1255 */
1256
1257
1258 static int
handle_alignment_fault_from_user(arm_saved_state_t * state,kern_return_t * vmfr)1259 handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1260 {
1261 int ret = -1;
1262
1263 #pragma unused (state)
1264 #pragma unused (vmfr)
1265
1266 return ret;
1267 }
1268
1269
1270 static void
handle_sw_step_debug(arm_saved_state_t * state)1271 handle_sw_step_debug(arm_saved_state_t *state)
1272 {
1273 thread_t thread = current_thread();
1274 exception_type_t exc;
1275 mach_exception_data_type_t codes[2];
1276 mach_msg_type_number_t numcodes = 2;
1277
1278 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1279 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
1280 }
1281
1282 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1283 if (thread->machine.DebugData != NULL) {
1284 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1285 } else {
1286 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
1287 }
1288
1289 mask_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);
1290
1291 // Special encoding for gdb single step event on ARM
1292 exc = EXC_BREAKPOINT;
1293 codes[0] = 1;
1294 codes[1] = 0;
1295
1296 exception_triage(exc, codes, numcodes);
1297 __builtin_unreachable();
1298 }
1299
1300 static void
handle_user_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1301 handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1302 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1303 {
1304 exception_type_t exc = EXC_BAD_ACCESS;
1305 mach_exception_data_type_t codes[2];
1306 mach_msg_type_number_t numcodes = 2;
1307 thread_t thread = current_thread();
1308
1309 (void)esr;
1310 (void)expected_fault_handler;
1311
1312 if (ml_at_interrupt_context()) {
1313 panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state);
1314 }
1315
1316 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1317
1318 if (is_vm_fault(fault_code)) {
1319 vm_map_t map = thread->map;
1320 vm_offset_t vm_fault_addr = fault_addr;
1321 kern_return_t result = KERN_FAILURE;
1322
1323 assert(map != kernel_map);
1324
1325 if (!(fault_type & VM_PROT_EXECUTE)) {
1326 vm_fault_addr = tbi_clear(fault_addr);
1327 }
1328
1329 /* check to see if it is just a pmap ref/modify fault */
1330 if (!is_translation_fault(fault_code)) {
1331 result = arm_fast_fault(map->pmap,
1332 vm_fault_addr,
1333 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1334 }
1335 if (result == KERN_SUCCESS) {
1336 return;
1337 }
1338
1339 {
1340 /* We have to fault the page in */
1341 result = vm_fault(map, vm_fault_addr, fault_type,
1342 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1343 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1344 }
1345 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1346 return;
1347 }
1348
1349 /*
1350 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1351 * If it does, we're leaking preemption disables somewhere in the kernel.
1352 */
1353 if (__improbable(result == KERN_FAILURE)) {
1354 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1355 }
1356
1357 codes[0] = result;
1358 } else if (is_alignment_fault(fault_code)) {
1359 kern_return_t vmfkr = KERN_SUCCESS;
1360 thread->machine.recover_esr = 0;
1361 thread->machine.recover_far = 0;
1362 int result = handle_alignment_fault_from_user(state, &vmfkr);
1363 if (result == 0) {
1364 /* Successfully emulated, or instruction
1365 * copyin() for decode/emulation failed.
1366 * Continue, or redrive instruction.
1367 */
1368 thread_exception_return();
1369 } else if (((result == EFAULT) || (result == EINVAL)) &&
1370 (thread->machine.recover_esr == 0)) {
1371 /*
1372 * If we didn't actually take a fault, but got one of
1373 * these errors, then we failed basic sanity checks of
1374 * the fault address. Treat this as an invalid
1375 * address.
1376 */
1377 codes[0] = KERN_INVALID_ADDRESS;
1378 } else if ((result == EFAULT) &&
1379 (thread->machine.recover_esr)) {
1380 /*
1381 * Since alignment aborts are prioritized
1382 * ahead of translation aborts, the misaligned
1383 * atomic emulation flow may have triggered a
1384 * VM pagefault, which the VM could not resolve.
1385 * Report the VM fault error in codes[]
1386 */
1387
1388 codes[0] = vmfkr;
1389 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1390 /* Cause ESR_EC to reflect an EL0 abort */
1391 thread->machine.recover_esr &= ~ESR_EC_MASK;
1392 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1393 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr);
1394 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far);
1395 fault_addr = thread->machine.recover_far;
1396 } else {
1397 /* This was just an unsupported alignment
1398 * exception. Misaligned atomic emulation
1399 * timeouts fall in this category.
1400 */
1401 codes[0] = EXC_ARM_DA_ALIGN;
1402 }
1403 } else if (is_parity_error(fault_code)) {
1404 #if defined(APPLE_ARM64_ARCH_FAMILY)
1405 if (fault_code == FSC_SYNC_PARITY) {
1406 arm64_platform_error(state, esr, fault_addr);
1407 return;
1408 }
1409 #else
1410 panic("User parity error.");
1411 #endif
1412 } else {
1413 codes[0] = KERN_FAILURE;
1414 }
1415
1416 codes[1] = fault_addr;
1417 #if __has_feature(ptrauth_calls)
1418 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1419 if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) {
1420 exc |= EXC_PTRAUTH_BIT;
1421 }
1422 #endif /* __has_feature(ptrauth_calls) */
1423 exception_triage(exc, codes, numcodes);
1424 __builtin_unreachable();
1425 }
1426
1427 #if __ARM_PAN_AVAILABLE__
1428 static int
is_pan_fault(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code)1429 is_pan_fault(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr, fault_status_t fault_code)
1430 {
1431 // PAN (Privileged Access Never) fault occurs for data read/write in EL1 to
1432 // virtual address that is readable/writeable from both EL1 and EL0
1433
1434 // To check for PAN fault, we evaluate if the following conditions are true:
1435 // 1. This is a permission fault
1436 // 2. PAN is enabled
1437 // 3. AT instruction (on which PAN has no effect) on the same faulting address
1438 // succeeds
1439
1440 vm_offset_t pa;
1441
1442 if (!(is_permission_fault(fault_code) && get_saved_state_cpsr(state) & PSR64_PAN)) {
1443 return FALSE;
1444 }
1445
1446 if (esr & ISS_DA_WNR) {
1447 pa = mmu_kvtop_wpreflight(fault_addr);
1448 } else {
1449 pa = mmu_kvtop(fault_addr);
1450 }
1451 return (pa)? TRUE: FALSE;
1452 }
1453 #endif
1454
1455 static void
handle_kernel_abort_recover(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,thread_t thread)1456 handle_kernel_abort_recover(
1457 arm_saved_state_t *state,
1458 uint32_t esr,
1459 vm_offset_t fault_addr,
1460 thread_t thread)
1461 {
1462 thread->machine.recover_esr = esr;
1463 thread->machine.recover_far = fault_addr;
1464 #if defined(HAS_APPLE_PAC)
1465 MANIPULATE_SIGNED_THREAD_STATE(state,
1466 "mov x1, %[pc] \n"
1467 "str x1, [x0, %[SS64_PC]] \n",
1468 [pc] "r"(copyio_recovery_get_recover_addr(state))
1469 );
1470 #else
1471 saved_state64(state)->pc = copyio_recovery_get_recover_addr(state);
1472 #endif
1473 }
1474
1475 static void
handle_kernel_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1476 handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1477 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1478 {
1479 thread_t thread = current_thread();
1480 bool recover = find_copyio_recovery_entry(state) != 0;
1481
1482 #ifndef CONFIG_XNUPOST
1483 (void)expected_fault_handler;
1484 #endif /* CONFIG_XNUPOST */
1485
1486 #if CONFIG_DTRACE
1487 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
1488 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1489 /*
1490 * Point to next instruction, or recovery handler if set.
1491 */
1492 if (recover) {
1493 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1494 } else {
1495 add_saved_state_pc(state, 4);
1496 }
1497 return;
1498 } else {
1499 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
1500 }
1501 }
1502 #endif
1503
1504 if (ml_at_interrupt_context()) {
1505 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1506 }
1507
1508 if (is_vm_fault(fault_code)) {
1509 kern_return_t result = KERN_FAILURE;
1510 vm_map_t map;
1511 int interruptible;
1512
1513 /*
1514 * Ensure no faults in the physical aperture. This could happen if
1515 * a page table is incorrectly allocated from the read only region
1516 * when running with KTRR.
1517 */
1518
1519 #ifdef CONFIG_XNUPOST
1520 if (expected_fault_handler && expected_fault_handler(state)) {
1521 return;
1522 }
1523 #endif /* CONFIG_XNUPOST */
1524
1525 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1526 panic_with_thread_kernel_state("Unexpected fault in kernel static region\n", state);
1527 }
1528
1529 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL) {
1530 map = kernel_map;
1531 interruptible = THREAD_UNINT;
1532 } else {
1533 map = thread->map;
1534
1535 /**
1536 * In the case that the recovery handler is set (e.g., during copyio
1537 * and dtrace probes), we don't want the vm_fault() operation to be
1538 * aborted early. Those code paths can't handle restarting the
1539 * vm_fault() operation so don't allow it to return early without
1540 * creating the wanted mapping.
1541 */
1542 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
1543 }
1544
1545 /* check to see if it is just a pmap ref/modify fault */
1546 if (!is_translation_fault(fault_code)) {
1547 result = arm_fast_fault(map->pmap,
1548 fault_addr,
1549 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
1550 if (result == KERN_SUCCESS) {
1551 return;
1552 }
1553 }
1554
1555 if (result != KERN_PROTECTION_FAILURE) {
1556 /*
1557 * We have to "fault" the page in.
1558 */
1559 result = vm_fault(map, fault_addr, fault_type,
1560 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1561 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1562 }
1563
1564 if (result == KERN_SUCCESS) {
1565 return;
1566 }
1567
1568 /*
1569 * If we have a recover handler, invoke it now.
1570 */
1571 if (recover) {
1572 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1573 return;
1574 }
1575
1576 #if __ARM_PAN_AVAILABLE__
1577 if (is_pan_fault(state, esr, fault_addr, fault_code)) {
1578 panic_with_thread_kernel_state("Privileged access never abort.", state);
1579 }
1580 #endif
1581 } else if (is_alignment_fault(fault_code)) {
1582 if (recover) {
1583 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1584 return;
1585 }
1586 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
1587 } else if (is_parity_error(fault_code)) {
1588 #if defined(APPLE_ARM64_ARCH_FAMILY)
1589 if (fault_code == FSC_SYNC_PARITY) {
1590 arm64_platform_error(state, esr, fault_addr);
1591 return;
1592 }
1593 #else
1594 panic_with_thread_kernel_state("Kernel parity error.", state);
1595 #endif
1596 } else {
1597 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1598 }
1599
1600 panic_with_thread_kernel_state("Kernel data abort.", state);
1601 }
1602
1603 extern void syscall_trace(struct arm_saved_state * regs);
1604
1605 static void
handle_svc(arm_saved_state_t * state)1606 handle_svc(arm_saved_state_t *state)
1607 {
1608 int trap_no = get_saved_state_svc_number(state);
1609 thread_t thread = current_thread();
1610 struct proc *p;
1611
1612 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1613
1614 #define TRACE_SYSCALL 1
1615 #if TRACE_SYSCALL
1616 syscall_trace(state);
1617 #endif
1618
1619 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1620
1621 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1622 platform_syscall(state);
1623 panic("Returned from platform_syscall()?");
1624 }
1625
1626 mach_kauth_cred_thread_update();
1627
1628 if (trap_no < 0) {
1629 switch (trap_no) {
1630 case MACH_ARM_TRAP_ABSTIME:
1631 handle_mach_absolute_time_trap(state);
1632 return;
1633 case MACH_ARM_TRAP_CONTTIME:
1634 handle_mach_continuous_time_trap(state);
1635 return;
1636 }
1637
1638 /* Counting perhaps better in the handler, but this is how it's been done */
1639 thread->syscalls_mach++;
1640 mach_syscall(state);
1641 } else {
1642 /* Counting perhaps better in the handler, but this is how it's been done */
1643 thread->syscalls_unix++;
1644 p = get_bsdthreadtask_info(thread);
1645
1646 assert(p);
1647
1648 unix_syscall(state, thread, p);
1649 }
1650 }
1651
1652 static void
handle_mach_absolute_time_trap(arm_saved_state_t * state)1653 handle_mach_absolute_time_trap(arm_saved_state_t *state)
1654 {
1655 uint64_t now = mach_absolute_time();
1656 saved_state64(state)->x[0] = now;
1657 }
1658
1659 static void
handle_mach_continuous_time_trap(arm_saved_state_t * state)1660 handle_mach_continuous_time_trap(arm_saved_state_t *state)
1661 {
1662 uint64_t now = mach_continuous_time();
1663 saved_state64(state)->x[0] = now;
1664 }
1665
1666
1667 __attribute__((noreturn))
1668 static void
handle_msr_trap(arm_saved_state_t * state,uint32_t esr)1669 handle_msr_trap(arm_saved_state_t *state, uint32_t esr)
1670 {
1671 exception_type_t exception = EXC_BAD_INSTRUCTION;
1672 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1673 mach_msg_type_number_t numcodes = 2;
1674 uint32_t instr = 0;
1675
1676 if (!is_saved_state64(state)) {
1677 panic("MSR/MRS trap (ESR 0x%x) from 32-bit state", esr);
1678 }
1679
1680 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1681 panic("MSR/MRS trap (ESR 0x%x) from kernel", esr);
1682 }
1683
1684 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1685 codes[1] = instr;
1686
1687 exception_triage(exception, codes, numcodes);
1688 __builtin_unreachable();
1689 }
1690
1691 #ifdef __ARM_ARCH_8_6__
1692 static void
autxx_instruction_extract_reg(uint32_t instr,char reg[4])1693 autxx_instruction_extract_reg(uint32_t instr, char reg[4])
1694 {
1695 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
1696 switch (rd) {
1697 case 29:
1698 strncpy(reg, "fp", 4);
1699 return;
1700
1701 case 30:
1702 strncpy(reg, "lr", 4);
1703 return;
1704
1705 case 31:
1706 strncpy(reg, "xzr", 4);
1707 return;
1708
1709 default:
1710 snprintf(reg, 4, "x%u", rd);
1711 return;
1712 }
1713 }
1714
1715 static const char *
autix_system_instruction_extract_reg(uint32_t instr)1716 autix_system_instruction_extract_reg(uint32_t instr)
1717 {
1718 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
1719 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
1720 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
1721 return "x17";
1722 } else {
1723 return "lr";
1724 }
1725 }
1726
1727 static void
handle_pac_fail(arm_saved_state_t * state,uint32_t esr)1728 handle_pac_fail(arm_saved_state_t *state, uint32_t esr)
1729 {
1730 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
1731 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
1732 mach_msg_type_number_t numcodes = 2;
1733 uint32_t instr = 0;
1734
1735 if (!is_saved_state64(state)) {
1736 panic("PAC failure (ESR 0x%x) from 32-bit state", esr);
1737 }
1738
1739 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1740
1741 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1742 #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
1743 #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
1744 #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
1745
1746 char msg[strlen(AUTXX_MSG_FMT)
1747 - strlen("%s") + strlen("IA")
1748 - strlen("%s") + strlen("xzr")
1749 + 1];
1750 ptrauth_key key = (ptrauth_key)(esr & 0x3);
1751 const char *key_str = ptrauth_key_to_string(key);
1752
1753 if (ARM64_INSTR_IS_AUTxx(instr)) {
1754 char reg[4];
1755 autxx_instruction_extract_reg(instr, reg);
1756 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1757 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
1758 const char *reg = autix_system_instruction_extract_reg(instr);
1759 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1760 } else {
1761 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
1762 }
1763 panic_with_thread_kernel_state(msg, state);
1764 }
1765
1766 codes[1] = instr;
1767
1768 exception_triage(exception, codes, numcodes);
1769 __builtin_unreachable();
1770 }
1771 #endif /* __ARM_ARCH_8_6__ */
1772
1773 static void
handle_user_trapped_instruction32(arm_saved_state_t * state,uint32_t esr)1774 handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
1775 {
1776 exception_type_t exception = EXC_BAD_INSTRUCTION;
1777 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1778 mach_msg_type_number_t numcodes = 2;
1779 uint32_t instr;
1780
1781 if (is_saved_state64(state)) {
1782 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
1783 }
1784
1785 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1786 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
1787 }
1788
1789 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1790 codes[1] = instr;
1791
1792 exception_triage(exception, codes, numcodes);
1793 __builtin_unreachable();
1794 }
1795
1796 static void
handle_simd_trap(arm_saved_state_t * state,uint32_t esr)1797 handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
1798 {
1799 exception_type_t exception = EXC_BAD_INSTRUCTION;
1800 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1801 mach_msg_type_number_t numcodes = 2;
1802 uint32_t instr = 0;
1803
1804 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1805 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
1806 }
1807
1808 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1809 codes[1] = instr;
1810
1811 exception_triage(exception, codes, numcodes);
1812 __builtin_unreachable();
1813 }
1814
1815 void
sleh_irq(arm_saved_state_t * state)1816 sleh_irq(arm_saved_state_t *state)
1817 {
1818 cpu_data_t * cdp __unused = getCpuDatap();
1819 #if MACH_ASSERT
1820 int preemption_level = sleh_get_preemption_level();
1821 #endif
1822
1823
1824 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
1825
1826 #if USE_APPLEARMSMP
1827 PE_handle_ext_interrupt();
1828 #else
1829 /* Run the registered interrupt handler. */
1830 cdp->interrupt_handler(cdp->interrupt_target,
1831 cdp->interrupt_refCon,
1832 cdp->interrupt_nub,
1833 cdp->interrupt_source);
1834 #endif
1835
1836 entropy_collect();
1837
1838
1839 sleh_interrupt_handler_epilogue();
1840 #if MACH_ASSERT
1841 if (preemption_level != sleh_get_preemption_level()) {
1842 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
1843 }
1844 #endif
1845 }
1846
1847 void
sleh_fiq(arm_saved_state_t * state)1848 sleh_fiq(arm_saved_state_t *state)
1849 {
1850 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
1851 #if MACH_ASSERT
1852 int preemption_level = sleh_get_preemption_level();
1853 #endif
1854
1855 #if MONOTONIC_FIQ
1856 uint64_t pmcr0 = 0, upmsr = 0;
1857 #endif /* MONOTONIC_FIQ */
1858
1859 #if defined(HAS_IPI)
1860 boolean_t is_ipi = FALSE;
1861 uint64_t ipi_sr = 0;
1862
1863 if (gFastIPI) {
1864 MRS(ipi_sr, "S3_5_C15_C1_1");
1865
1866 if (ipi_sr & 1) {
1867 is_ipi = TRUE;
1868 }
1869 }
1870
1871 if (is_ipi) {
1872 type = DBG_INTR_TYPE_IPI;
1873 } else
1874 #endif /* defined(HAS_IPI) */
1875 if (ml_get_timer_pending()) {
1876 type = DBG_INTR_TYPE_TIMER;
1877 }
1878 #if MONOTONIC_FIQ
1879 /* Consult the PMI sysregs last, after IPI/timer
1880 * classification.
1881 */
1882 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
1883 type = DBG_INTR_TYPE_PMI;
1884 }
1885 #endif /* MONOTONIC_FIQ */
1886
1887 sleh_interrupt_handler_prologue(state, type);
1888
1889 #if APPLEVIRTUALPLATFORM
1890 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
1891 #endif
1892
1893 #if defined(HAS_IPI)
1894 if (is_ipi) {
1895 /*
1896 * Order is important here: we must ack the IPI by writing IPI_SR
1897 * before we call cpu_signal_handler(). Otherwise, there will be
1898 * a window between the completion of pending-signal processing in
1899 * cpu_signal_handler() and the ack during which a newly-issued
1900 * IPI to this CPU may be lost. ISB is required to ensure the msr
1901 * is retired before execution of cpu_signal_handler().
1902 */
1903 MSR("S3_5_C15_C1_1", ipi_sr);
1904 __builtin_arm_isb(ISB_SY);
1905 cpu_signal_handler();
1906 } else
1907 #endif /* defined(HAS_IPI) */
1908 #if MONOTONIC_FIQ
1909 if (type == DBG_INTR_TYPE_PMI) {
1910 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
1911 mt_fiq(getCpuDatap(), pmcr0, upmsr);
1912 INTERRUPT_MASKED_DEBUG_END();
1913 } else
1914 #endif /* MONOTONIC_FIQ */
1915 {
1916 /*
1917 * We don't know that this is a timer, but we don't have insight into
1918 * the other interrupts that go down this path.
1919 */
1920
1921 cpu_data_t *cdp = getCpuDatap();
1922
1923 cdp->cpu_decrementer = -1; /* Large */
1924
1925 /*
1926 * ARM64_TODO: whether we're coming from userland is ignored right now.
1927 * We can easily thread it through, but not bothering for the
1928 * moment (AArch32 doesn't either).
1929 */
1930 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
1931 rtclock_intr(TRUE);
1932 INTERRUPT_MASKED_DEBUG_END();
1933 }
1934
1935 #if APPLEVIRTUALPLATFORM
1936 if (iar != GIC_SPURIOUS_IRQ) {
1937 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
1938 __builtin_arm_isb(ISB_SY);
1939 }
1940 #endif
1941
1942 sleh_interrupt_handler_epilogue();
1943 #if MACH_ASSERT
1944 if (preemption_level != sleh_get_preemption_level()) {
1945 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
1946 }
1947 #endif
1948 }
1949
1950 void
sleh_serror(arm_context_t * context,uint32_t esr,vm_offset_t far)1951 sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
1952 {
1953 task_vtimer_check(current_thread());
1954
1955 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
1956 esr, VM_KERNEL_ADDRHIDE(far));
1957 arm_saved_state_t *state = &context->ss;
1958 #if MACH_ASSERT
1959 int preemption_level = sleh_get_preemption_level();
1960 #endif
1961
1962
1963 ASSERT_CONTEXT_SANITY(context);
1964 arm64_platform_error(state, esr, far);
1965 #if MACH_ASSERT
1966 if (preemption_level != sleh_get_preemption_level()) {
1967 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
1968 }
1969 #endif
1970 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
1971 esr, VM_KERNEL_ADDRHIDE(far));
1972 }
1973
1974 void
mach_syscall_trace_exit(unsigned int retval,unsigned int call_number)1975 mach_syscall_trace_exit(unsigned int retval,
1976 unsigned int call_number)
1977 {
1978 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
1979 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
1980 DBG_FUNC_END, retval, 0, 0, 0, 0);
1981 }
1982
1983 __attribute__((noreturn))
1984 void
thread_syscall_return(kern_return_t error)1985 thread_syscall_return(kern_return_t error)
1986 {
1987 thread_t thread;
1988 struct arm_saved_state *state;
1989
1990 thread = current_thread();
1991 state = get_user_regs(thread);
1992
1993 assert(is_saved_state64(state));
1994 saved_state64(state)->x[0] = error;
1995
1996 #if MACH_ASSERT
1997 kern_allocation_name_t
1998 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
1999 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2000 #endif /* MACH_ASSERT */
2001
2002 if (kdebug_enable) {
2003 /* Invert syscall number (negative for a mach syscall) */
2004 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
2005 }
2006
2007 thread_exception_return();
2008 }
2009
2010 void
syscall_trace(struct arm_saved_state * regs __unused)2011 syscall_trace(
2012 struct arm_saved_state * regs __unused)
2013 {
2014 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2015 }
2016
2017 static void
sleh_interrupt_handler_prologue(arm_saved_state_t * state,unsigned int type)2018 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2019 {
2020 boolean_t is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2021
2022 task_vtimer_check(current_thread());
2023
2024 uint64_t pc = is_user ? get_saved_state_pc(state) :
2025 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2026
2027 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2028 0, pc, is_user, type);
2029
2030 #if CONFIG_TELEMETRY
2031 if (telemetry_needs_record) {
2032 telemetry_mark_curthread(is_user, FALSE);
2033 }
2034 #endif /* CONFIG_TELEMETRY */
2035 }
2036
2037 static void
sleh_interrupt_handler_epilogue(void)2038 sleh_interrupt_handler_epilogue(void)
2039 {
2040 #if KPERF
2041 kperf_interrupt();
2042 #endif /* KPERF */
2043 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2044 }
2045
2046 void
sleh_invalid_stack(arm_context_t * context,uint32_t esr __unused,vm_offset_t far __unused)2047 sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
2048 {
2049 thread_t thread = current_thread();
2050 vm_offset_t kernel_stack_bottom, sp;
2051
2052 sp = get_saved_state_sp(&context->ss);
2053 kernel_stack_bottom = round_page(thread->machine.kstackptr) - KERNEL_STACK_SIZE;
2054
2055 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2056 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
2057 }
2058
2059 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
2060 }
2061
2062