1 /*
2 * Copyright (c) 2012-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39 #include <arm64/instructions.h>
40
41 #include <kern/debug.h>
42 #include <kern/restartable.h>
43 #include <kern/socd_client.h>
44 #include <kern/thread.h>
45 #include <mach/exception.h>
46 #include <mach/arm/traps.h>
47 #include <mach/vm_types.h>
48 #include <mach/machine/thread_status.h>
49
50 #include <machine/atomic.h>
51 #include <machine/limits.h>
52
53 #include <pexpert/arm/protos.h>
54 #include <pexpert/arm64/apple_arm64_regs.h>
55 #include <pexpert/arm64/board_config.h>
56
57 #include <vm/vm_page.h>
58 #include <vm/pmap.h>
59 #include <vm/vm_fault.h>
60 #include <vm/vm_kern.h>
61
62 #include <sys/errno.h>
63 #include <sys/kdebug.h>
64 #include <kperf/kperf.h>
65
66 #include <kern/policy_internal.h>
67 #if CONFIG_TELEMETRY
68 #include <kern/telemetry.h>
69 #endif
70
71 #include <prng/entropy.h>
72
73
74
75
76 #include <arm64/platform_error_handler.h>
77
78 #if KASAN_TBI
79 #include <san/kasan.h>
80 #endif /* KASAN_TBI */
81
82 #if CONFIG_UBSAN_MINIMAL
83 #include <san/ubsan_minimal.h>
84 #endif /* CONFIG_UBSAN_MINIMAL */
85
86
87 #ifndef __arm64__
88 #error Should only be compiling for arm64.
89 #endif
90
91 #define TEST_CONTEXT32_SANITY(context) \
92 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
93 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
94
95 #define TEST_CONTEXT64_SANITY(context) \
96 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
97 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
98
99 #define ASSERT_CONTEXT_SANITY(context) \
100 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
101
102
103 #define COPYIN(src, dst, size) \
104 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
105 copyin_kern(src, dst, size) : \
106 copyin(src, dst, size)
107
108 #define COPYOUT(src, dst, size) \
109 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
110 copyout_kern(src, dst, size) : \
111 copyout(src, dst, size)
112
113 // Below is for concatenating a string param to a string literal
114 #define STR1(x) #x
115 #define STR(x) STR1(x)
116
117 #define ARM64_KDBG_CODE_KERNEL (0 << 8)
118 #define ARM64_KDBG_CODE_USER (1 << 8)
119 #define ARM64_KDBG_CODE_GUEST (2 << 8)
120
121 _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
122 _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
123
124 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
125
126 void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike;
127 void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t);
128
129
130
131 void sleh_irq(arm_saved_state_t *);
132 void sleh_fiq(arm_saved_state_t *);
133 void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
134 void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2;
135
136 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
137 static void sleh_interrupt_handler_epilogue(void);
138
139 static void handle_svc(arm_saved_state_t *);
140 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
141 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
142
143 static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr);
144 #if __has_feature(ptrauth_calls)
145 static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2;
146 #endif
147
148 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
149
150 static void handle_uncategorized(arm_saved_state_t *);
151
152 /*
153 * For UBSan trap and continue handling, we must be able to recover
154 * from handle_kernel_breakpoint().
155 */
156 #if !CONFIG_UBSAN_MINIMAL
157 __dead2
158 #endif /* CONFIG_UBSAN_MINIMAL */
159 static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t);
160
161 static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2;
162
163 typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
164 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
165 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
166
167 static int is_vm_fault(fault_status_t);
168 static int is_translation_fault(fault_status_t);
169 static int is_alignment_fault(fault_status_t);
170
171 typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
172 static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
173 static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
174
175 static void handle_pc_align(arm_saved_state_t *ss) __dead2;
176 static void handle_sp_align(arm_saved_state_t *ss) __dead2;
177 static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
178 static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
179 static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2;
180
181 static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
182
183 static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
184
185 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2;
186
187 static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2;
188
189 extern void mach_kauth_cred_thread_update(void);
190 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
191
192 struct proc;
193
194 typedef uint32_t arm64_instr_t;
195
196 extern void
197 unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
198
199 extern void
200 mach_syscall(struct arm_saved_state*);
201
202 #if CONFIG_DTRACE
203 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
204 extern boolean_t dtrace_tally_fault(user_addr_t);
205
206 /*
207 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
208 * and paste the trap instructions
209 * over from that file. Need to keep these in sync!
210 */
211 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
212 #define FASTTRAP_THUMB32_INSTR 0xdefc
213 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
214
215 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
216 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
217 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
218
219 /* See <rdar://problem/4613924> */
220 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
221 #endif
222
223
224
225 extern void arm64_thread_exception_return(void) __dead2;
226
227 #if defined(APPLETYPHOON)
228 #define CPU_NAME "Typhoon"
229 #elif defined(APPLETWISTER)
230 #define CPU_NAME "Twister"
231 #elif defined(APPLEHURRICANE)
232 #define CPU_NAME "Hurricane"
233 #elif defined(APPLELIGHTNING)
234 #define CPU_NAME "Lightning"
235 #else
236 #define CPU_NAME "Unknown"
237 #endif
238
239 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
240 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
241 #define ESR_WT_REASON(esr) ((esr) & 0xff)
242
243 #define WT_REASON_NONE 0
244 #define WT_REASON_INTEGRITY_FAIL 1
245 #define WT_REASON_BAD_SYSCALL 2
246 #define WT_REASON_NOT_LOCKED 3
247 #define WT_REASON_ALREADY_LOCKED 4
248 #define WT_REASON_SW_REQ 5
249 #define WT_REASON_PT_INVALID 6
250 #define WT_REASON_PT_VIOLATION 7
251 #define WT_REASON_REG_VIOLATION 8
252 #endif
253
254 #if defined(HAS_IPI)
255 void cpu_signal_handler(void);
256 extern unsigned int gFastIPI;
257 #endif /* defined(HAS_IPI) */
258
259 static arm_saved_state64_t *original_faulting_state = NULL;
260
261 TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
262
263 extern vm_offset_t static_memory_end;
264
265 /*
266 * Fault copyio_recovery_entry in copyin/copyout routines.
267 *
268 * Offets are expressed in bytes from ©_recovery_table
269 */
270 struct copyio_recovery_entry {
271 ptrdiff_t cre_start;
272 ptrdiff_t cre_end;
273 ptrdiff_t cre_recovery;
274 };
275
276 extern struct copyio_recovery_entry copyio_recover_table[];
277 extern struct copyio_recovery_entry copyio_recover_table_end[];
278
279 static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)280 copyio_recovery_offset(uintptr_t addr)
281 {
282 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
283 }
284
285 static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)286 copyio_recovery_addr(ptrdiff_t offset)
287 {
288 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
289 }
290
291 static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(arm_saved_state_t * state)292 find_copyio_recovery_entry(arm_saved_state_t *state)
293 {
294 ptrdiff_t offset = copyio_recovery_offset(get_saved_state_pc(state));
295 struct copyio_recovery_entry *e;
296
297 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
298 if (offset >= e->cre_start && offset < e->cre_end) {
299 return e;
300 }
301 }
302
303 return NULL;
304 }
305
306 static inline uintptr_t
copyio_recovery_get_recover_addr(arm_saved_state_t * state)307 copyio_recovery_get_recover_addr(
308 arm_saved_state_t *state)
309 {
310 struct copyio_recovery_entry *e = find_copyio_recovery_entry(state);
311 if (e == NULL) {
312 panic("copyio recovery: couldn't find a range for %p",
313 (void *)get_saved_state_pc(state));
314 }
315 return copyio_recovery_addr(e->cre_recovery);
316 }
317
318 static inline int
is_vm_fault(fault_status_t status)319 is_vm_fault(fault_status_t status)
320 {
321 switch (status) {
322 case FSC_TRANSLATION_FAULT_L0:
323 case FSC_TRANSLATION_FAULT_L1:
324 case FSC_TRANSLATION_FAULT_L2:
325 case FSC_TRANSLATION_FAULT_L3:
326 case FSC_ACCESS_FLAG_FAULT_L1:
327 case FSC_ACCESS_FLAG_FAULT_L2:
328 case FSC_ACCESS_FLAG_FAULT_L3:
329 case FSC_PERMISSION_FAULT_L1:
330 case FSC_PERMISSION_FAULT_L2:
331 case FSC_PERMISSION_FAULT_L3:
332 return TRUE;
333 default:
334 return FALSE;
335 }
336 }
337
338 static inline int
is_translation_fault(fault_status_t status)339 is_translation_fault(fault_status_t status)
340 {
341 switch (status) {
342 case FSC_TRANSLATION_FAULT_L0:
343 case FSC_TRANSLATION_FAULT_L1:
344 case FSC_TRANSLATION_FAULT_L2:
345 case FSC_TRANSLATION_FAULT_L3:
346 return TRUE;
347 default:
348 return FALSE;
349 }
350 }
351
352 static inline int
is_permission_fault(fault_status_t status)353 is_permission_fault(fault_status_t status)
354 {
355 switch (status) {
356 case FSC_PERMISSION_FAULT_L1:
357 case FSC_PERMISSION_FAULT_L2:
358 case FSC_PERMISSION_FAULT_L3:
359 return TRUE;
360 default:
361 return FALSE;
362 }
363 }
364
365 static inline int
is_alignment_fault(fault_status_t status)366 is_alignment_fault(fault_status_t status)
367 {
368 return status == FSC_ALIGNMENT_FAULT;
369 }
370
371 static inline int
is_parity_error(fault_status_t status)372 is_parity_error(fault_status_t status)
373 {
374 switch (status) {
375 /*
376 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
377 * only applicable to AArch32 HSR register. Can this be removed?
378 */
379 case FSC_ASYNC_PARITY:
380 case FSC_SYNC_PARITY:
381 case FSC_SYNC_PARITY_TT_L1:
382 case FSC_SYNC_PARITY_TT_L2:
383 case FSC_SYNC_PARITY_TT_L3:
384 return TRUE;
385 default:
386 return FALSE;
387 }
388 }
389
390
391 __dead2 __unused
392 static void
arm64_implementation_specific_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far)393 arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
394 {
395 #pragma unused (state, esr, far)
396 panic_plain("Unhandled implementation specific error\n");
397 }
398
399 #if CONFIG_KERNEL_INTEGRITY
400 #pragma clang diagnostic push
401 #pragma clang diagnostic ignored "-Wunused-parameter"
402 static void
kernel_integrity_error_handler(uint32_t esr,vm_offset_t far)403 kernel_integrity_error_handler(uint32_t esr, vm_offset_t far)
404 {
405 #if defined(KERNEL_INTEGRITY_WT)
406 #if (DEVELOPMENT || DEBUG)
407 if (ESR_WT_SERROR(esr)) {
408 switch (ESR_WT_REASON(esr)) {
409 case WT_REASON_INTEGRITY_FAIL:
410 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
411 case WT_REASON_BAD_SYSCALL:
412 panic_plain("Kernel integrity, bad syscall.");
413 case WT_REASON_NOT_LOCKED:
414 panic_plain("Kernel integrity, not locked.");
415 case WT_REASON_ALREADY_LOCKED:
416 panic_plain("Kernel integrity, already locked.");
417 case WT_REASON_SW_REQ:
418 panic_plain("Kernel integrity, software request.");
419 case WT_REASON_PT_INVALID:
420 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
421 "walking 0x%016lx.", far);
422 case WT_REASON_PT_VIOLATION:
423 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
424 far);
425 case WT_REASON_REG_VIOLATION:
426 panic_plain("Kernel integrity, violation in system register %d.",
427 (unsigned) far);
428 default:
429 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
430 }
431 }
432 #else
433 if (ESR_WT_SERROR(esr)) {
434 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
435 }
436 #endif
437 #endif
438 }
439 #pragma clang diagnostic pop
440 #endif
441
442 static void
arm64_platform_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far,platform_error_source_t source)443 arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far, platform_error_source_t source)
444 {
445 #if CONFIG_KERNEL_INTEGRITY
446 kernel_integrity_error_handler(esr, far);
447 #endif
448
449 (void)source;
450 cpu_data_t *cdp = getCpuDatap();
451
452 if (PE_handle_platform_error(far)) {
453 return;
454 } else if (cdp->platform_error_handler != NULL) {
455 cdp->platform_error_handler(cdp->cpu_id, far);
456 } else {
457 arm64_implementation_specific_error(state, esr, far);
458 }
459 }
460
461 void
panic_with_thread_kernel_state(const char * msg,arm_saved_state_t * ss)462 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
463 {
464 boolean_t ss_valid;
465
466 ss_valid = is_saved_state64(ss);
467 arm_saved_state64_t *state = saved_state64(ss);
468
469 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
470
471 // rdar://80659177
472 // Read SoCD tracepoints up to twice — once the first time we call panic and
473 // another time if we encounter a nested panic after that.
474 static int twice = 2;
475 if (twice > 0) {
476 twice--;
477 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc),
478 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
479 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
480 VALUE(state->far));
481 }
482
483 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
484 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
485 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
486 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
487 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
488 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
489 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
490 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
491 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
492 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
493 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
494 state->x[0], state->x[1], state->x[2], state->x[3],
495 state->x[4], state->x[5], state->x[6], state->x[7],
496 state->x[8], state->x[9], state->x[10], state->x[11],
497 state->x[12], state->x[13], state->x[14], state->x[15],
498 state->x[16], state->x[17], state->x[18], state->x[19],
499 state->x[20], state->x[21], state->x[22], state->x[23],
500 state->x[24], state->x[25], state->x[26], state->x[27],
501 state->x[28], state->fp, state->lr, state->sp,
502 state->pc, state->cpsr, state->esr, state->far);
503 }
504
505 void
sleh_synchronous_sp1(arm_context_t * context,uint32_t esr,vm_offset_t far __unused)506 sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
507 {
508 esr_exception_class_t class = ESR_EC(esr);
509 arm_saved_state_t * state = &context->ss;
510
511 switch (class) {
512 case ESR_EC_UNCATEGORIZED:
513 {
514 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
515 if (IS_ARM_GDB_TRAP(instr)) {
516 DebuggerCall(EXC_BREAKPOINT, state);
517 }
518 }
519 OS_FALLTHROUGH; // panic if we return from the debugger
520 default:
521 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
522 }
523 }
524
525
526 __attribute__((noreturn))
527 void
thread_exception_return()528 thread_exception_return()
529 {
530 thread_t thread = current_thread();
531 if (thread->machine.exception_trace_code != 0) {
532 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
533 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
534 thread->machine.exception_trace_code = 0;
535 }
536
537 #if KASAN_TBI
538 kasan_unpoison_curstack(true);
539 #endif /* KASAN_TBI */
540 arm64_thread_exception_return();
541 __builtin_unreachable();
542 }
543
544 /*
545 * check whether task vtimers are running and set thread and CPU BSD AST
546 *
547 * must be called with interrupts masked so updates of fields are atomic
548 * must be emitted inline to avoid generating an FBT probe on the exception path
549 *
550 */
551 __attribute__((__always_inline__))
552 static inline void
task_vtimer_check(thread_t thread)553 task_vtimer_check(thread_t thread)
554 {
555 task_t task = get_threadtask_early(thread);
556
557 if (__improbable(task != NULL && task->vtimers)) {
558 thread_ast_set(thread, AST_BSD);
559 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
560 }
561 }
562
563 #if MACH_ASSERT
564 /**
565 * A version of get_preemption_level() that works in early boot.
566 *
567 * If an exception is raised in early boot before the initial thread has been
568 * set up, then calling get_preemption_level() in the SLEH will trigger an
569 * infinitely-recursing exception. This function handles this edge case.
570 */
571 static inline int
sleh_get_preemption_level(void)572 sleh_get_preemption_level(void)
573 {
574 if (__improbable(current_thread() == NULL)) {
575 return 0;
576 }
577 return get_preemption_level();
578 }
579 #endif // MACH_ASSERT
580
581 static inline bool
is_platform_error(uint32_t esr)582 is_platform_error(uint32_t esr)
583 {
584 esr_exception_class_t class = ESR_EC(esr);
585 uint32_t iss = ESR_ISS(esr);
586 fault_status_t fault_code;
587
588 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
589 fault_code = ISS_DA_FSC(iss);
590 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
591 fault_code = ISS_IA_FSC(iss);
592 } else {
593 return false;
594 }
595
596 return fault_code == FSC_SYNC_PARITY;
597 }
598
599 void
sleh_synchronous(arm_context_t * context,uint32_t esr,vm_offset_t far)600 sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far)
601 {
602 esr_exception_class_t class = ESR_EC(esr);
603 arm_saved_state_t * state = &context->ss;
604 thread_t thread = current_thread();
605 #if MACH_ASSERT
606 int preemption_level = sleh_get_preemption_level();
607 #endif
608 expected_fault_handler_t expected_fault_handler = NULL;
609 #ifdef CONFIG_XNUPOST
610 expected_fault_handler_t saved_expected_fault_handler = NULL;
611 uintptr_t saved_expected_fault_addr = 0;
612 #endif /* CONFIG_XNUPOST */
613
614 ASSERT_CONTEXT_SANITY(context);
615
616 task_vtimer_check(thread);
617
618 #if CONFIG_DTRACE
619 /*
620 * Handle kernel DTrace probes as early as possible to minimize the likelihood
621 * that this path will itself trigger a DTrace probe, which would lead to infinite
622 * probe recursion.
623 */
624 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
625 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
626 return;
627 }
628 #endif
629 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
630
631 /*
632 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
633 * that would disclose the behavior of PT_DENY_ATTACH processes.
634 */
635 if (is_user) {
636 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
637 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
638 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
639 esr, far, get_saved_state_pc(state), 0, 0);
640 } else {
641 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
642 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
643 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
644 }
645
646 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
647 /*
648 * We no longer support 32-bit, which means no 2-byte
649 * instructions.
650 */
651 if (is_user) {
652 panic("Exception on 2-byte instruction, "
653 "context=%p, esr=%#x, far=%p",
654 context, esr, (void *)far);
655 } else {
656 panic_with_thread_kernel_state("Exception on 2-byte instruction", state);
657 }
658 }
659
660 #ifdef CONFIG_XNUPOST
661 if (thread->machine.expected_fault_handler != NULL) {
662 saved_expected_fault_handler = thread->machine.expected_fault_handler;
663 saved_expected_fault_addr = thread->machine.expected_fault_addr;
664
665 thread->machine.expected_fault_handler = NULL;
666 thread->machine.expected_fault_addr = 0;
667
668 if (saved_expected_fault_addr == far) {
669 expected_fault_handler = saved_expected_fault_handler;
670 }
671 }
672 #endif /* CONFIG_XNUPOST */
673
674 if (__improbable(is_platform_error(esr))) {
675 /*
676 * Must gather error info in platform error handler before
677 * thread is preempted to another core/cluster to guarantee
678 * accurate error details
679 */
680
681 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_SYNC);
682 return;
683 }
684
685 if (is_user && class == ESR_EC_DABORT_EL0) {
686 thread_reset_pcs_will_fault(thread);
687 }
688
689 /* Inherit the interrupt masks from previous context */
690 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
691 ml_set_interrupts_enabled(TRUE);
692 }
693
694 switch (class) {
695 case ESR_EC_SVC_64:
696 if (!is_saved_state64(state) || !is_user) {
697 panic("Invalid SVC_64 context");
698 }
699
700 handle_svc(state);
701 break;
702
703 case ESR_EC_DABORT_EL0:
704 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
705 break;
706
707 case ESR_EC_MSR_TRAP:
708 handle_msr_trap(state, esr);
709 break;
710 /**
711 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for
712 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host
713 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC
714 * handler here needs to be built into the guest kernels to prevent the exception to fall
715 * through.
716 */
717 #if __has_feature(ptrauth_calls)
718 case ESR_EC_PAC_FAIL:
719 handle_pac_fail(state, esr);
720 __builtin_unreachable();
721
722 #endif /* __has_feature(ptrauth_calls) */
723
724 case ESR_EC_IABORT_EL0:
725 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
726 break;
727
728 case ESR_EC_IABORT_EL1:
729 #ifdef CONFIG_XNUPOST
730 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
731 break;
732 }
733 #endif /* CONFIG_XNUPOST */
734
735 panic_with_thread_kernel_state("Kernel instruction fetch abort", state);
736
737 case ESR_EC_PC_ALIGN:
738 handle_pc_align(state);
739 __builtin_unreachable();
740
741 case ESR_EC_DABORT_EL1:
742 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
743 break;
744
745 case ESR_EC_UNCATEGORIZED:
746 assert(!ESR_ISS(esr));
747
748 handle_uncategorized(&context->ss);
749 break;
750
751 case ESR_EC_SP_ALIGN:
752 handle_sp_align(state);
753 __builtin_unreachable();
754
755 case ESR_EC_BKPT_AARCH32:
756 handle_breakpoint(state, esr);
757 __builtin_unreachable();
758
759 case ESR_EC_BRK_AARCH64:
760 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
761 handle_kernel_breakpoint(state, esr);
762 #if CONFIG_UBSAN_MINIMAL
763 /* UBSan breakpoints are recoverable */
764 break;
765 #endif /* CONFIG_UBSAN_MINIMAL */
766 } else {
767 handle_breakpoint(state, esr);
768 __builtin_unreachable();
769 }
770
771 case ESR_EC_BKPT_REG_MATCH_EL0:
772 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
773 handle_breakpoint(state, esr);
774 }
775 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
776 class, state, class, esr, (void *)far);
777 __builtin_unreachable();
778
779 case ESR_EC_BKPT_REG_MATCH_EL1:
780 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state);
781 __builtin_unreachable();
782
783 case ESR_EC_SW_STEP_DEBUG_EL0:
784 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
785 handle_sw_step_debug(state);
786 }
787 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
788 class, state, class, esr, (void *)far);
789 __builtin_unreachable();
790
791 case ESR_EC_SW_STEP_DEBUG_EL1:
792 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state);
793 __builtin_unreachable();
794
795 case ESR_EC_WATCHPT_MATCH_EL0:
796 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
797 handle_watchpoint(far);
798 }
799 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
800 class, state, class, esr, (void *)far);
801 __builtin_unreachable();
802
803 case ESR_EC_WATCHPT_MATCH_EL1:
804 /*
805 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
806 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
807 */
808 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
809 arm_debug_set(NULL);
810 break; /* return to first level handler */
811 }
812 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
813 class, state, class, esr, (void *)far);
814 __builtin_unreachable();
815
816 case ESR_EC_TRAP_SIMD_FP:
817 handle_simd_trap(state, esr);
818 __builtin_unreachable();
819
820 case ESR_EC_ILLEGAL_INSTR_SET:
821 if (EXCB_ACTION_RERUN !=
822 ex_cb_invoke(EXCB_CLASS_ILLEGAL_INSTR_SET, far)) {
823 // instruction is not re-executed
824 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
825 state, class, esr, (void *)far, get_saved_state_cpsr(state));
826 }
827 // must clear this fault in PSR to re-run
828 mask_saved_state_cpsr(state, 0, PSR64_IL);
829 break;
830
831 case ESR_EC_MCR_MRC_CP15_TRAP:
832 case ESR_EC_MCRR_MRRC_CP15_TRAP:
833 case ESR_EC_MCR_MRC_CP14_TRAP:
834 case ESR_EC_LDC_STC_CP14_TRAP:
835 case ESR_EC_MCRR_MRRC_CP14_TRAP:
836 handle_user_trapped_instruction32(state, esr);
837 __builtin_unreachable();
838
839 case ESR_EC_WFI_WFE:
840 // Use of WFI or WFE instruction when they have been disabled for EL0
841 handle_wf_trap(state);
842 __builtin_unreachable();
843
844 case ESR_EC_FLOATING_POINT_64:
845 handle_fp_trap(state, esr);
846 __builtin_unreachable();
847
848 default:
849 handle_uncategorized(state);
850 }
851
852 #ifdef CONFIG_XNUPOST
853 if (saved_expected_fault_handler != NULL) {
854 thread->machine.expected_fault_handler = saved_expected_fault_handler;
855 thread->machine.expected_fault_addr = saved_expected_fault_addr;
856 }
857 #endif /* CONFIG_XNUPOST */
858
859 if (is_user) {
860 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
861 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
862 esr, far, get_saved_state_pc(state), 0, 0);
863 thread->machine.exception_trace_code = 0;
864 } else {
865 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
866 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
867 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
868 }
869 #if MACH_ASSERT
870 if (preemption_level != sleh_get_preemption_level()) {
871 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
872 }
873 #endif
874 }
875
876 /*
877 * Uncategorized exceptions are a catch-all for general execution errors.
878 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
879 */
880 static void
handle_uncategorized(arm_saved_state_t * state)881 handle_uncategorized(arm_saved_state_t *state)
882 {
883 exception_type_t exception = EXC_BAD_INSTRUCTION;
884 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
885 mach_msg_type_number_t numcodes = 2;
886 uint32_t instr = 0;
887
888 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
889
890 #if CONFIG_DTRACE
891
892 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
893 /*
894 * For a 64bit user process, we care about all 4 bytes of the
895 * instr.
896 */
897 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
898 if (dtrace_user_probe(state) == KERN_SUCCESS) {
899 return;
900 }
901 }
902 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
903 /*
904 * For a 32bit user process, we check for thumb mode, in
905 * which case we only care about a 2 byte instruction length.
906 * For non-thumb mode, we care about all 4 bytes of the instructin.
907 */
908 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
909 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
910 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
911 if (dtrace_user_probe(state) == KERN_SUCCESS) {
912 return;
913 }
914 }
915 } else {
916 if ((instr == FASTTRAP_ARM32_INSTR) ||
917 (instr == FASTTRAP_ARM32_RET_INSTR)) {
918 if (dtrace_user_probe(state) == KERN_SUCCESS) {
919 return;
920 }
921 }
922 }
923 }
924
925 #endif /* CONFIG_DTRACE */
926
927 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
928 if (IS_ARM_GDB_TRAP(instr)) {
929 boolean_t interrupt_state;
930 exception = EXC_BREAKPOINT;
931
932 interrupt_state = ml_set_interrupts_enabled(FALSE);
933
934 /* Save off the context here (so that the debug logic
935 * can see the original state of this thread).
936 */
937 current_thread()->machine.kpcb = state;
938
939 /* Hop into the debugger (typically either due to a
940 * fatal exception, an explicit panic, or a stackshot
941 * request.
942 */
943 DebuggerCall(exception, state);
944
945 current_thread()->machine.kpcb = NULL;
946 (void) ml_set_interrupts_enabled(interrupt_state);
947 return;
948 } else {
949 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
950 }
951 }
952
953 /*
954 * Check for GDB breakpoint via illegal opcode.
955 */
956 if (IS_ARM_GDB_TRAP(instr)) {
957 exception = EXC_BREAKPOINT;
958 codes[0] = EXC_ARM_BREAKPOINT;
959 codes[1] = instr;
960 } else {
961 codes[1] = instr;
962 }
963
964 exception_triage(exception, codes, numcodes);
965 __builtin_unreachable();
966 }
967
968 #if __has_feature(ptrauth_calls)
969 static const uint16_t ptrauth_brk_comment_base = 0xc470;
970
971 static inline bool
brk_comment_is_ptrauth(uint16_t comment)972 brk_comment_is_ptrauth(uint16_t comment)
973 {
974 return comment >= ptrauth_brk_comment_base &&
975 comment <= ptrauth_brk_comment_base + ptrauth_key_asdb;
976 }
977
978 static inline const char *
ptrauth_key_to_string(ptrauth_key key)979 ptrauth_key_to_string(ptrauth_key key)
980 {
981 switch (key) {
982 case ptrauth_key_asia:
983 return "IA";
984 case ptrauth_key_asib:
985 return "IB";
986 case ptrauth_key_asda:
987 return "DA";
988 case ptrauth_key_asdb:
989 return "DB";
990 default:
991 __builtin_unreachable();
992 }
993 }
994 #endif /* __has_feature(ptrauth_calls) */
995
996 #if KASAN_TBI
997 static inline bool
brk_comment_is_kasan_failure(uint16_t comment)998 brk_comment_is_kasan_failure(uint16_t comment)
999 {
1000 return comment >= KASAN_TBI_ESR_BASE &&
1001 comment <= KASAN_TBI_ESR_TOP;
1002 }
1003 #endif /* KASAN_TBI */
1004
1005 #if CONFIG_UBSAN_MINIMAL
1006 static inline bool
brk_comment_is_ubsan(uint16_t comment)1007 brk_comment_is_ubsan(uint16_t comment)
1008 {
1009 return comment >= UBSAN_MINIMAL_TRAPS_START &&
1010 comment < UBSAN_MINIMAL_TRAPS_END;
1011 }
1012 #endif /* CONFIG_UBSAN_MINIMAL */
1013
1014 static void
handle_kernel_breakpoint(arm_saved_state_t * state,uint32_t esr)1015 handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr)
1016 {
1017 uint16_t comment = ISS_BRK_COMMENT(esr);
1018
1019 #if __has_feature(ptrauth_calls)
1020 if (brk_comment_is_ptrauth(comment)) {
1021 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx"
1022 char msg[strlen(MSG_FMT)
1023 - strlen("0x%04X") + strlen("0xFFFF")
1024 - strlen("%s") + strlen("IA")
1025 - strlen("0x%016llx") + strlen("0xFFFFFFFFFFFFFFFF")
1026 + 1];
1027 ptrauth_key key = (ptrauth_key)(comment - ptrauth_brk_comment_base);
1028 const char *key_str = ptrauth_key_to_string(key);
1029 snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]);
1030
1031 panic_with_thread_kernel_state(msg, state);
1032 __builtin_unreachable();
1033 #undef MSG_FMT
1034 }
1035 #endif /* __has_feature(ptrauth_calls) */
1036
1037 #if KASAN_TBI
1038 if (brk_comment_is_kasan_failure(comment)) {
1039 kasan_handle_brk_failure(saved_state64(state)->x[0], comment);
1040 __builtin_unreachable();
1041 }
1042 #endif /* KASAN_TBI */
1043
1044 #if CONFIG_UBSAN_MINIMAL
1045 if (brk_comment_is_ubsan(comment)) {
1046 ubsan_handle_brk_trap(comment, get_saved_state_pc(state),
1047 get_saved_state_fp(state));
1048 add_saved_state_pc(state, 4);
1049 return;
1050 }
1051 #endif /* CONFIG_UBSAN_MINIMAL */
1052
1053 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)"
1054 char msg[strlen(MSG_FMT) - strlen("0x%04X") + strlen("0xFFFF") + 1];
1055 snprintf(msg, sizeof(msg), MSG_FMT, comment);
1056 #undef MSG_FMT
1057
1058 panic_with_thread_kernel_state(msg, state);
1059 __builtin_unreachable();
1060 }
1061
1062 static void
handle_breakpoint(arm_saved_state_t * state,uint32_t esr __unused)1063 handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused)
1064 {
1065 exception_type_t exception = EXC_BREAKPOINT;
1066 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1067 mach_msg_type_number_t numcodes = 2;
1068
1069 #if __has_feature(ptrauth_calls)
1070 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1071 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1072 exception |= EXC_PTRAUTH_BIT;
1073 }
1074 #endif /* __has_feature(ptrauth_calls) */
1075
1076 codes[1] = get_saved_state_pc(state);
1077 exception_triage(exception, codes, numcodes);
1078 __builtin_unreachable();
1079 }
1080
1081 static void
handle_watchpoint(vm_offset_t fault_addr)1082 handle_watchpoint(vm_offset_t fault_addr)
1083 {
1084 exception_type_t exception = EXC_BREAKPOINT;
1085 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1086 mach_msg_type_number_t numcodes = 2;
1087
1088 codes[1] = fault_addr;
1089 exception_triage(exception, codes, numcodes);
1090 __builtin_unreachable();
1091 }
1092
1093 static void
handle_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,abort_inspector_t inspect_abort,abort_handler_t handler,expected_fault_handler_t expected_fault_handler)1094 handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1095 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1096 {
1097 fault_status_t fault_code;
1098 vm_prot_t fault_type;
1099
1100 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1101 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1102 }
1103
1104 static void
inspect_instruction_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1105 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1106 {
1107 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1108 *fault_code = ISS_IA_FSC(iss);
1109 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1110 }
1111
1112 static void
inspect_data_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1113 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1114 {
1115 getCpuDatap()->cpu_stat.data_ex_cnt++;
1116 *fault_code = ISS_DA_FSC(iss);
1117
1118 /*
1119 * Cache maintenance operations always report faults as write access.
1120 * Change these to read access, unless they report a permission fault.
1121 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1122 * access to the mapping, but if a cache maintenance operation that only requires
1123 * read access generates a permission fault, then we will not be able to handle
1124 * the fault regardless of whether we treat it as a read or write fault.
1125 */
1126 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
1127 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1128 } else {
1129 *fault_type = (VM_PROT_READ);
1130 }
1131 }
1132
1133 #if __has_feature(ptrauth_calls)
1134 static inline uint64_t
fault_addr_bitmask(unsigned int bit_from,unsigned int bit_to)1135 fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1136 {
1137 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1138 }
1139
1140 static inline bool
fault_addr_bit(vm_offset_t fault_addr,unsigned int bit)1141 fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1142 {
1143 return (bool)((fault_addr >> bit) & 1);
1144 }
1145
1146 extern int gARM_FEAT_PAuth2;
1147
1148 /**
1149 * Determines whether a fault address taken at EL0 contains a PAC error code
1150 * corresponding to the specified kind of ptrauth key.
1151 */
1152 static bool
user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr,bool data_key)1153 user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key)
1154 {
1155 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1156 bool tbi = data_key || __improbable(instruction_tbi);
1157
1158 if (gARM_FEAT_PAuth2) {
1159 /*
1160 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1161 * treat all non-canonical address bits like potential poison bits.
1162 */
1163 uint64_t mask = fault_addr_bitmask(T0SZ_BOOT, 54);
1164 if (!tbi) {
1165 mask |= fault_addr_bitmask(56, 63);
1166 }
1167 return (fault_addr & mask) != 0;
1168 } else {
1169 unsigned int poison_shift;
1170 if (tbi) {
1171 poison_shift = 53;
1172 } else {
1173 poison_shift = 61;
1174 }
1175
1176 /* PAC error codes are always in the form key_number:NOT(key_number) */
1177 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1178 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1179 return poison_bit_1 != poison_bit_2;
1180 }
1181 }
1182 #endif /* __has_feature(ptrauth_calls) */
1183
1184 static void
handle_pc_align(arm_saved_state_t * ss)1185 handle_pc_align(arm_saved_state_t *ss)
1186 {
1187 exception_type_t exc;
1188 mach_exception_data_type_t codes[2];
1189 mach_msg_type_number_t numcodes = 2;
1190
1191 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1192 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
1193 }
1194
1195 exc = EXC_BAD_ACCESS;
1196 #if __has_feature(ptrauth_calls)
1197 if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) {
1198 exc |= EXC_PTRAUTH_BIT;
1199 }
1200 #endif /* __has_feature(ptrauth_calls) */
1201
1202 codes[0] = EXC_ARM_DA_ALIGN;
1203 codes[1] = get_saved_state_pc(ss);
1204
1205 exception_triage(exc, codes, numcodes);
1206 __builtin_unreachable();
1207 }
1208
1209 static void
handle_sp_align(arm_saved_state_t * ss)1210 handle_sp_align(arm_saved_state_t *ss)
1211 {
1212 exception_type_t exc;
1213 mach_exception_data_type_t codes[2];
1214 mach_msg_type_number_t numcodes = 2;
1215
1216 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1217 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
1218 }
1219
1220 exc = EXC_BAD_ACCESS;
1221 #if __has_feature(ptrauth_calls)
1222 if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) {
1223 exc |= EXC_PTRAUTH_BIT;
1224 }
1225 #endif /* __has_feature(ptrauth_calls) */
1226
1227 codes[0] = EXC_ARM_SP_ALIGN;
1228 codes[1] = get_saved_state_sp(ss);
1229
1230 exception_triage(exc, codes, numcodes);
1231 __builtin_unreachable();
1232 }
1233
1234 static void
handle_wf_trap(arm_saved_state_t * state)1235 handle_wf_trap(arm_saved_state_t *state)
1236 {
1237 exception_type_t exc;
1238 mach_exception_data_type_t codes[2];
1239 mach_msg_type_number_t numcodes = 2;
1240 uint32_t instr = 0;
1241
1242 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1243
1244 exc = EXC_BAD_INSTRUCTION;
1245 codes[0] = EXC_ARM_UNDEFINED;
1246 codes[1] = instr;
1247
1248 exception_triage(exc, codes, numcodes);
1249 __builtin_unreachable();
1250 }
1251
1252 static void
handle_fp_trap(arm_saved_state_t * state,uint32_t esr)1253 handle_fp_trap(arm_saved_state_t *state, uint32_t esr)
1254 {
1255 exception_type_t exc = EXC_ARITHMETIC;
1256 mach_exception_data_type_t codes[2];
1257 mach_msg_type_number_t numcodes = 2;
1258 uint32_t instr = 0;
1259
1260 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1261 panic_with_thread_kernel_state("Floating point exception from kernel", state);
1262 }
1263
1264 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1265 codes[1] = instr;
1266
1267 /* The floating point trap flags are only valid if TFV is set. */
1268 if (!fp_exceptions_enabled) {
1269 exc = EXC_BAD_INSTRUCTION;
1270 codes[0] = EXC_ARM_UNDEFINED;
1271 } else if (!(esr & ISS_FP_TFV)) {
1272 codes[0] = EXC_ARM_FP_UNDEFINED;
1273 } else if (esr & ISS_FP_UFF) {
1274 codes[0] = EXC_ARM_FP_UF;
1275 } else if (esr & ISS_FP_OFF) {
1276 codes[0] = EXC_ARM_FP_OF;
1277 } else if (esr & ISS_FP_IOF) {
1278 codes[0] = EXC_ARM_FP_IO;
1279 } else if (esr & ISS_FP_DZF) {
1280 codes[0] = EXC_ARM_FP_DZ;
1281 } else if (esr & ISS_FP_IDF) {
1282 codes[0] = EXC_ARM_FP_ID;
1283 } else if (esr & ISS_FP_IXF) {
1284 codes[0] = EXC_ARM_FP_IX;
1285 } else {
1286 panic("Unrecognized floating point exception, state=%p, esr=%#x", state, esr);
1287 }
1288
1289 exception_triage(exc, codes, numcodes);
1290 __builtin_unreachable();
1291 }
1292
1293
1294
1295 /*
1296 * handle_alignment_fault_from_user:
1297 * state: Saved state
1298 *
1299 * Attempts to deal with an alignment fault from userspace (possibly by
1300 * emulating the faulting instruction). If emulation failed due to an
1301 * unservicable fault, the ESR for that fault will be stored in the
1302 * recovery_esr field of the thread by the exception code.
1303 *
1304 * Returns:
1305 * -1: Emulation failed (emulation of state/instr not supported)
1306 * 0: Successfully emulated the instruction
1307 * EFAULT: Emulation failed (probably due to permissions)
1308 * EINVAL: Emulation failed (probably due to a bad address)
1309 */
1310
1311
1312 static int
handle_alignment_fault_from_user(arm_saved_state_t * state,kern_return_t * vmfr)1313 handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1314 {
1315 int ret = -1;
1316
1317 #pragma unused (state)
1318 #pragma unused (vmfr)
1319
1320 return ret;
1321 }
1322
1323
1324 static void
handle_sw_step_debug(arm_saved_state_t * state)1325 handle_sw_step_debug(arm_saved_state_t *state)
1326 {
1327 thread_t thread = current_thread();
1328 exception_type_t exc;
1329 mach_exception_data_type_t codes[2];
1330 mach_msg_type_number_t numcodes = 2;
1331
1332 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1333 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
1334 }
1335
1336 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1337 if (thread->machine.DebugData != NULL) {
1338 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1339 } else {
1340 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
1341 }
1342
1343 mask_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);
1344
1345 // Special encoding for gdb single step event on ARM
1346 exc = EXC_BREAKPOINT;
1347 codes[0] = 1;
1348 codes[1] = 0;
1349
1350 exception_triage(exc, codes, numcodes);
1351 __builtin_unreachable();
1352 }
1353
1354 static void
handle_user_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1355 handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1356 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1357 {
1358 exception_type_t exc = EXC_BAD_ACCESS;
1359 mach_exception_data_type_t codes[2];
1360 mach_msg_type_number_t numcodes = 2;
1361 thread_t thread = current_thread();
1362
1363 (void)esr;
1364 (void)expected_fault_handler;
1365
1366 if (ml_at_interrupt_context()) {
1367 panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state);
1368 }
1369
1370 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1371
1372 if (!is_vm_fault(fault_code) &&
1373 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1374 thread_reset_pcs_done_faulting(thread);
1375 }
1376
1377 if (is_vm_fault(fault_code)) {
1378 vm_map_t map = thread->map;
1379 vm_offset_t vm_fault_addr = fault_addr;
1380 kern_return_t result = KERN_FAILURE;
1381
1382 assert(map != kernel_map);
1383
1384 if (!(fault_type & VM_PROT_EXECUTE)) {
1385 vm_fault_addr = tbi_clear(fault_addr);
1386 }
1387
1388 /* check to see if it is just a pmap ref/modify fault */
1389 if (!is_translation_fault(fault_code)) {
1390 result = arm_fast_fault(map->pmap,
1391 vm_fault_addr,
1392 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1393 }
1394 if (result != KERN_SUCCESS) {
1395
1396 {
1397 /* We have to fault the page in */
1398 result = vm_fault(map, vm_fault_addr, fault_type,
1399 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1400 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1401 }
1402 }
1403 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1404 thread_reset_pcs_done_faulting(thread);
1405 }
1406 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1407 return;
1408 }
1409
1410 /*
1411 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1412 * If it does, we're leaking preemption disables somewhere in the kernel.
1413 */
1414 if (__improbable(result == KERN_FAILURE)) {
1415 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1416 }
1417
1418 codes[0] = result;
1419 } else if (is_alignment_fault(fault_code)) {
1420 kern_return_t vmfkr = KERN_SUCCESS;
1421 thread->machine.recover_esr = 0;
1422 thread->machine.recover_far = 0;
1423 int result = handle_alignment_fault_from_user(state, &vmfkr);
1424 if (result == 0) {
1425 /* Successfully emulated, or instruction
1426 * copyin() for decode/emulation failed.
1427 * Continue, or redrive instruction.
1428 */
1429 thread_exception_return();
1430 } else if (((result == EFAULT) || (result == EINVAL)) &&
1431 (thread->machine.recover_esr == 0)) {
1432 /*
1433 * If we didn't actually take a fault, but got one of
1434 * these errors, then we failed basic sanity checks of
1435 * the fault address. Treat this as an invalid
1436 * address.
1437 */
1438 codes[0] = KERN_INVALID_ADDRESS;
1439 } else if ((result == EFAULT) &&
1440 (thread->machine.recover_esr)) {
1441 /*
1442 * Since alignment aborts are prioritized
1443 * ahead of translation aborts, the misaligned
1444 * atomic emulation flow may have triggered a
1445 * VM pagefault, which the VM could not resolve.
1446 * Report the VM fault error in codes[]
1447 */
1448
1449 codes[0] = vmfkr;
1450 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1451 /* Cause ESR_EC to reflect an EL0 abort */
1452 thread->machine.recover_esr &= ~ESR_EC_MASK;
1453 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1454 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr);
1455 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far);
1456 fault_addr = thread->machine.recover_far;
1457 } else {
1458 /* This was just an unsupported alignment
1459 * exception. Misaligned atomic emulation
1460 * timeouts fall in this category.
1461 */
1462 codes[0] = EXC_ARM_DA_ALIGN;
1463 }
1464 } else if (is_parity_error(fault_code)) {
1465 #if defined(APPLE_ARM64_ARCH_FAMILY)
1466 /*
1467 * Platform errors are handled in sleh_sync before interrupts are enabled.
1468 */
1469 #else
1470 panic("User parity error.");
1471 #endif
1472 } else {
1473 codes[0] = KERN_FAILURE;
1474 }
1475
1476 codes[1] = fault_addr;
1477 #if __has_feature(ptrauth_calls)
1478 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1479 if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) {
1480 exc |= EXC_PTRAUTH_BIT;
1481 }
1482 #endif /* __has_feature(ptrauth_calls) */
1483 exception_triage(exc, codes, numcodes);
1484 __builtin_unreachable();
1485 }
1486
1487 static void
handle_kernel_abort_recover(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,thread_t thread)1488 handle_kernel_abort_recover(
1489 arm_saved_state_t *state,
1490 uint32_t esr,
1491 vm_offset_t fault_addr,
1492 thread_t thread)
1493 {
1494 thread->machine.recover_esr = esr;
1495 thread->machine.recover_far = fault_addr;
1496 #if defined(HAS_APPLE_PAC)
1497 MANIPULATE_SIGNED_THREAD_STATE(state,
1498 "mov x1, %[pc] \n"
1499 "str x1, [x0, %[SS64_PC]] \n",
1500 [pc] "r"(copyio_recovery_get_recover_addr(state))
1501 );
1502 #else
1503 saved_state64(state)->pc = copyio_recovery_get_recover_addr(state);
1504 #endif
1505 }
1506
1507 static void
handle_kernel_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1508 handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1509 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1510 {
1511 thread_t thread = current_thread();
1512 bool recover = find_copyio_recovery_entry(state) != 0;
1513
1514 #ifndef CONFIG_XNUPOST
1515 (void)expected_fault_handler;
1516 #endif /* CONFIG_XNUPOST */
1517
1518 #if CONFIG_DTRACE
1519 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
1520 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1521 /*
1522 * Point to next instruction, or recovery handler if set.
1523 */
1524 if (recover) {
1525 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1526 } else {
1527 add_saved_state_pc(state, 4);
1528 }
1529 return;
1530 } else {
1531 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
1532 }
1533 }
1534 #endif
1535
1536 if (ml_at_interrupt_context()) {
1537 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1538 }
1539
1540 if (is_vm_fault(fault_code)) {
1541 kern_return_t result = KERN_FAILURE;
1542 vm_map_t map;
1543 int interruptible;
1544
1545 /*
1546 * Ensure no faults in the physical aperture. This could happen if
1547 * a page table is incorrectly allocated from the read only region
1548 * when running with KTRR.
1549 */
1550
1551 #ifdef CONFIG_XNUPOST
1552 if (expected_fault_handler && expected_fault_handler(state)) {
1553 return;
1554 }
1555 #endif /* CONFIG_XNUPOST */
1556
1557 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1558 panic_with_thread_kernel_state("Unexpected fault in kernel static region\n", state);
1559 }
1560
1561 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) {
1562 /*
1563 * If no recovery handler is supplied, always drive the fault against
1564 * the kernel map. If the fault was taken against a userspace VA, indicating
1565 * an unprotected access to user address space, vm_fault() should fail and
1566 * ultimately lead to a panic here.
1567 */
1568 map = kernel_map;
1569 interruptible = THREAD_UNINT;
1570 } else {
1571 map = thread->map;
1572
1573 /**
1574 * In the case that the recovery handler is set (e.g., during copyio
1575 * and dtrace probes), we don't want the vm_fault() operation to be
1576 * aborted early. Those code paths can't handle restarting the
1577 * vm_fault() operation so don't allow it to return early without
1578 * creating the wanted mapping.
1579 */
1580 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
1581 }
1582
1583 /* check to see if it is just a pmap ref/modify fault */
1584 if (!is_translation_fault(fault_code)) {
1585 result = arm_fast_fault(map->pmap,
1586 fault_addr,
1587 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
1588 if (result == KERN_SUCCESS) {
1589 return;
1590 }
1591 }
1592
1593 if (result != KERN_PROTECTION_FAILURE) {
1594 /*
1595 * We have to "fault" the page in.
1596 */
1597 result = vm_fault(map, fault_addr, fault_type,
1598 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1599 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1600 }
1601
1602 if (result == KERN_SUCCESS) {
1603 return;
1604 }
1605
1606 /*
1607 * If we have a recover handler, invoke it now.
1608 */
1609 if (recover) {
1610 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1611 return;
1612 }
1613 } else if (is_alignment_fault(fault_code)) {
1614 if (recover) {
1615 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1616 return;
1617 }
1618 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
1619 } else if (is_parity_error(fault_code)) {
1620 #if defined(APPLE_ARM64_ARCH_FAMILY)
1621 /*
1622 * Platform errors are handled in sleh_sync before interrupts are enabled.
1623 */
1624 #else
1625 panic_with_thread_kernel_state("Kernel parity error.", state);
1626 #endif
1627 } else {
1628 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1629 }
1630
1631 panic_with_thread_kernel_state("Kernel data abort.", state);
1632 }
1633
1634 extern void syscall_trace(struct arm_saved_state * regs);
1635
1636 static void
handle_svc(arm_saved_state_t * state)1637 handle_svc(arm_saved_state_t *state)
1638 {
1639 int trap_no = get_saved_state_svc_number(state);
1640 thread_t thread = current_thread();
1641 struct proc *p;
1642
1643 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1644
1645 #define TRACE_SYSCALL 1
1646 #if TRACE_SYSCALL
1647 syscall_trace(state);
1648 #endif
1649
1650 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1651
1652 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1653 platform_syscall(state);
1654 panic("Returned from platform_syscall()?");
1655 }
1656
1657 mach_kauth_cred_thread_update();
1658
1659 if (trap_no < 0) {
1660 switch (trap_no) {
1661 case MACH_ARM_TRAP_ABSTIME:
1662 handle_mach_absolute_time_trap(state);
1663 return;
1664 case MACH_ARM_TRAP_CONTTIME:
1665 handle_mach_continuous_time_trap(state);
1666 return;
1667 }
1668
1669 /* Counting perhaps better in the handler, but this is how it's been done */
1670 thread->syscalls_mach++;
1671 mach_syscall(state);
1672 } else {
1673 /* Counting perhaps better in the handler, but this is how it's been done */
1674 thread->syscalls_unix++;
1675 p = get_bsdthreadtask_info(thread);
1676
1677 assert(p);
1678
1679 unix_syscall(state, thread, p);
1680 }
1681 }
1682
1683 static void
handle_mach_absolute_time_trap(arm_saved_state_t * state)1684 handle_mach_absolute_time_trap(arm_saved_state_t *state)
1685 {
1686 uint64_t now = mach_absolute_time();
1687 saved_state64(state)->x[0] = now;
1688 }
1689
1690 static void
handle_mach_continuous_time_trap(arm_saved_state_t * state)1691 handle_mach_continuous_time_trap(arm_saved_state_t *state)
1692 {
1693 uint64_t now = mach_continuous_time();
1694 saved_state64(state)->x[0] = now;
1695 }
1696
1697
1698 __attribute__((noreturn))
1699 static void
handle_msr_trap(arm_saved_state_t * state,uint32_t esr)1700 handle_msr_trap(arm_saved_state_t *state, uint32_t esr)
1701 {
1702 exception_type_t exception = EXC_BAD_INSTRUCTION;
1703 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1704 mach_msg_type_number_t numcodes = 2;
1705 uint32_t instr = 0;
1706
1707 if (!is_saved_state64(state)) {
1708 panic("MSR/MRS trap (ESR 0x%x) from 32-bit state", esr);
1709 }
1710
1711 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1712 panic("MSR/MRS trap (ESR 0x%x) from kernel", esr);
1713 }
1714
1715 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1716 codes[1] = instr;
1717
1718 exception_triage(exception, codes, numcodes);
1719 __builtin_unreachable();
1720 }
1721
1722 #if __has_feature(ptrauth_calls)
1723 static void
autxx_instruction_extract_reg(uint32_t instr,char reg[4])1724 autxx_instruction_extract_reg(uint32_t instr, char reg[4])
1725 {
1726 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
1727 switch (rd) {
1728 case 29:
1729 strncpy(reg, "fp", 4);
1730 return;
1731
1732 case 30:
1733 strncpy(reg, "lr", 4);
1734 return;
1735
1736 case 31:
1737 strncpy(reg, "xzr", 4);
1738 return;
1739
1740 default:
1741 snprintf(reg, 4, "x%u", rd);
1742 return;
1743 }
1744 }
1745
1746 static const char *
autix_system_instruction_extract_reg(uint32_t instr)1747 autix_system_instruction_extract_reg(uint32_t instr)
1748 {
1749 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
1750 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
1751 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
1752 return "x17";
1753 } else {
1754 return "lr";
1755 }
1756 }
1757
1758 static void
handle_pac_fail(arm_saved_state_t * state,uint32_t esr)1759 handle_pac_fail(arm_saved_state_t *state, uint32_t esr)
1760 {
1761 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
1762 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
1763 mach_msg_type_number_t numcodes = 2;
1764 uint32_t instr = 0;
1765
1766 if (!is_saved_state64(state)) {
1767 panic("PAC failure (ESR 0x%x) from 32-bit state", esr);
1768 }
1769
1770 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1771
1772 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1773 #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
1774 #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
1775 #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
1776
1777 char msg[strlen(AUTXX_MSG_FMT)
1778 - strlen("%s") + strlen("IA")
1779 - strlen("%s") + strlen("xzr")
1780 + 1];
1781 ptrauth_key key = (ptrauth_key)(esr & 0x3);
1782 const char *key_str = ptrauth_key_to_string(key);
1783
1784 if (ARM64_INSTR_IS_AUTxx(instr)) {
1785 char reg[4];
1786 autxx_instruction_extract_reg(instr, reg);
1787 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1788 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
1789 const char *reg = autix_system_instruction_extract_reg(instr);
1790 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1791 } else {
1792 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
1793 }
1794 panic_with_thread_kernel_state(msg, state);
1795 }
1796
1797 codes[1] = instr;
1798
1799 exception_triage(exception, codes, numcodes);
1800 __builtin_unreachable();
1801 }
1802 #endif /* __has_feature(ptrauth_calls) */
1803
1804 static void
handle_user_trapped_instruction32(arm_saved_state_t * state,uint32_t esr)1805 handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
1806 {
1807 exception_type_t exception = EXC_BAD_INSTRUCTION;
1808 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1809 mach_msg_type_number_t numcodes = 2;
1810 uint32_t instr;
1811
1812 if (is_saved_state64(state)) {
1813 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
1814 }
1815
1816 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1817 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
1818 }
1819
1820 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1821 codes[1] = instr;
1822
1823 exception_triage(exception, codes, numcodes);
1824 __builtin_unreachable();
1825 }
1826
1827 static void
handle_simd_trap(arm_saved_state_t * state,uint32_t esr)1828 handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
1829 {
1830 exception_type_t exception = EXC_BAD_INSTRUCTION;
1831 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1832 mach_msg_type_number_t numcodes = 2;
1833 uint32_t instr = 0;
1834
1835 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1836 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
1837 }
1838
1839 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1840 codes[1] = instr;
1841
1842 exception_triage(exception, codes, numcodes);
1843 __builtin_unreachable();
1844 }
1845
1846 void
sleh_irq(arm_saved_state_t * state)1847 sleh_irq(arm_saved_state_t *state)
1848 {
1849 cpu_data_t * cdp __unused = getCpuDatap();
1850 #if MACH_ASSERT
1851 int preemption_level = sleh_get_preemption_level();
1852 #endif
1853
1854
1855 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
1856
1857 #if USE_APPLEARMSMP
1858 PE_handle_ext_interrupt();
1859 #else
1860 /* Run the registered interrupt handler. */
1861 cdp->interrupt_handler(cdp->interrupt_target,
1862 cdp->interrupt_refCon,
1863 cdp->interrupt_nub,
1864 cdp->interrupt_source);
1865 #endif
1866
1867 entropy_collect();
1868
1869
1870 sleh_interrupt_handler_epilogue();
1871 #if MACH_ASSERT
1872 if (preemption_level != sleh_get_preemption_level()) {
1873 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
1874 }
1875 #endif
1876 }
1877
1878 void
sleh_fiq(arm_saved_state_t * state)1879 sleh_fiq(arm_saved_state_t *state)
1880 {
1881 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
1882 #if MACH_ASSERT
1883 int preemption_level = sleh_get_preemption_level();
1884 #endif
1885
1886 #if MONOTONIC_FIQ
1887 uint64_t pmcr0 = 0, upmsr = 0;
1888 #endif /* MONOTONIC_FIQ */
1889
1890 #if defined(HAS_IPI)
1891 boolean_t is_ipi = FALSE;
1892 uint64_t ipi_sr = 0;
1893
1894 if (gFastIPI) {
1895 MRS(ipi_sr, "S3_5_C15_C1_1");
1896
1897 if (ipi_sr & ARM64_IPISR_IPI_PENDING) {
1898 is_ipi = TRUE;
1899 }
1900 }
1901
1902 if (is_ipi) {
1903 type = DBG_INTR_TYPE_IPI;
1904 } else
1905 #endif /* defined(HAS_IPI) */
1906 if (ml_get_timer_pending()) {
1907 type = DBG_INTR_TYPE_TIMER;
1908 }
1909 #if MONOTONIC_FIQ
1910 /* Consult the PMI sysregs last, after IPI/timer
1911 * classification.
1912 */
1913 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
1914 type = DBG_INTR_TYPE_PMI;
1915 }
1916 #endif /* MONOTONIC_FIQ */
1917
1918 sleh_interrupt_handler_prologue(state, type);
1919
1920 #if APPLEVIRTUALPLATFORM
1921 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
1922 #endif
1923
1924 #if defined(HAS_IPI)
1925 if (type == DBG_INTR_TYPE_IPI) {
1926 /*
1927 * Order is important here: we must ack the IPI by writing IPI_SR
1928 * before we call cpu_signal_handler(). Otherwise, there will be
1929 * a window between the completion of pending-signal processing in
1930 * cpu_signal_handler() and the ack during which a newly-issued
1931 * IPI to this CPU may be lost. ISB is required to ensure the msr
1932 * is retired before execution of cpu_signal_handler().
1933 */
1934 MSR("S3_5_C15_C1_1", ARM64_IPISR_IPI_PENDING);
1935 __builtin_arm_isb(ISB_SY);
1936 cpu_signal_handler();
1937 } else
1938 #endif /* defined(HAS_IPI) */
1939 #if MONOTONIC_FIQ
1940 if (type == DBG_INTR_TYPE_PMI) {
1941 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
1942 mt_fiq(getCpuDatap(), pmcr0, upmsr);
1943 INTERRUPT_MASKED_DEBUG_END();
1944 } else
1945 #endif /* MONOTONIC_FIQ */
1946 {
1947 /*
1948 * We don't know that this is a timer, but we don't have insight into
1949 * the other interrupts that go down this path.
1950 */
1951
1952 cpu_data_t *cdp = getCpuDatap();
1953
1954 cdp->cpu_decrementer = -1; /* Large */
1955
1956 /*
1957 * ARM64_TODO: whether we're coming from userland is ignored right now.
1958 * We can easily thread it through, but not bothering for the
1959 * moment (AArch32 doesn't either).
1960 */
1961 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
1962 rtclock_intr(TRUE);
1963 INTERRUPT_MASKED_DEBUG_END();
1964 }
1965
1966 #if APPLEVIRTUALPLATFORM
1967 if (iar != GIC_SPURIOUS_IRQ) {
1968 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
1969 __builtin_arm_isb(ISB_SY);
1970 }
1971 #endif
1972
1973 sleh_interrupt_handler_epilogue();
1974 #if MACH_ASSERT
1975 if (preemption_level != sleh_get_preemption_level()) {
1976 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
1977 }
1978 #endif
1979 }
1980
1981 void
sleh_serror(arm_context_t * context,uint32_t esr,vm_offset_t far)1982 sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
1983 {
1984 task_vtimer_check(current_thread());
1985
1986 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
1987 esr, VM_KERNEL_ADDRHIDE(far));
1988 arm_saved_state_t *state = &context->ss;
1989 #if MACH_ASSERT
1990 int preemption_level = sleh_get_preemption_level();
1991 #endif
1992
1993
1994 ASSERT_CONTEXT_SANITY(context);
1995 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC);
1996 #if MACH_ASSERT
1997 if (preemption_level != sleh_get_preemption_level()) {
1998 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
1999 }
2000 #endif
2001 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
2002 esr, VM_KERNEL_ADDRHIDE(far));
2003 }
2004
2005 void
mach_syscall_trace_exit(unsigned int retval,unsigned int call_number)2006 mach_syscall_trace_exit(unsigned int retval,
2007 unsigned int call_number)
2008 {
2009 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2010 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
2011 DBG_FUNC_END, retval, 0, 0, 0, 0);
2012 }
2013
2014 __attribute__((noreturn))
2015 void
thread_syscall_return(kern_return_t error)2016 thread_syscall_return(kern_return_t error)
2017 {
2018 thread_t thread;
2019 struct arm_saved_state *state;
2020
2021 thread = current_thread();
2022 state = get_user_regs(thread);
2023
2024 assert(is_saved_state64(state));
2025 saved_state64(state)->x[0] = error;
2026
2027 #if MACH_ASSERT
2028 kern_allocation_name_t
2029 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
2030 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2031 #endif /* MACH_ASSERT */
2032
2033 if (kdebug_enable) {
2034 /* Invert syscall number (negative for a mach syscall) */
2035 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
2036 }
2037
2038 thread_exception_return();
2039 }
2040
2041 void
syscall_trace(struct arm_saved_state * regs __unused)2042 syscall_trace(
2043 struct arm_saved_state * regs __unused)
2044 {
2045 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2046 }
2047
2048 static void
sleh_interrupt_handler_prologue(arm_saved_state_t * state,unsigned int type)2049 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2050 {
2051 boolean_t is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2052
2053 task_vtimer_check(current_thread());
2054
2055 uint64_t pc = is_user ? get_saved_state_pc(state) :
2056 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2057
2058 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2059 0, pc, is_user, type);
2060
2061 #if CONFIG_TELEMETRY
2062 if (telemetry_needs_record) {
2063 telemetry_mark_curthread(is_user, FALSE);
2064 }
2065 #endif /* CONFIG_TELEMETRY */
2066 }
2067
2068 static void
sleh_interrupt_handler_epilogue(void)2069 sleh_interrupt_handler_epilogue(void)
2070 {
2071 #if KPERF
2072 kperf_interrupt();
2073 #endif /* KPERF */
2074 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2075 }
2076
2077 void
sleh_invalid_stack(arm_context_t * context,uint32_t esr __unused,vm_offset_t far __unused)2078 sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
2079 {
2080 thread_t thread = current_thread();
2081 vm_offset_t kernel_stack_bottom, sp;
2082
2083 sp = get_saved_state_sp(&context->ss);
2084 kernel_stack_bottom = round_page(thread->machine.kstackptr) - KERNEL_STACK_SIZE;
2085
2086 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2087 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
2088 }
2089
2090 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
2091 }
2092
2093