1 /*
2 * Copyright (c) 2012-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39 #include <arm64/instructions.h>
40
41 #include <kern/debug.h>
42 #include <kern/restartable.h>
43 #include <kern/socd_client.h>
44 #include <kern/thread.h>
45 #include <mach/exception.h>
46 #include <mach/arm/traps.h>
47 #include <mach/vm_types.h>
48 #include <mach/machine/thread_status.h>
49
50 #include <machine/atomic.h>
51 #include <machine/limits.h>
52
53 #include <pexpert/arm/protos.h>
54
55 #include <vm/vm_page.h>
56 #include <vm/pmap.h>
57 #include <vm/vm_fault.h>
58 #include <vm/vm_kern.h>
59
60 #include <sys/errno.h>
61 #include <sys/kdebug.h>
62 #include <kperf/kperf.h>
63
64 #include <kern/policy_internal.h>
65 #if CONFIG_TELEMETRY
66 #include <kern/telemetry.h>
67 #endif
68
69 #include <prng/entropy.h>
70
71
72
73
74 #include <arm64/platform_error_handler.h>
75
76 #if CONFIG_KERNEL_TBI && KASAN_TBI
77 #include <san/kasan.h>
78 #endif /* CONFIG_KERNEL_TBI && KASAN_TBI */
79
80 #if CONFIG_UBSAN_MINIMAL
81 #include <san/ubsan_minimal.h>
82 #endif /* CONFIG_UBSAN_MINIMAL */
83
84 #ifndef __arm64__
85 #error Should only be compiling for arm64.
86 #endif
87
88 #define TEST_CONTEXT32_SANITY(context) \
89 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
90 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
91
92 #define TEST_CONTEXT64_SANITY(context) \
93 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
94 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
95
96 #define ASSERT_CONTEXT_SANITY(context) \
97 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
98
99
100 #define COPYIN(src, dst, size) \
101 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
102 copyin_kern(src, dst, size) : \
103 copyin(src, dst, size)
104
105 #define COPYOUT(src, dst, size) \
106 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
107 copyout_kern(src, dst, size) : \
108 copyout(src, dst, size)
109
110 // Below is for concatenating a string param to a string literal
111 #define STR1(x) #x
112 #define STR(x) STR1(x)
113
114 #define ARM64_KDBG_CODE_KERNEL (0 << 8)
115 #define ARM64_KDBG_CODE_USER (1 << 8)
116 #define ARM64_KDBG_CODE_GUEST (2 << 8)
117
118 _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
119 _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
120
121 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
122
123 void sleh_synchronous_sp1(arm_context_t *, uint32_t, vm_offset_t) __abortlike;
124 void sleh_synchronous(arm_context_t *, uint32_t, vm_offset_t);
125
126
127
128 void sleh_irq(arm_saved_state_t *);
129 void sleh_fiq(arm_saved_state_t *);
130 void sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far);
131 void sleh_invalid_stack(arm_context_t *context, uint32_t esr, vm_offset_t far) __dead2;
132
133 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
134 static void sleh_interrupt_handler_epilogue(void);
135
136 static void handle_svc(arm_saved_state_t *);
137 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
138 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
139
140 static void handle_msr_trap(arm_saved_state_t *state, uint32_t esr);
141 #if __has_feature(ptrauth_calls)
142 static void handle_pac_fail(arm_saved_state_t *state, uint32_t esr) __dead2;
143 #endif
144
145 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
146
147 static void handle_uncategorized(arm_saved_state_t *);
148
149 /*
150 * For UBSan trap and continue handling, we must be able to recover
151 * from handle_kernel_breakpoint().
152 */
153 #if !CONFIG_UBSAN_MINIMAL
154 __dead2
155 #endif /* CONFIG_UBSAN_MINIMAL */
156 static void handle_kernel_breakpoint(arm_saved_state_t *, uint32_t);
157
158 static void handle_breakpoint(arm_saved_state_t *, uint32_t) __dead2;
159
160 typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
161 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
162 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
163
164 static int is_vm_fault(fault_status_t);
165 static int is_translation_fault(fault_status_t);
166 static int is_alignment_fault(fault_status_t);
167
168 typedef void (*abort_handler_t)(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
169 static void handle_user_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
170 static void handle_kernel_abort(arm_saved_state_t *, uint32_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
171
172 static void handle_pc_align(arm_saved_state_t *ss) __dead2;
173 static void handle_sp_align(arm_saved_state_t *ss) __dead2;
174 static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
175 static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
176 static void handle_fp_trap(arm_saved_state_t *ss, uint32_t esr) __dead2;
177
178 static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
179
180 static void handle_abort(arm_saved_state_t *, uint32_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
181
182 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint32_t esr) __dead2;
183
184 static void handle_simd_trap(arm_saved_state_t *, uint32_t esr) __dead2;
185
186 extern void mach_kauth_cred_thread_update(void);
187 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
188
189 struct proc;
190
191 typedef uint32_t arm64_instr_t;
192
193 extern void
194 unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
195
196 extern void
197 mach_syscall(struct arm_saved_state*);
198
199 #if CONFIG_DTRACE
200 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
201 extern boolean_t dtrace_tally_fault(user_addr_t);
202
203 /*
204 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
205 * and paste the trap instructions
206 * over from that file. Need to keep these in sync!
207 */
208 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
209 #define FASTTRAP_THUMB32_INSTR 0xdefc
210 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
211
212 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
213 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
214 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
215
216 /* See <rdar://problem/4613924> */
217 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
218 #endif
219
220
221
222 extern void arm64_thread_exception_return(void) __dead2;
223
224 #if defined(APPLETYPHOON)
225 #define CPU_NAME "Typhoon"
226 #elif defined(APPLETWISTER)
227 #define CPU_NAME "Twister"
228 #elif defined(APPLEHURRICANE)
229 #define CPU_NAME "Hurricane"
230 #elif defined(APPLELIGHTNING)
231 #define CPU_NAME "Lightning"
232 #else
233 #define CPU_NAME "Unknown"
234 #endif
235
236 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
237 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
238 #define ESR_WT_REASON(esr) ((esr) & 0xff)
239
240 #define WT_REASON_NONE 0
241 #define WT_REASON_INTEGRITY_FAIL 1
242 #define WT_REASON_BAD_SYSCALL 2
243 #define WT_REASON_NOT_LOCKED 3
244 #define WT_REASON_ALREADY_LOCKED 4
245 #define WT_REASON_SW_REQ 5
246 #define WT_REASON_PT_INVALID 6
247 #define WT_REASON_PT_VIOLATION 7
248 #define WT_REASON_REG_VIOLATION 8
249 #endif
250
251 #if defined(HAS_IPI)
252 void cpu_signal_handler(void);
253 extern unsigned int gFastIPI;
254 #endif /* defined(HAS_IPI) */
255
256 static arm_saved_state64_t *original_faulting_state = NULL;
257
258 TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
259
260 extern vm_offset_t static_memory_end;
261
262 /*
263 * Fault copyio_recovery_entry in copyin/copyout routines.
264 *
265 * Offets are expressed in bytes from ©_recovery_table
266 */
267 struct copyio_recovery_entry {
268 ptrdiff_t cre_start;
269 ptrdiff_t cre_end;
270 ptrdiff_t cre_recovery;
271 };
272
273 extern struct copyio_recovery_entry copyio_recover_table[];
274 extern struct copyio_recovery_entry copyio_recover_table_end[];
275
276 static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)277 copyio_recovery_offset(uintptr_t addr)
278 {
279 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
280 }
281
282 static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)283 copyio_recovery_addr(ptrdiff_t offset)
284 {
285 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
286 }
287
288 static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(arm_saved_state_t * state)289 find_copyio_recovery_entry(arm_saved_state_t *state)
290 {
291 ptrdiff_t offset = copyio_recovery_offset(get_saved_state_pc(state));
292 struct copyio_recovery_entry *e;
293
294 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
295 if (offset >= e->cre_start && offset < e->cre_end) {
296 return e;
297 }
298 }
299
300 return NULL;
301 }
302
303 static inline uintptr_t
copyio_recovery_get_recover_addr(arm_saved_state_t * state)304 copyio_recovery_get_recover_addr(
305 arm_saved_state_t *state)
306 {
307 struct copyio_recovery_entry *e = find_copyio_recovery_entry(state);
308 if (e == NULL) {
309 panic("copyio recovery: couldn't find a range for %p",
310 (void *)get_saved_state_pc(state));
311 }
312 return copyio_recovery_addr(e->cre_recovery);
313 }
314
315 static inline int
is_vm_fault(fault_status_t status)316 is_vm_fault(fault_status_t status)
317 {
318 switch (status) {
319 case FSC_TRANSLATION_FAULT_L0:
320 case FSC_TRANSLATION_FAULT_L1:
321 case FSC_TRANSLATION_FAULT_L2:
322 case FSC_TRANSLATION_FAULT_L3:
323 case FSC_ACCESS_FLAG_FAULT_L1:
324 case FSC_ACCESS_FLAG_FAULT_L2:
325 case FSC_ACCESS_FLAG_FAULT_L3:
326 case FSC_PERMISSION_FAULT_L1:
327 case FSC_PERMISSION_FAULT_L2:
328 case FSC_PERMISSION_FAULT_L3:
329 return TRUE;
330 default:
331 return FALSE;
332 }
333 }
334
335 static inline int
is_translation_fault(fault_status_t status)336 is_translation_fault(fault_status_t status)
337 {
338 switch (status) {
339 case FSC_TRANSLATION_FAULT_L0:
340 case FSC_TRANSLATION_FAULT_L1:
341 case FSC_TRANSLATION_FAULT_L2:
342 case FSC_TRANSLATION_FAULT_L3:
343 return TRUE;
344 default:
345 return FALSE;
346 }
347 }
348
349 static inline int
is_permission_fault(fault_status_t status)350 is_permission_fault(fault_status_t status)
351 {
352 switch (status) {
353 case FSC_PERMISSION_FAULT_L1:
354 case FSC_PERMISSION_FAULT_L2:
355 case FSC_PERMISSION_FAULT_L3:
356 return TRUE;
357 default:
358 return FALSE;
359 }
360 }
361
362 static inline int
is_alignment_fault(fault_status_t status)363 is_alignment_fault(fault_status_t status)
364 {
365 return status == FSC_ALIGNMENT_FAULT;
366 }
367
368 static inline int
is_parity_error(fault_status_t status)369 is_parity_error(fault_status_t status)
370 {
371 switch (status) {
372 /*
373 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
374 * only applicable to AArch32 HSR register. Can this be removed?
375 */
376 case FSC_ASYNC_PARITY:
377 case FSC_SYNC_PARITY:
378 case FSC_SYNC_PARITY_TT_L1:
379 case FSC_SYNC_PARITY_TT_L2:
380 case FSC_SYNC_PARITY_TT_L3:
381 return TRUE;
382 default:
383 return FALSE;
384 }
385 }
386
387
388 __dead2 __unused
389 static void
arm64_implementation_specific_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far)390 arm64_implementation_specific_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far)
391 {
392 #pragma unused (state, esr, far)
393 panic_plain("Unhandled implementation specific error\n");
394 }
395
396 #if CONFIG_KERNEL_INTEGRITY
397 #pragma clang diagnostic push
398 #pragma clang diagnostic ignored "-Wunused-parameter"
399 static void
kernel_integrity_error_handler(uint32_t esr,vm_offset_t far)400 kernel_integrity_error_handler(uint32_t esr, vm_offset_t far)
401 {
402 #if defined(KERNEL_INTEGRITY_WT)
403 #if (DEVELOPMENT || DEBUG)
404 if (ESR_WT_SERROR(esr)) {
405 switch (ESR_WT_REASON(esr)) {
406 case WT_REASON_INTEGRITY_FAIL:
407 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
408 case WT_REASON_BAD_SYSCALL:
409 panic_plain("Kernel integrity, bad syscall.");
410 case WT_REASON_NOT_LOCKED:
411 panic_plain("Kernel integrity, not locked.");
412 case WT_REASON_ALREADY_LOCKED:
413 panic_plain("Kernel integrity, already locked.");
414 case WT_REASON_SW_REQ:
415 panic_plain("Kernel integrity, software request.");
416 case WT_REASON_PT_INVALID:
417 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
418 "walking 0x%016lx.", far);
419 case WT_REASON_PT_VIOLATION:
420 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
421 far);
422 case WT_REASON_REG_VIOLATION:
423 panic_plain("Kernel integrity, violation in system register %d.",
424 (unsigned) far);
425 default:
426 panic_plain("Kernel integrity, unknown (esr=0x%08x).", esr);
427 }
428 }
429 #else
430 if (ESR_WT_SERROR(esr)) {
431 panic_plain("SError esr: 0x%08x far: 0x%016lx.", esr, far);
432 }
433 #endif
434 #endif
435 }
436 #pragma clang diagnostic pop
437 #endif
438
439 static void
arm64_platform_error(arm_saved_state_t * state,uint32_t esr,vm_offset_t far,platform_error_source_t source)440 arm64_platform_error(arm_saved_state_t *state, uint32_t esr, vm_offset_t far, platform_error_source_t source)
441 {
442 #if CONFIG_KERNEL_INTEGRITY
443 kernel_integrity_error_handler(esr, far);
444 #endif
445
446 (void)source;
447 cpu_data_t *cdp = getCpuDatap();
448
449 if (PE_handle_platform_error(far)) {
450 return;
451 } else if (cdp->platform_error_handler != NULL) {
452 cdp->platform_error_handler(cdp->cpu_id, far);
453 } else {
454 arm64_implementation_specific_error(state, esr, far);
455 }
456 }
457
458 void
panic_with_thread_kernel_state(const char * msg,arm_saved_state_t * ss)459 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
460 {
461 boolean_t ss_valid;
462
463 ss_valid = is_saved_state64(ss);
464 arm_saved_state64_t *state = saved_state64(ss);
465
466 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
467
468 // rdar://80659177
469 // Read SoCD tracepoints up to twice — once the first time we call panic and
470 // another time if we encounter a nested panic after that.
471 static int twice = 2;
472 if (twice > 0) {
473 twice--;
474 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, ADDR(state->pc),
475 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
476 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
477 VALUE(state->far));
478 }
479
480 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
481 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
482 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
483 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
484 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
485 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
486 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
487 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
488 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
489 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%08x far: 0x%016llx\n",
490 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
491 state->x[0], state->x[1], state->x[2], state->x[3],
492 state->x[4], state->x[5], state->x[6], state->x[7],
493 state->x[8], state->x[9], state->x[10], state->x[11],
494 state->x[12], state->x[13], state->x[14], state->x[15],
495 state->x[16], state->x[17], state->x[18], state->x[19],
496 state->x[20], state->x[21], state->x[22], state->x[23],
497 state->x[24], state->x[25], state->x[26], state->x[27],
498 state->x[28], state->fp, state->lr, state->sp,
499 state->pc, state->cpsr, state->esr, state->far);
500 }
501
502 void
sleh_synchronous_sp1(arm_context_t * context,uint32_t esr,vm_offset_t far __unused)503 sleh_synchronous_sp1(arm_context_t *context, uint32_t esr, vm_offset_t far __unused)
504 {
505 esr_exception_class_t class = ESR_EC(esr);
506 arm_saved_state_t * state = &context->ss;
507
508 switch (class) {
509 case ESR_EC_UNCATEGORIZED:
510 {
511 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
512 if (IS_ARM_GDB_TRAP(instr)) {
513 DebuggerCall(EXC_BREAKPOINT, state);
514 }
515 }
516 OS_FALLTHROUGH; // panic if we return from the debugger
517 default:
518 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
519 }
520 }
521
522
523 __attribute__((noreturn))
524 void
thread_exception_return()525 thread_exception_return()
526 {
527 thread_t thread = current_thread();
528 if (thread->machine.exception_trace_code != 0) {
529 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
530 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
531 thread->machine.exception_trace_code = 0;
532 }
533
534 #if KASAN && CONFIG_KERNEL_TBI
535 kasan_unpoison_curstack(true);
536 #endif /* KASAN && CONFIG_KERNEL_TBI */
537 arm64_thread_exception_return();
538 __builtin_unreachable();
539 }
540
541 /*
542 * check whether task vtimers are running and set thread and CPU BSD AST
543 *
544 * must be called with interrupts masked so updates of fields are atomic
545 * must be emitted inline to avoid generating an FBT probe on the exception path
546 *
547 */
548 __attribute__((__always_inline__))
549 static inline void
task_vtimer_check(thread_t thread)550 task_vtimer_check(thread_t thread)
551 {
552 task_t task = get_threadtask_early(thread);
553
554 if (__improbable(task != NULL && task->vtimers)) {
555 thread_ast_set(thread, AST_BSD);
556 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
557 }
558 }
559
560 #if MACH_ASSERT
561 /**
562 * A version of get_preemption_level() that works in early boot.
563 *
564 * If an exception is raised in early boot before the initial thread has been
565 * set up, then calling get_preemption_level() in the SLEH will trigger an
566 * infinitely-recursing exception. This function handles this edge case.
567 */
568 static inline int
sleh_get_preemption_level(void)569 sleh_get_preemption_level(void)
570 {
571 if (__improbable(current_thread() == NULL)) {
572 return 0;
573 }
574 return get_preemption_level();
575 }
576 #endif // MACH_ASSERT
577
578 static inline bool
is_platform_error(uint32_t esr)579 is_platform_error(uint32_t esr)
580 {
581 esr_exception_class_t class = ESR_EC(esr);
582 uint32_t iss = ESR_ISS(esr);
583 fault_status_t fault_code;
584
585 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
586 fault_code = ISS_DA_FSC(iss);
587 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
588 fault_code = ISS_IA_FSC(iss);
589 } else {
590 return false;
591 }
592
593 return fault_code == FSC_SYNC_PARITY;
594 }
595
596 void
sleh_synchronous(arm_context_t * context,uint32_t esr,vm_offset_t far)597 sleh_synchronous(arm_context_t *context, uint32_t esr, vm_offset_t far)
598 {
599 esr_exception_class_t class = ESR_EC(esr);
600 arm_saved_state_t * state = &context->ss;
601 thread_t thread = current_thread();
602 #if MACH_ASSERT
603 int preemption_level = sleh_get_preemption_level();
604 #endif
605 expected_fault_handler_t expected_fault_handler = NULL;
606 #ifdef CONFIG_XNUPOST
607 expected_fault_handler_t saved_expected_fault_handler = NULL;
608 uintptr_t saved_expected_fault_addr = 0;
609 #endif /* CONFIG_XNUPOST */
610
611 ASSERT_CONTEXT_SANITY(context);
612
613 task_vtimer_check(thread);
614
615 #if CONFIG_DTRACE
616 /*
617 * Handle kernel DTrace probes as early as possible to minimize the likelihood
618 * that this path will itself trigger a DTrace probe, which would lead to infinite
619 * probe recursion.
620 */
621 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
622 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
623 return;
624 }
625 #endif
626 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
627
628 /*
629 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
630 * that would disclose the behavior of PT_DENY_ATTACH processes.
631 */
632 if (is_user) {
633 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
634 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
635 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
636 esr, far, get_saved_state_pc(state), 0, 0);
637 } else {
638 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
639 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
640 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
641 }
642
643 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
644 /*
645 * We no longer support 32-bit, which means no 2-byte
646 * instructions.
647 */
648 if (is_user) {
649 panic("Exception on 2-byte instruction, "
650 "context=%p, esr=%#x, far=%p",
651 context, esr, (void *)far);
652 } else {
653 panic_with_thread_kernel_state("Exception on 2-byte instruction", state);
654 }
655 }
656
657 #ifdef CONFIG_XNUPOST
658 if (thread->machine.expected_fault_handler != NULL) {
659 saved_expected_fault_handler = thread->machine.expected_fault_handler;
660 saved_expected_fault_addr = thread->machine.expected_fault_addr;
661
662 thread->machine.expected_fault_handler = NULL;
663 thread->machine.expected_fault_addr = 0;
664
665 if (saved_expected_fault_addr == far) {
666 expected_fault_handler = saved_expected_fault_handler;
667 }
668 }
669 #endif /* CONFIG_XNUPOST */
670
671 if (__improbable(is_platform_error(esr))) {
672 /*
673 * Must gather error info in platform error handler before
674 * thread is preempted to another core/cluster to guarantee
675 * accurate error details
676 */
677
678 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_SYNC);
679 return;
680 }
681
682 if (is_user && class == ESR_EC_DABORT_EL0) {
683 thread_reset_pcs_will_fault(thread);
684 }
685
686 /* Inherit the interrupt masks from previous context */
687 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
688 ml_set_interrupts_enabled(TRUE);
689 }
690
691 switch (class) {
692 case ESR_EC_SVC_64:
693 if (!is_saved_state64(state) || !is_user) {
694 panic("Invalid SVC_64 context");
695 }
696
697 handle_svc(state);
698 break;
699
700 case ESR_EC_DABORT_EL0:
701 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
702 break;
703
704 case ESR_EC_MSR_TRAP:
705 handle_msr_trap(state, esr);
706 break;
707 /**
708 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for
709 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host
710 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC
711 * handler here needs to be built into the guest kernels to prevent the exception to fall
712 * through.
713 */
714 #if __has_feature(ptrauth_calls)
715 case ESR_EC_PAC_FAIL:
716 handle_pac_fail(state, esr);
717 __builtin_unreachable();
718
719 #endif /* __has_feature(ptrauth_calls) */
720
721 case ESR_EC_IABORT_EL0:
722 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
723 break;
724
725 case ESR_EC_IABORT_EL1:
726 #ifdef CONFIG_XNUPOST
727 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
728 break;
729 }
730 #endif /* CONFIG_XNUPOST */
731
732 panic_with_thread_kernel_state("Kernel instruction fetch abort", state);
733
734 case ESR_EC_PC_ALIGN:
735 handle_pc_align(state);
736 __builtin_unreachable();
737
738 case ESR_EC_DABORT_EL1:
739 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
740 break;
741
742 case ESR_EC_UNCATEGORIZED:
743 assert(!ESR_ISS(esr));
744
745 handle_uncategorized(&context->ss);
746 break;
747
748 case ESR_EC_SP_ALIGN:
749 handle_sp_align(state);
750 __builtin_unreachable();
751
752 case ESR_EC_BKPT_AARCH32:
753 handle_breakpoint(state, esr);
754 __builtin_unreachable();
755
756 case ESR_EC_BRK_AARCH64:
757 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
758 handle_kernel_breakpoint(state, esr);
759 #if CONFIG_UBSAN_MINIMAL
760 /* UBSan breakpoints are recoverable */
761 break;
762 #endif /* CONFIG_UBSAN_MINIMAL */
763 } else {
764 handle_breakpoint(state, esr);
765 __builtin_unreachable();
766 }
767
768 case ESR_EC_BKPT_REG_MATCH_EL0:
769 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
770 handle_breakpoint(state, esr);
771 }
772 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
773 class, state, class, esr, (void *)far);
774 __builtin_unreachable();
775
776 case ESR_EC_BKPT_REG_MATCH_EL1:
777 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state);
778 __builtin_unreachable();
779
780 case ESR_EC_SW_STEP_DEBUG_EL0:
781 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
782 handle_sw_step_debug(state);
783 }
784 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
785 class, state, class, esr, (void *)far);
786 __builtin_unreachable();
787
788 case ESR_EC_SW_STEP_DEBUG_EL1:
789 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state);
790 __builtin_unreachable();
791
792 case ESR_EC_WATCHPT_MATCH_EL0:
793 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
794 handle_watchpoint(far);
795 }
796 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
797 class, state, class, esr, (void *)far);
798 __builtin_unreachable();
799
800 case ESR_EC_WATCHPT_MATCH_EL1:
801 /*
802 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
803 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
804 */
805 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
806 arm_debug_set(NULL);
807 break; /* return to first level handler */
808 }
809 panic("Unsupported Class %u event code. state=%p class=%u esr=%u far=%p",
810 class, state, class, esr, (void *)far);
811 __builtin_unreachable();
812
813 case ESR_EC_TRAP_SIMD_FP:
814 handle_simd_trap(state, esr);
815 __builtin_unreachable();
816
817 case ESR_EC_ILLEGAL_INSTR_SET:
818 if (EXCB_ACTION_RERUN !=
819 ex_cb_invoke(EXCB_CLASS_ILLEGAL_INSTR_SET, far)) {
820 // instruction is not re-executed
821 panic("Illegal instruction set exception. state=%p class=%u esr=%u far=%p spsr=0x%x",
822 state, class, esr, (void *)far, get_saved_state_cpsr(state));
823 }
824 // must clear this fault in PSR to re-run
825 mask_saved_state_cpsr(state, 0, PSR64_IL);
826 break;
827
828 case ESR_EC_MCR_MRC_CP15_TRAP:
829 case ESR_EC_MCRR_MRRC_CP15_TRAP:
830 case ESR_EC_MCR_MRC_CP14_TRAP:
831 case ESR_EC_LDC_STC_CP14_TRAP:
832 case ESR_EC_MCRR_MRRC_CP14_TRAP:
833 handle_user_trapped_instruction32(state, esr);
834 __builtin_unreachable();
835
836 case ESR_EC_WFI_WFE:
837 // Use of WFI or WFE instruction when they have been disabled for EL0
838 handle_wf_trap(state);
839 __builtin_unreachable();
840
841 case ESR_EC_FLOATING_POINT_64:
842 handle_fp_trap(state, esr);
843 __builtin_unreachable();
844
845 default:
846 handle_uncategorized(state);
847 }
848
849 #ifdef CONFIG_XNUPOST
850 if (saved_expected_fault_handler != NULL) {
851 thread->machine.expected_fault_handler = saved_expected_fault_handler;
852 thread->machine.expected_fault_addr = saved_expected_fault_addr;
853 }
854 #endif /* CONFIG_XNUPOST */
855
856 if (is_user) {
857 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
858 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
859 esr, far, get_saved_state_pc(state), 0, 0);
860 thread->machine.exception_trace_code = 0;
861 } else {
862 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
863 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
864 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
865 }
866 #if MACH_ASSERT
867 if (preemption_level != sleh_get_preemption_level()) {
868 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
869 }
870 #endif
871 }
872
873 /*
874 * Uncategorized exceptions are a catch-all for general execution errors.
875 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
876 */
877 static void
handle_uncategorized(arm_saved_state_t * state)878 handle_uncategorized(arm_saved_state_t *state)
879 {
880 exception_type_t exception = EXC_BAD_INSTRUCTION;
881 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
882 mach_msg_type_number_t numcodes = 2;
883 uint32_t instr = 0;
884
885 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
886
887 #if CONFIG_DTRACE
888
889 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
890 /*
891 * For a 64bit user process, we care about all 4 bytes of the
892 * instr.
893 */
894 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
895 if (dtrace_user_probe(state) == KERN_SUCCESS) {
896 return;
897 }
898 }
899 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
900 /*
901 * For a 32bit user process, we check for thumb mode, in
902 * which case we only care about a 2 byte instruction length.
903 * For non-thumb mode, we care about all 4 bytes of the instructin.
904 */
905 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
906 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
907 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
908 if (dtrace_user_probe(state) == KERN_SUCCESS) {
909 return;
910 }
911 }
912 } else {
913 if ((instr == FASTTRAP_ARM32_INSTR) ||
914 (instr == FASTTRAP_ARM32_RET_INSTR)) {
915 if (dtrace_user_probe(state) == KERN_SUCCESS) {
916 return;
917 }
918 }
919 }
920 }
921
922 #endif /* CONFIG_DTRACE */
923
924 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
925 if (IS_ARM_GDB_TRAP(instr)) {
926 boolean_t interrupt_state;
927 exception = EXC_BREAKPOINT;
928
929 interrupt_state = ml_set_interrupts_enabled(FALSE);
930
931 /* Save off the context here (so that the debug logic
932 * can see the original state of this thread).
933 */
934 current_thread()->machine.kpcb = state;
935
936 /* Hop into the debugger (typically either due to a
937 * fatal exception, an explicit panic, or a stackshot
938 * request.
939 */
940 DebuggerCall(exception, state);
941
942 current_thread()->machine.kpcb = NULL;
943 (void) ml_set_interrupts_enabled(interrupt_state);
944 return;
945 } else {
946 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
947 }
948 }
949
950 /*
951 * Check for GDB breakpoint via illegal opcode.
952 */
953 if (IS_ARM_GDB_TRAP(instr)) {
954 exception = EXC_BREAKPOINT;
955 codes[0] = EXC_ARM_BREAKPOINT;
956 codes[1] = instr;
957 } else {
958 codes[1] = instr;
959 }
960
961 exception_triage(exception, codes, numcodes);
962 __builtin_unreachable();
963 }
964
965 #if __has_feature(ptrauth_calls)
966 static const uint16_t ptrauth_brk_comment_base = 0xc470;
967
968 static inline bool
brk_comment_is_ptrauth(uint16_t comment)969 brk_comment_is_ptrauth(uint16_t comment)
970 {
971 return comment >= ptrauth_brk_comment_base &&
972 comment <= ptrauth_brk_comment_base + ptrauth_key_asdb;
973 }
974
975 static inline const char *
ptrauth_key_to_string(ptrauth_key key)976 ptrauth_key_to_string(ptrauth_key key)
977 {
978 switch (key) {
979 case ptrauth_key_asia:
980 return "IA";
981 case ptrauth_key_asib:
982 return "IB";
983 case ptrauth_key_asda:
984 return "DA";
985 case ptrauth_key_asdb:
986 return "DB";
987 default:
988 __builtin_unreachable();
989 }
990 }
991 #endif /* __has_feature(ptrauth_calls) */
992
993 #if CONFIG_KERNEL_TBI && KASAN_TBI
994 static inline bool
brk_comment_is_kasan_failure(uint16_t comment)995 brk_comment_is_kasan_failure(uint16_t comment)
996 {
997 return comment >= KASAN_TBI_ESR_BASE &&
998 comment <= KASAN_TBI_ESR_TOP;
999 }
1000 #endif /* CONFIG_KERNEL_TBI && KASAN_TBI */
1001
1002 #if CONFIG_UBSAN_MINIMAL
1003 static inline bool
brk_comment_is_ubsan(uint16_t comment)1004 brk_comment_is_ubsan(uint16_t comment)
1005 {
1006 return comment >= UBSAN_MINIMAL_TRAPS_START &&
1007 comment < UBSAN_MINIMAL_TRAPS_END;
1008 }
1009 #endif /* CONFIG_UBSAN_MINIMAL */
1010
1011 static void
handle_kernel_breakpoint(arm_saved_state_t * state,uint32_t esr)1012 handle_kernel_breakpoint(arm_saved_state_t *state, uint32_t esr)
1013 {
1014 uint16_t comment = ISS_BRK_COMMENT(esr);
1015
1016 #if __has_feature(ptrauth_calls)
1017 if (brk_comment_is_ptrauth(comment)) {
1018 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Ptrauth failure with %s key resulted in 0x%016llx"
1019 char msg[strlen(MSG_FMT)
1020 - strlen("0x%04X") + strlen("0xFFFF")
1021 - strlen("%s") + strlen("IA")
1022 - strlen("0x%016llx") + strlen("0xFFFFFFFFFFFFFFFF")
1023 + 1];
1024 ptrauth_key key = (ptrauth_key)(comment - ptrauth_brk_comment_base);
1025 const char *key_str = ptrauth_key_to_string(key);
1026 snprintf(msg, sizeof(msg), MSG_FMT, comment, key_str, saved_state64(state)->x[16]);
1027
1028 panic_with_thread_kernel_state(msg, state);
1029 __builtin_unreachable();
1030 #undef MSG_FMT
1031 }
1032 #endif /* __has_feature(ptrauth_calls) */
1033
1034 #if CONFIG_KERNEL_TBI && KASAN_TBI
1035 if (brk_comment_is_kasan_failure(comment)) {
1036 kasan_handle_brk_failure(saved_state64(state)->x[0], comment);
1037 __builtin_unreachable();
1038 }
1039 #endif /* CONFIG_KERNEL_TBI && KASAN_TBI */
1040
1041 #if CONFIG_UBSAN_MINIMAL
1042 if (brk_comment_is_ubsan(comment)) {
1043 ubsan_handle_brk_trap(comment, get_saved_state_pc(state),
1044 get_saved_state_fp(state));
1045 add_saved_state_pc(state, 4);
1046 return;
1047 }
1048 #endif /* CONFIG_UBSAN_MINIMAL */
1049
1050 #define MSG_FMT "Break 0x%04X instruction exception from kernel. Panic (by design)"
1051 char msg[strlen(MSG_FMT) - strlen("0x%04X") + strlen("0xFFFF") + 1];
1052 snprintf(msg, sizeof(msg), MSG_FMT, comment);
1053 #undef MSG_FMT
1054
1055 panic_with_thread_kernel_state(msg, state);
1056 __builtin_unreachable();
1057 }
1058
1059 static void
handle_breakpoint(arm_saved_state_t * state,uint32_t esr __unused)1060 handle_breakpoint(arm_saved_state_t *state, uint32_t esr __unused)
1061 {
1062 exception_type_t exception = EXC_BREAKPOINT;
1063 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1064 mach_msg_type_number_t numcodes = 2;
1065
1066 #if __has_feature(ptrauth_calls)
1067 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1068 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1069 exception |= EXC_PTRAUTH_BIT;
1070 }
1071 #endif /* __has_feature(ptrauth_calls) */
1072
1073 codes[1] = get_saved_state_pc(state);
1074 exception_triage(exception, codes, numcodes);
1075 __builtin_unreachable();
1076 }
1077
1078 static void
handle_watchpoint(vm_offset_t fault_addr)1079 handle_watchpoint(vm_offset_t fault_addr)
1080 {
1081 exception_type_t exception = EXC_BREAKPOINT;
1082 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1083 mach_msg_type_number_t numcodes = 2;
1084
1085 codes[1] = fault_addr;
1086 exception_triage(exception, codes, numcodes);
1087 __builtin_unreachable();
1088 }
1089
1090 static void
handle_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,abort_inspector_t inspect_abort,abort_handler_t handler,expected_fault_handler_t expected_fault_handler)1091 handle_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1092 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1093 {
1094 fault_status_t fault_code;
1095 vm_prot_t fault_type;
1096
1097 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1098 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1099 }
1100
1101 static void
inspect_instruction_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1102 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1103 {
1104 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1105 *fault_code = ISS_IA_FSC(iss);
1106 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1107 }
1108
1109 static void
inspect_data_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1110 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1111 {
1112 getCpuDatap()->cpu_stat.data_ex_cnt++;
1113 *fault_code = ISS_DA_FSC(iss);
1114
1115 /*
1116 * Cache maintenance operations always report faults as write access.
1117 * Change these to read access, unless they report a permission fault.
1118 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1119 * access to the mapping, but if a cache maintenance operation that only requires
1120 * read access generates a permission fault, then we will not be able to handle
1121 * the fault regardless of whether we treat it as a read or write fault.
1122 */
1123 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
1124 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1125 } else {
1126 *fault_type = (VM_PROT_READ);
1127 }
1128 }
1129
1130 #if __has_feature(ptrauth_calls)
1131 static inline uint64_t
fault_addr_bitmask(unsigned int bit_from,unsigned int bit_to)1132 fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1133 {
1134 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1135 }
1136
1137 static inline bool
fault_addr_bit(vm_offset_t fault_addr,unsigned int bit)1138 fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1139 {
1140 return (bool)((fault_addr >> bit) & 1);
1141 }
1142
1143 extern int gARM_FEAT_PAuth2;
1144
1145 /**
1146 * Determines whether a fault address taken at EL0 contains a PAC error code
1147 * corresponding to the specified kind of ptrauth key.
1148 */
1149 static bool
user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr,bool data_key)1150 user_fault_addr_matches_pac_error_code(vm_offset_t fault_addr, bool data_key)
1151 {
1152 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1153 bool tbi = data_key || __improbable(instruction_tbi);
1154
1155 if (gARM_FEAT_PAuth2) {
1156 /*
1157 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1158 * treat all non-canonical address bits like potential poison bits.
1159 */
1160 uint64_t mask = fault_addr_bitmask(T0SZ_BOOT, 54);
1161 if (!tbi) {
1162 mask |= fault_addr_bitmask(56, 63);
1163 }
1164 return (fault_addr & mask) != 0;
1165 } else {
1166 unsigned int poison_shift;
1167 if (tbi) {
1168 poison_shift = 53;
1169 } else {
1170 poison_shift = 61;
1171 }
1172
1173 /* PAC error codes are always in the form key_number:NOT(key_number) */
1174 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1175 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1176 return poison_bit_1 != poison_bit_2;
1177 }
1178 }
1179 #endif /* __has_feature(ptrauth_calls) */
1180
1181 static void
handle_pc_align(arm_saved_state_t * ss)1182 handle_pc_align(arm_saved_state_t *ss)
1183 {
1184 exception_type_t exc;
1185 mach_exception_data_type_t codes[2];
1186 mach_msg_type_number_t numcodes = 2;
1187
1188 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1189 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
1190 }
1191
1192 exc = EXC_BAD_ACCESS;
1193 #if __has_feature(ptrauth_calls)
1194 if (user_fault_addr_matches_pac_error_code(get_saved_state_pc(ss), false)) {
1195 exc |= EXC_PTRAUTH_BIT;
1196 }
1197 #endif /* __has_feature(ptrauth_calls) */
1198
1199 codes[0] = EXC_ARM_DA_ALIGN;
1200 codes[1] = get_saved_state_pc(ss);
1201
1202 exception_triage(exc, codes, numcodes);
1203 __builtin_unreachable();
1204 }
1205
1206 static void
handle_sp_align(arm_saved_state_t * ss)1207 handle_sp_align(arm_saved_state_t *ss)
1208 {
1209 exception_type_t exc;
1210 mach_exception_data_type_t codes[2];
1211 mach_msg_type_number_t numcodes = 2;
1212
1213 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1214 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
1215 }
1216
1217 exc = EXC_BAD_ACCESS;
1218 #if __has_feature(ptrauth_calls)
1219 if (user_fault_addr_matches_pac_error_code(get_saved_state_sp(ss), true)) {
1220 exc |= EXC_PTRAUTH_BIT;
1221 }
1222 #endif /* __has_feature(ptrauth_calls) */
1223
1224 codes[0] = EXC_ARM_SP_ALIGN;
1225 codes[1] = get_saved_state_sp(ss);
1226
1227 exception_triage(exc, codes, numcodes);
1228 __builtin_unreachable();
1229 }
1230
1231 static void
handle_wf_trap(arm_saved_state_t * state)1232 handle_wf_trap(arm_saved_state_t *state)
1233 {
1234 exception_type_t exc;
1235 mach_exception_data_type_t codes[2];
1236 mach_msg_type_number_t numcodes = 2;
1237 uint32_t instr = 0;
1238
1239 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1240
1241 exc = EXC_BAD_INSTRUCTION;
1242 codes[0] = EXC_ARM_UNDEFINED;
1243 codes[1] = instr;
1244
1245 exception_triage(exc, codes, numcodes);
1246 __builtin_unreachable();
1247 }
1248
1249 static void
handle_fp_trap(arm_saved_state_t * state,uint32_t esr)1250 handle_fp_trap(arm_saved_state_t *state, uint32_t esr)
1251 {
1252 exception_type_t exc = EXC_ARITHMETIC;
1253 mach_exception_data_type_t codes[2];
1254 mach_msg_type_number_t numcodes = 2;
1255 uint32_t instr = 0;
1256
1257 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1258 panic_with_thread_kernel_state("Floating point exception from kernel", state);
1259 }
1260
1261 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1262 codes[1] = instr;
1263
1264 /* The floating point trap flags are only valid if TFV is set. */
1265 if (!fp_exceptions_enabled) {
1266 exc = EXC_BAD_INSTRUCTION;
1267 codes[0] = EXC_ARM_UNDEFINED;
1268 } else if (!(esr & ISS_FP_TFV)) {
1269 codes[0] = EXC_ARM_FP_UNDEFINED;
1270 } else if (esr & ISS_FP_UFF) {
1271 codes[0] = EXC_ARM_FP_UF;
1272 } else if (esr & ISS_FP_OFF) {
1273 codes[0] = EXC_ARM_FP_OF;
1274 } else if (esr & ISS_FP_IOF) {
1275 codes[0] = EXC_ARM_FP_IO;
1276 } else if (esr & ISS_FP_DZF) {
1277 codes[0] = EXC_ARM_FP_DZ;
1278 } else if (esr & ISS_FP_IDF) {
1279 codes[0] = EXC_ARM_FP_ID;
1280 } else if (esr & ISS_FP_IXF) {
1281 codes[0] = EXC_ARM_FP_IX;
1282 } else {
1283 panic("Unrecognized floating point exception, state=%p, esr=%#x", state, esr);
1284 }
1285
1286 exception_triage(exc, codes, numcodes);
1287 __builtin_unreachable();
1288 }
1289
1290
1291
1292 /*
1293 * handle_alignment_fault_from_user:
1294 * state: Saved state
1295 *
1296 * Attempts to deal with an alignment fault from userspace (possibly by
1297 * emulating the faulting instruction). If emulation failed due to an
1298 * unservicable fault, the ESR for that fault will be stored in the
1299 * recovery_esr field of the thread by the exception code.
1300 *
1301 * Returns:
1302 * -1: Emulation failed (emulation of state/instr not supported)
1303 * 0: Successfully emulated the instruction
1304 * EFAULT: Emulation failed (probably due to permissions)
1305 * EINVAL: Emulation failed (probably due to a bad address)
1306 */
1307
1308
1309 static int
handle_alignment_fault_from_user(arm_saved_state_t * state,kern_return_t * vmfr)1310 handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1311 {
1312 int ret = -1;
1313
1314 #pragma unused (state)
1315 #pragma unused (vmfr)
1316
1317 return ret;
1318 }
1319
1320
1321 static void
handle_sw_step_debug(arm_saved_state_t * state)1322 handle_sw_step_debug(arm_saved_state_t *state)
1323 {
1324 thread_t thread = current_thread();
1325 exception_type_t exc;
1326 mach_exception_data_type_t codes[2];
1327 mach_msg_type_number_t numcodes = 2;
1328
1329 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1330 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
1331 }
1332
1333 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1334 if (thread->machine.DebugData != NULL) {
1335 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1336 } else {
1337 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
1338 }
1339
1340 mask_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);
1341
1342 // Special encoding for gdb single step event on ARM
1343 exc = EXC_BREAKPOINT;
1344 codes[0] = 1;
1345 codes[1] = 0;
1346
1347 exception_triage(exc, codes, numcodes);
1348 __builtin_unreachable();
1349 }
1350
1351 static void
handle_user_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1352 handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1353 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1354 {
1355 exception_type_t exc = EXC_BAD_ACCESS;
1356 mach_exception_data_type_t codes[2];
1357 mach_msg_type_number_t numcodes = 2;
1358 thread_t thread = current_thread();
1359
1360 (void)esr;
1361 (void)expected_fault_handler;
1362
1363 if (ml_at_interrupt_context()) {
1364 panic_with_thread_kernel_state("Apparently on interrupt stack when taking user abort!\n", state);
1365 }
1366
1367 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1368
1369 if (!is_vm_fault(fault_code) &&
1370 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1371 thread_reset_pcs_done_faulting(thread);
1372 }
1373
1374 if (is_vm_fault(fault_code)) {
1375 vm_map_t map = thread->map;
1376 vm_offset_t vm_fault_addr = fault_addr;
1377 kern_return_t result = KERN_FAILURE;
1378
1379 assert(map != kernel_map);
1380
1381 if (!(fault_type & VM_PROT_EXECUTE)) {
1382 vm_fault_addr = tbi_clear(fault_addr);
1383 }
1384
1385 /* check to see if it is just a pmap ref/modify fault */
1386 if (!is_translation_fault(fault_code)) {
1387 result = arm_fast_fault(map->pmap,
1388 vm_fault_addr,
1389 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1390 }
1391 if (result != KERN_SUCCESS) {
1392
1393 {
1394 /* We have to fault the page in */
1395 result = vm_fault(map, vm_fault_addr, fault_type,
1396 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1397 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1398 }
1399 }
1400 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1401 thread_reset_pcs_done_faulting(thread);
1402 }
1403 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1404 return;
1405 }
1406
1407 /*
1408 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1409 * If it does, we're leaking preemption disables somewhere in the kernel.
1410 */
1411 if (__improbable(result == KERN_FAILURE)) {
1412 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1413 }
1414
1415 codes[0] = result;
1416 } else if (is_alignment_fault(fault_code)) {
1417 kern_return_t vmfkr = KERN_SUCCESS;
1418 thread->machine.recover_esr = 0;
1419 thread->machine.recover_far = 0;
1420 int result = handle_alignment_fault_from_user(state, &vmfkr);
1421 if (result == 0) {
1422 /* Successfully emulated, or instruction
1423 * copyin() for decode/emulation failed.
1424 * Continue, or redrive instruction.
1425 */
1426 thread_exception_return();
1427 } else if (((result == EFAULT) || (result == EINVAL)) &&
1428 (thread->machine.recover_esr == 0)) {
1429 /*
1430 * If we didn't actually take a fault, but got one of
1431 * these errors, then we failed basic sanity checks of
1432 * the fault address. Treat this as an invalid
1433 * address.
1434 */
1435 codes[0] = KERN_INVALID_ADDRESS;
1436 } else if ((result == EFAULT) &&
1437 (thread->machine.recover_esr)) {
1438 /*
1439 * Since alignment aborts are prioritized
1440 * ahead of translation aborts, the misaligned
1441 * atomic emulation flow may have triggered a
1442 * VM pagefault, which the VM could not resolve.
1443 * Report the VM fault error in codes[]
1444 */
1445
1446 codes[0] = vmfkr;
1447 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1448 /* Cause ESR_EC to reflect an EL0 abort */
1449 thread->machine.recover_esr &= ~ESR_EC_MASK;
1450 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1451 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr);
1452 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far);
1453 fault_addr = thread->machine.recover_far;
1454 } else {
1455 /* This was just an unsupported alignment
1456 * exception. Misaligned atomic emulation
1457 * timeouts fall in this category.
1458 */
1459 codes[0] = EXC_ARM_DA_ALIGN;
1460 }
1461 } else if (is_parity_error(fault_code)) {
1462 #if defined(APPLE_ARM64_ARCH_FAMILY)
1463 /*
1464 * Platform errors are handled in sleh_sync before interrupts are enabled.
1465 */
1466 #else
1467 panic("User parity error.");
1468 #endif
1469 } else {
1470 codes[0] = KERN_FAILURE;
1471 }
1472
1473 codes[1] = fault_addr;
1474 #if __has_feature(ptrauth_calls)
1475 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1476 if (user_fault_addr_matches_pac_error_code(fault_addr, is_data_abort)) {
1477 exc |= EXC_PTRAUTH_BIT;
1478 }
1479 #endif /* __has_feature(ptrauth_calls) */
1480 exception_triage(exc, codes, numcodes);
1481 __builtin_unreachable();
1482 }
1483
1484 static void
handle_kernel_abort_recover(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,thread_t thread)1485 handle_kernel_abort_recover(
1486 arm_saved_state_t *state,
1487 uint32_t esr,
1488 vm_offset_t fault_addr,
1489 thread_t thread)
1490 {
1491 thread->machine.recover_esr = esr;
1492 thread->machine.recover_far = fault_addr;
1493 #if defined(HAS_APPLE_PAC)
1494 MANIPULATE_SIGNED_THREAD_STATE(state,
1495 "mov x1, %[pc] \n"
1496 "str x1, [x0, %[SS64_PC]] \n",
1497 [pc] "r"(copyio_recovery_get_recover_addr(state))
1498 );
1499 #else
1500 saved_state64(state)->pc = copyio_recovery_get_recover_addr(state);
1501 #endif
1502 }
1503
1504 static void
handle_kernel_abort(arm_saved_state_t * state,uint32_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1505 handle_kernel_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr,
1506 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1507 {
1508 thread_t thread = current_thread();
1509 bool recover = find_copyio_recovery_entry(state) != 0;
1510
1511 #ifndef CONFIG_XNUPOST
1512 (void)expected_fault_handler;
1513 #endif /* CONFIG_XNUPOST */
1514
1515 #if CONFIG_DTRACE
1516 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
1517 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
1518 /*
1519 * Point to next instruction, or recovery handler if set.
1520 */
1521 if (recover) {
1522 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1523 } else {
1524 add_saved_state_pc(state, 4);
1525 }
1526 return;
1527 } else {
1528 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
1529 }
1530 }
1531 #endif
1532
1533 if (ml_at_interrupt_context()) {
1534 panic_with_thread_kernel_state("Unexpected abort while on interrupt stack.", state);
1535 }
1536
1537 if (is_vm_fault(fault_code)) {
1538 kern_return_t result = KERN_FAILURE;
1539 vm_map_t map;
1540 int interruptible;
1541
1542 /*
1543 * Ensure no faults in the physical aperture. This could happen if
1544 * a page table is incorrectly allocated from the read only region
1545 * when running with KTRR.
1546 */
1547
1548 #ifdef CONFIG_XNUPOST
1549 if (expected_fault_handler && expected_fault_handler(state)) {
1550 return;
1551 }
1552 #endif /* CONFIG_XNUPOST */
1553
1554 if (fault_addr >= gVirtBase && fault_addr < static_memory_end) {
1555 panic_with_thread_kernel_state("Unexpected fault in kernel static region\n", state);
1556 }
1557
1558 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) {
1559 /*
1560 * If no recovery handler is supplied, always drive the fault against
1561 * the kernel map. If the fault was taken against a userspace VA, indicating
1562 * an unprotected access to user address space, vm_fault() should fail and
1563 * ultimately lead to a panic here.
1564 */
1565 map = kernel_map;
1566 interruptible = THREAD_UNINT;
1567 } else {
1568 map = thread->map;
1569
1570 /**
1571 * In the case that the recovery handler is set (e.g., during copyio
1572 * and dtrace probes), we don't want the vm_fault() operation to be
1573 * aborted early. Those code paths can't handle restarting the
1574 * vm_fault() operation so don't allow it to return early without
1575 * creating the wanted mapping.
1576 */
1577 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
1578 }
1579
1580 /* check to see if it is just a pmap ref/modify fault */
1581 if (!is_translation_fault(fault_code)) {
1582 result = arm_fast_fault(map->pmap,
1583 fault_addr,
1584 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
1585 if (result == KERN_SUCCESS) {
1586 return;
1587 }
1588 }
1589
1590 if (result != KERN_PROTECTION_FAILURE) {
1591 /*
1592 * We have to "fault" the page in.
1593 */
1594 result = vm_fault(map, fault_addr, fault_type,
1595 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
1596 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1597 }
1598
1599 if (result == KERN_SUCCESS) {
1600 return;
1601 }
1602
1603 /*
1604 * If we have a recover handler, invoke it now.
1605 */
1606 if (recover) {
1607 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1608 return;
1609 }
1610 } else if (is_alignment_fault(fault_code)) {
1611 if (recover) {
1612 handle_kernel_abort_recover(state, esr, fault_addr, thread);
1613 return;
1614 }
1615 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
1616 } else if (is_parity_error(fault_code)) {
1617 #if defined(APPLE_ARM64_ARCH_FAMILY)
1618 /*
1619 * Platform errors are handled in sleh_sync before interrupts are enabled.
1620 */
1621 #else
1622 panic_with_thread_kernel_state("Kernel parity error.", state);
1623 #endif
1624 } else {
1625 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
1626 }
1627
1628 panic_with_thread_kernel_state("Kernel data abort.", state);
1629 }
1630
1631 extern void syscall_trace(struct arm_saved_state * regs);
1632
1633 static void
handle_svc(arm_saved_state_t * state)1634 handle_svc(arm_saved_state_t *state)
1635 {
1636 int trap_no = get_saved_state_svc_number(state);
1637 thread_t thread = current_thread();
1638 struct proc *p;
1639
1640 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
1641
1642 #define TRACE_SYSCALL 1
1643 #if TRACE_SYSCALL
1644 syscall_trace(state);
1645 #endif
1646
1647 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
1648
1649 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
1650 platform_syscall(state);
1651 panic("Returned from platform_syscall()?");
1652 }
1653
1654 mach_kauth_cred_thread_update();
1655
1656 if (trap_no < 0) {
1657 switch (trap_no) {
1658 case MACH_ARM_TRAP_ABSTIME:
1659 handle_mach_absolute_time_trap(state);
1660 return;
1661 case MACH_ARM_TRAP_CONTTIME:
1662 handle_mach_continuous_time_trap(state);
1663 return;
1664 }
1665
1666 /* Counting perhaps better in the handler, but this is how it's been done */
1667 thread->syscalls_mach++;
1668 mach_syscall(state);
1669 } else {
1670 /* Counting perhaps better in the handler, but this is how it's been done */
1671 thread->syscalls_unix++;
1672 p = get_bsdthreadtask_info(thread);
1673
1674 assert(p);
1675
1676 unix_syscall(state, thread, p);
1677 }
1678 }
1679
1680 static void
handle_mach_absolute_time_trap(arm_saved_state_t * state)1681 handle_mach_absolute_time_trap(arm_saved_state_t *state)
1682 {
1683 uint64_t now = mach_absolute_time();
1684 saved_state64(state)->x[0] = now;
1685 }
1686
1687 static void
handle_mach_continuous_time_trap(arm_saved_state_t * state)1688 handle_mach_continuous_time_trap(arm_saved_state_t *state)
1689 {
1690 uint64_t now = mach_continuous_time();
1691 saved_state64(state)->x[0] = now;
1692 }
1693
1694
1695 __attribute__((noreturn))
1696 static void
handle_msr_trap(arm_saved_state_t * state,uint32_t esr)1697 handle_msr_trap(arm_saved_state_t *state, uint32_t esr)
1698 {
1699 exception_type_t exception = EXC_BAD_INSTRUCTION;
1700 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1701 mach_msg_type_number_t numcodes = 2;
1702 uint32_t instr = 0;
1703
1704 if (!is_saved_state64(state)) {
1705 panic("MSR/MRS trap (ESR 0x%x) from 32-bit state", esr);
1706 }
1707
1708 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1709 panic("MSR/MRS trap (ESR 0x%x) from kernel", esr);
1710 }
1711
1712 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1713 codes[1] = instr;
1714
1715 exception_triage(exception, codes, numcodes);
1716 __builtin_unreachable();
1717 }
1718
1719 #if __has_feature(ptrauth_calls)
1720 static void
autxx_instruction_extract_reg(uint32_t instr,char reg[4])1721 autxx_instruction_extract_reg(uint32_t instr, char reg[4])
1722 {
1723 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
1724 switch (rd) {
1725 case 29:
1726 strncpy(reg, "fp", 4);
1727 return;
1728
1729 case 30:
1730 strncpy(reg, "lr", 4);
1731 return;
1732
1733 case 31:
1734 strncpy(reg, "xzr", 4);
1735 return;
1736
1737 default:
1738 snprintf(reg, 4, "x%u", rd);
1739 return;
1740 }
1741 }
1742
1743 static const char *
autix_system_instruction_extract_reg(uint32_t instr)1744 autix_system_instruction_extract_reg(uint32_t instr)
1745 {
1746 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
1747 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
1748 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
1749 return "x17";
1750 } else {
1751 return "lr";
1752 }
1753 }
1754
1755 static void
handle_pac_fail(arm_saved_state_t * state,uint32_t esr)1756 handle_pac_fail(arm_saved_state_t *state, uint32_t esr)
1757 {
1758 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
1759 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
1760 mach_msg_type_number_t numcodes = 2;
1761 uint32_t instr = 0;
1762
1763 if (!is_saved_state64(state)) {
1764 panic("PAC failure (ESR 0x%x) from 32-bit state", esr);
1765 }
1766
1767 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1768
1769 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1770 #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
1771 #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
1772 #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
1773
1774 char msg[strlen(AUTXX_MSG_FMT)
1775 - strlen("%s") + strlen("IA")
1776 - strlen("%s") + strlen("xzr")
1777 + 1];
1778 ptrauth_key key = (ptrauth_key)(esr & 0x3);
1779 const char *key_str = ptrauth_key_to_string(key);
1780
1781 if (ARM64_INSTR_IS_AUTxx(instr)) {
1782 char reg[4];
1783 autxx_instruction_extract_reg(instr, reg);
1784 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1785 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
1786 const char *reg = autix_system_instruction_extract_reg(instr);
1787 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
1788 } else {
1789 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
1790 }
1791 panic_with_thread_kernel_state(msg, state);
1792 }
1793
1794 codes[1] = instr;
1795
1796 exception_triage(exception, codes, numcodes);
1797 __builtin_unreachable();
1798 }
1799 #endif /* __has_feature(ptrauth_calls) */
1800
1801 static void
handle_user_trapped_instruction32(arm_saved_state_t * state,uint32_t esr)1802 handle_user_trapped_instruction32(arm_saved_state_t *state, uint32_t esr)
1803 {
1804 exception_type_t exception = EXC_BAD_INSTRUCTION;
1805 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1806 mach_msg_type_number_t numcodes = 2;
1807 uint32_t instr;
1808
1809 if (is_saved_state64(state)) {
1810 panic("ESR (0x%x) for instruction trapped from U32, but saved state is 64-bit.", esr);
1811 }
1812
1813 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1814 panic("ESR (0x%x) for instruction trapped from U32, actually came from kernel?", esr);
1815 }
1816
1817 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1818 codes[1] = instr;
1819
1820 exception_triage(exception, codes, numcodes);
1821 __builtin_unreachable();
1822 }
1823
1824 static void
handle_simd_trap(arm_saved_state_t * state,uint32_t esr)1825 handle_simd_trap(arm_saved_state_t *state, uint32_t esr)
1826 {
1827 exception_type_t exception = EXC_BAD_INSTRUCTION;
1828 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1829 mach_msg_type_number_t numcodes = 2;
1830 uint32_t instr = 0;
1831
1832 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1833 panic("ESR (0x%x) for SIMD trap from userland, actually came from kernel?", esr);
1834 }
1835
1836 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1837 codes[1] = instr;
1838
1839 exception_triage(exception, codes, numcodes);
1840 __builtin_unreachable();
1841 }
1842
1843 void
sleh_irq(arm_saved_state_t * state)1844 sleh_irq(arm_saved_state_t *state)
1845 {
1846 cpu_data_t * cdp __unused = getCpuDatap();
1847 #if MACH_ASSERT
1848 int preemption_level = sleh_get_preemption_level();
1849 #endif
1850
1851
1852 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
1853
1854 #if USE_APPLEARMSMP
1855 PE_handle_ext_interrupt();
1856 #else
1857 /* Run the registered interrupt handler. */
1858 cdp->interrupt_handler(cdp->interrupt_target,
1859 cdp->interrupt_refCon,
1860 cdp->interrupt_nub,
1861 cdp->interrupt_source);
1862 #endif
1863
1864 entropy_collect();
1865
1866
1867 sleh_interrupt_handler_epilogue();
1868 #if MACH_ASSERT
1869 if (preemption_level != sleh_get_preemption_level()) {
1870 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
1871 }
1872 #endif
1873 }
1874
1875 void
sleh_fiq(arm_saved_state_t * state)1876 sleh_fiq(arm_saved_state_t *state)
1877 {
1878 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
1879 #if MACH_ASSERT
1880 int preemption_level = sleh_get_preemption_level();
1881 #endif
1882
1883 #if MONOTONIC_FIQ
1884 uint64_t pmcr0 = 0, upmsr = 0;
1885 #endif /* MONOTONIC_FIQ */
1886
1887 #if defined(HAS_IPI)
1888 boolean_t is_ipi = FALSE;
1889 uint64_t ipi_sr = 0;
1890
1891 if (gFastIPI) {
1892 MRS(ipi_sr, "S3_5_C15_C1_1");
1893
1894 if (ipi_sr & 1) {
1895 is_ipi = TRUE;
1896 }
1897 }
1898
1899 if (is_ipi) {
1900 type = DBG_INTR_TYPE_IPI;
1901 } else
1902 #endif /* defined(HAS_IPI) */
1903 if (ml_get_timer_pending()) {
1904 type = DBG_INTR_TYPE_TIMER;
1905 }
1906 #if MONOTONIC_FIQ
1907 /* Consult the PMI sysregs last, after IPI/timer
1908 * classification.
1909 */
1910 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
1911 type = DBG_INTR_TYPE_PMI;
1912 }
1913 #endif /* MONOTONIC_FIQ */
1914
1915 sleh_interrupt_handler_prologue(state, type);
1916
1917 #if APPLEVIRTUALPLATFORM
1918 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
1919 #endif
1920
1921 #if defined(HAS_IPI)
1922 if (is_ipi) {
1923 /*
1924 * Order is important here: we must ack the IPI by writing IPI_SR
1925 * before we call cpu_signal_handler(). Otherwise, there will be
1926 * a window between the completion of pending-signal processing in
1927 * cpu_signal_handler() and the ack during which a newly-issued
1928 * IPI to this CPU may be lost. ISB is required to ensure the msr
1929 * is retired before execution of cpu_signal_handler().
1930 */
1931 MSR("S3_5_C15_C1_1", ipi_sr);
1932 __builtin_arm_isb(ISB_SY);
1933 cpu_signal_handler();
1934 } else
1935 #endif /* defined(HAS_IPI) */
1936 #if MONOTONIC_FIQ
1937 if (type == DBG_INTR_TYPE_PMI) {
1938 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
1939 mt_fiq(getCpuDatap(), pmcr0, upmsr);
1940 INTERRUPT_MASKED_DEBUG_END();
1941 } else
1942 #endif /* MONOTONIC_FIQ */
1943 {
1944 /*
1945 * We don't know that this is a timer, but we don't have insight into
1946 * the other interrupts that go down this path.
1947 */
1948
1949 cpu_data_t *cdp = getCpuDatap();
1950
1951 cdp->cpu_decrementer = -1; /* Large */
1952
1953 /*
1954 * ARM64_TODO: whether we're coming from userland is ignored right now.
1955 * We can easily thread it through, but not bothering for the
1956 * moment (AArch32 doesn't either).
1957 */
1958 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
1959 rtclock_intr(TRUE);
1960 INTERRUPT_MASKED_DEBUG_END();
1961 }
1962
1963 #if APPLEVIRTUALPLATFORM
1964 if (iar != GIC_SPURIOUS_IRQ) {
1965 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
1966 __builtin_arm_isb(ISB_SY);
1967 }
1968 #endif
1969
1970 sleh_interrupt_handler_epilogue();
1971 #if MACH_ASSERT
1972 if (preemption_level != sleh_get_preemption_level()) {
1973 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
1974 }
1975 #endif
1976 }
1977
1978 void
sleh_serror(arm_context_t * context,uint32_t esr,vm_offset_t far)1979 sleh_serror(arm_context_t *context, uint32_t esr, vm_offset_t far)
1980 {
1981 task_vtimer_check(current_thread());
1982
1983 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
1984 esr, VM_KERNEL_ADDRHIDE(far));
1985 arm_saved_state_t *state = &context->ss;
1986 #if MACH_ASSERT
1987 int preemption_level = sleh_get_preemption_level();
1988 #endif
1989
1990
1991 ASSERT_CONTEXT_SANITY(context);
1992 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC);
1993 #if MACH_ASSERT
1994 if (preemption_level != sleh_get_preemption_level()) {
1995 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
1996 }
1997 #endif
1998 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
1999 esr, VM_KERNEL_ADDRHIDE(far));
2000 }
2001
2002 void
mach_syscall_trace_exit(unsigned int retval,unsigned int call_number)2003 mach_syscall_trace_exit(unsigned int retval,
2004 unsigned int call_number)
2005 {
2006 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2007 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
2008 DBG_FUNC_END, retval, 0, 0, 0, 0);
2009 }
2010
2011 __attribute__((noreturn))
2012 void
thread_syscall_return(kern_return_t error)2013 thread_syscall_return(kern_return_t error)
2014 {
2015 thread_t thread;
2016 struct arm_saved_state *state;
2017
2018 thread = current_thread();
2019 state = get_user_regs(thread);
2020
2021 assert(is_saved_state64(state));
2022 saved_state64(state)->x[0] = error;
2023
2024 #if MACH_ASSERT
2025 kern_allocation_name_t
2026 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
2027 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2028 #endif /* MACH_ASSERT */
2029
2030 if (kdebug_enable) {
2031 /* Invert syscall number (negative for a mach syscall) */
2032 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
2033 }
2034
2035 thread_exception_return();
2036 }
2037
2038 void
syscall_trace(struct arm_saved_state * regs __unused)2039 syscall_trace(
2040 struct arm_saved_state * regs __unused)
2041 {
2042 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2043 }
2044
2045 static void
sleh_interrupt_handler_prologue(arm_saved_state_t * state,unsigned int type)2046 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2047 {
2048 boolean_t is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2049
2050 task_vtimer_check(current_thread());
2051
2052 uint64_t pc = is_user ? get_saved_state_pc(state) :
2053 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2054
2055 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2056 0, pc, is_user, type);
2057
2058 #if CONFIG_TELEMETRY
2059 if (telemetry_needs_record) {
2060 telemetry_mark_curthread(is_user, FALSE);
2061 }
2062 #endif /* CONFIG_TELEMETRY */
2063 }
2064
2065 static void
sleh_interrupt_handler_epilogue(void)2066 sleh_interrupt_handler_epilogue(void)
2067 {
2068 #if KPERF
2069 kperf_interrupt();
2070 #endif /* KPERF */
2071 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2072 }
2073
2074 void
sleh_invalid_stack(arm_context_t * context,uint32_t esr __unused,vm_offset_t far __unused)2075 sleh_invalid_stack(arm_context_t *context, uint32_t esr __unused, vm_offset_t far __unused)
2076 {
2077 thread_t thread = current_thread();
2078 vm_offset_t kernel_stack_bottom, sp;
2079
2080 sp = get_saved_state_sp(&context->ss);
2081 kernel_stack_bottom = round_page(thread->machine.kstackptr) - KERNEL_STACK_SIZE;
2082
2083 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2084 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
2085 }
2086
2087 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
2088 }
2089
2090