1 /*
2 * Copyright (c) 2012-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/caches_internal.h>
30 #include <arm/cpu_data.h>
31 #include <arm/cpu_data_internal.h>
32 #include <arm/misc_protos.h>
33 #include <arm/thread.h>
34 #include <arm/rtclock.h>
35 #include <arm/trap_internal.h> /* for IS_ARM_GDB_TRAP() et al */
36 #include <arm64/proc_reg.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/monotonic.h>
39 #include <arm64/instructions.h>
40
41 #include <kern/debug.h>
42 #include <kern/exc_guard.h>
43 #include <kern/restartable.h>
44 #include <kern/socd_client.h>
45 #include <kern/task.h>
46 #include <kern/thread.h>
47 #include <kern/zalloc_internal.h>
48 #include <mach/exception.h>
49 #include <mach/arm/traps.h>
50 #include <mach/vm_types.h>
51 #include <mach/machine/thread_status.h>
52
53 #include <machine/atomic.h>
54 #include <machine/limits.h>
55
56 #include <pexpert/arm/protos.h>
57 #include <pexpert/arm64/apple_arm64_cpu.h>
58 #include <pexpert/arm64/apple_arm64_regs.h>
59 #include <pexpert/arm64/board_config.h>
60
61 #include <vm/vm_page.h>
62 #include <vm/pmap.h>
63 #include <vm/vm_fault.h>
64 #include <vm/vm_kern.h>
65 #include <vm/vm_map_xnu.h>
66
67 #include <sys/errno.h>
68 #include <sys/kdebug.h>
69 #include <sys/code_signing.h>
70 #include <sys/reason.h>
71 #include <kperf/kperf.h>
72
73 #include <kern/policy_internal.h>
74 #if CONFIG_TELEMETRY
75 #include <kern/telemetry.h>
76 #include <kern/trap_telemetry.h>
77 #endif
78
79 #include <prng/entropy.h>
80
81
82
83
84 #include <arm64/platform_error_handler.h>
85
86 #if KASAN_TBI
87 #include <san/kasan.h>
88 #endif /* KASAN_TBI */
89
90 #if CONFIG_UBSAN_MINIMAL
91 #include <san/ubsan_minimal.h>
92 #endif
93
94
95
96 #ifdef CONFIG_BTI_TELEMETRY
97 #include <arm64/bti_telemetry.h>
98 #endif /* CONFIG_BTI_TELEMETRY */
99
100 #ifndef __arm64__
101 #error Should only be compiling for arm64.
102 #endif
103
104 #if DEBUG || DEVELOPMENT
105 #define HAS_TELEMETRY_KERNEL_BRK 1
106 #endif
107
108
109 #define TEST_CONTEXT32_SANITY(context) \
110 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \
111 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT)
112
113 #define TEST_CONTEXT64_SANITY(context) \
114 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \
115 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT)
116
117 #define ASSERT_CONTEXT_SANITY(context) \
118 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context))
119
120
121 #define COPYIN(src, dst, size) \
122 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
123 copyin_kern(src, dst, size) : \
124 copyin(src, dst, size)
125
126 #define COPYOUT(src, dst, size) \
127 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \
128 copyout_kern(src, dst, size) : \
129 copyout(src, dst, size)
130
131 // Below is for concatenating a string param to a string literal
132 #define STR1(x) #x
133 #define STR(x) STR1(x)
134
135 #define ARM64_KDBG_CODE_KERNEL (0 << 8)
136 #define ARM64_KDBG_CODE_USER (1 << 8)
137 #define ARM64_KDBG_CODE_GUEST (2 << 8)
138
139 _Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range");
140 _Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range");
141
142 void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike;
143
144 void sleh_synchronous_sp1(arm_context_t *, uint64_t, vm_offset_t) __abortlike;
145 void sleh_synchronous(arm_context_t *, uint64_t, vm_offset_t, bool);
146
147
148
149 void sleh_irq(arm_saved_state_t *);
150 void sleh_fiq(arm_saved_state_t *);
151 void sleh_serror(arm_context_t *context, uint64_t esr, vm_offset_t far);
152 void sleh_invalid_stack(arm_context_t *context, uint64_t esr, vm_offset_t far) __dead2;
153
154 static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type);
155 static void sleh_interrupt_handler_epilogue(void);
156
157 static void handle_svc(arm_saved_state_t *);
158 static void handle_mach_absolute_time_trap(arm_saved_state_t *);
159 static void handle_mach_continuous_time_trap(arm_saved_state_t *);
160
161 static void handle_msr_trap(arm_saved_state_t *state, uint64_t esr);
162 #if __has_feature(ptrauth_calls)
163 static void handle_pac_fail(arm_saved_state_t *state, uint64_t esr) __dead2;
164 static inline uint64_t fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to);
165 #endif
166 static void handle_bti_fail(arm_saved_state_t *state, uint64_t esr);
167 extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);
168
169 static void handle_uncategorized(arm_saved_state_t *);
170
171 static void handle_kernel_breakpoint(arm_saved_state_t *, uint64_t);
172
173 static void handle_breakpoint(arm_saved_state_t *, uint64_t) __dead2;
174
175 typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);
176 static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *);
177 static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *);
178
179 static int is_vm_fault(fault_status_t);
180 static int is_translation_fault(fault_status_t);
181 static int is_alignment_fault(fault_status_t);
182
183 typedef void (*abort_handler_t)(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
184 static void handle_user_abort(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
185 static void handle_kernel_abort(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);
186
187 static void handle_pc_align(arm_saved_state_t *ss) __dead2;
188 static void handle_sp_align(arm_saved_state_t *ss) __dead2;
189 static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2;
190 static void handle_wf_trap(arm_saved_state_t *ss) __dead2;
191 static void handle_fp_trap(arm_saved_state_t *ss, uint64_t esr) __dead2;
192 #if HAS_ARM_FEAT_SME
193 static void handle_sme_trap(arm_saved_state_t *state, uint64_t esr);
194 #endif /* HAS_ARM_FEAT_SME */
195
196 static void handle_watchpoint(vm_offset_t fault_addr) __dead2;
197
198 static void handle_abort(arm_saved_state_t *, uint64_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t);
199
200 static void handle_user_trapped_instruction32(arm_saved_state_t *, uint64_t esr) __dead2;
201
202 static void handle_simd_trap(arm_saved_state_t *, uint64_t esr) __dead2;
203
204 extern void current_cached_proc_cred_update(void);
205 void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number);
206
207 struct proc;
208
209 typedef uint32_t arm64_instr_t;
210
211 extern void
212 unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);
213
214 extern void
215 mach_syscall(struct arm_saved_state*);
216
217 #if CONFIG_SPTM
218 bool sleh_panic_lockdown_should_initiate_el1_sp0_sync(uint64_t esr, uint64_t elr, uint64_t far, uint64_t spsr);
219 #endif /* CONFIG_SPTM */
220
221 #if CONFIG_DTRACE
222 extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);
223 extern boolean_t dtrace_tally_fault(user_addr_t);
224
225 /*
226 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy
227 * and paste the trap instructions
228 * over from that file. Need to keep these in sync!
229 */
230 #define FASTTRAP_ARM32_INSTR 0xe7ffdefc
231 #define FASTTRAP_THUMB32_INSTR 0xdefc
232 #define FASTTRAP_ARM64_INSTR 0xe7eeee7e
233
234 #define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb
235 #define FASTTRAP_THUMB32_RET_INSTR 0xdefb
236 #define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d
237
238 /* See <rdar://problem/4613924> */
239 perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
240 #endif
241
242
243
244 extern void arm64_thread_exception_return(void) __dead2;
245
246 #if defined(APPLETYPHOON)
247 #define CPU_NAME "Typhoon"
248 #elif defined(APPLETWISTER)
249 #define CPU_NAME "Twister"
250 #elif defined(APPLEHURRICANE)
251 #define CPU_NAME "Hurricane"
252 #elif defined(APPLELIGHTNING)
253 #define CPU_NAME "Lightning"
254 #elif defined(APPLEEVEREST)
255 #define CPU_NAME "Everest"
256 #elif defined(APPLEH16)
257 #define CPU_NAME "AppleH16"
258 #else
259 #define CPU_NAME "Unknown"
260 #endif
261
262 #if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT))
263 #define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400)
264 #define ESR_WT_REASON(esr) ((esr) & 0xff)
265
266 #define WT_REASON_NONE 0
267 #define WT_REASON_INTEGRITY_FAIL 1
268 #define WT_REASON_BAD_SYSCALL 2
269 #define WT_REASON_NOT_LOCKED 3
270 #define WT_REASON_ALREADY_LOCKED 4
271 #define WT_REASON_SW_REQ 5
272 #define WT_REASON_PT_INVALID 6
273 #define WT_REASON_PT_VIOLATION 7
274 #define WT_REASON_REG_VIOLATION 8
275 #endif
276
277 #if defined(HAS_IPI)
278 void cpu_signal_handler(void);
279 extern unsigned int gFastIPI;
280 #endif /* defined(HAS_IPI) */
281
282 static arm_saved_state64_t *original_faulting_state = NULL;
283
284
285 TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false);
286
287 extern const vm_map_address_t physmap_base;
288 extern const vm_map_address_t physmap_end;
289 extern vm_offset_t static_memory_end;
290
291 /*
292 * Fault copyio_recovery_entry in copyin/copyout routines.
293 *
294 * Offets are expressed in bytes from ©_recovery_table
295 */
296 struct copyio_recovery_entry {
297 ptrdiff_t cre_start;
298 ptrdiff_t cre_end;
299 ptrdiff_t cre_recovery;
300 };
301
302 extern struct copyio_recovery_entry copyio_recover_table[];
303 extern struct copyio_recovery_entry copyio_recover_table_end[];
304
305 static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)306 copyio_recovery_offset(uintptr_t addr)
307 {
308 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
309 }
310
311 #if !HAS_APPLE_PAC
312 static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)313 copyio_recovery_addr(ptrdiff_t offset)
314 {
315 return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
316 }
317 #endif
318
319 static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(uint64_t pc)320 find_copyio_recovery_entry(uint64_t pc)
321 {
322 ptrdiff_t offset = copyio_recovery_offset(pc);
323 struct copyio_recovery_entry *e;
324
325 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
326 if (offset >= e->cre_start && offset < e->cre_end) {
327 return e;
328 }
329 }
330
331 return NULL;
332 }
333
334 static inline int
is_vm_fault(fault_status_t status)335 is_vm_fault(fault_status_t status)
336 {
337 switch (status) {
338 case FSC_TRANSLATION_FAULT_L0:
339 case FSC_TRANSLATION_FAULT_L1:
340 case FSC_TRANSLATION_FAULT_L2:
341 case FSC_TRANSLATION_FAULT_L3:
342 case FSC_ACCESS_FLAG_FAULT_L1:
343 case FSC_ACCESS_FLAG_FAULT_L2:
344 case FSC_ACCESS_FLAG_FAULT_L3:
345 case FSC_PERMISSION_FAULT_L1:
346 case FSC_PERMISSION_FAULT_L2:
347 case FSC_PERMISSION_FAULT_L3:
348 return TRUE;
349 default:
350 return FALSE;
351 }
352 }
353
354 static inline int
is_translation_fault(fault_status_t status)355 is_translation_fault(fault_status_t status)
356 {
357 switch (status) {
358 case FSC_TRANSLATION_FAULT_L0:
359 case FSC_TRANSLATION_FAULT_L1:
360 case FSC_TRANSLATION_FAULT_L2:
361 case FSC_TRANSLATION_FAULT_L3:
362 return TRUE;
363 default:
364 return FALSE;
365 }
366 }
367
368 static inline int
is_permission_fault(fault_status_t status)369 is_permission_fault(fault_status_t status)
370 {
371 switch (status) {
372 case FSC_PERMISSION_FAULT_L1:
373 case FSC_PERMISSION_FAULT_L2:
374 case FSC_PERMISSION_FAULT_L3:
375 return TRUE;
376 default:
377 return FALSE;
378 }
379 }
380
381 static inline int
is_alignment_fault(fault_status_t status)382 is_alignment_fault(fault_status_t status)
383 {
384 return status == FSC_ALIGNMENT_FAULT;
385 }
386
387 static inline int
is_parity_error(fault_status_t status)388 is_parity_error(fault_status_t status)
389 {
390 switch (status) {
391 #if defined(ARM64_BOARD_CONFIG_T6020)
392 /*
393 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented,
394 * FSC_SYNC_PARITY_X can be reported for data and instruction aborts
395 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
396 */
397 #else
398 /*
399 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
400 * only applicable to AArch32 HSR register. Can this be removed?
401 */
402 case FSC_ASYNC_PARITY:
403 case FSC_SYNC_PARITY:
404 case FSC_SYNC_PARITY_TT_L1:
405 case FSC_SYNC_PARITY_TT_L2:
406 case FSC_SYNC_PARITY_TT_L3:
407 return TRUE;
408 #endif
409 default:
410 return FALSE;
411 }
412 }
413
414 static inline int
is_sync_external_abort(fault_status_t status)415 is_sync_external_abort(fault_status_t status)
416 {
417 switch (status) {
418 #if defined(ARM64_BOARD_CONFIG_T6020)
419 /*
420 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented,
421 * FSC_SYNC_PARITY_x can be reported for data and instruction aborts
422 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
423 */
424 case FSC_SYNC_PARITY:
425 #endif /* defined(ARM64_BOARD_CONFIG_T6020) */
426 case FSC_SYNC_EXT_ABORT:
427 return TRUE;
428 default:
429 return FALSE;
430 }
431 }
432
433 static inline int
is_table_walk_error(fault_status_t status)434 is_table_walk_error(fault_status_t status)
435 {
436 switch (status) {
437 case FSC_SYNC_EXT_ABORT_TT_L1:
438 case FSC_SYNC_EXT_ABORT_TT_L2:
439 case FSC_SYNC_EXT_ABORT_TT_L3:
440 #if defined(ARM64_BOARD_CONFIG_T6020)
441 /*
442 * H14 Erratum(rdar://61553243): Despite having FEAT_RAS implemented,
443 * FSC_SYNC_PARITY_x can be reported for data and instruction aborts
444 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
445 */
446 case FSC_SYNC_PARITY_TT_L1:
447 case FSC_SYNC_PARITY_TT_L2:
448 case FSC_SYNC_PARITY_TT_L3:
449 #endif /* defined(ARM64_BOARD_CONFIG_T6020) */
450 return TRUE;
451 default:
452 return FALSE;
453 }
454 }
455
456
457
458 static inline int
is_servicible_fault(fault_status_t status)459 is_servicible_fault(fault_status_t status)
460 {
461 return is_vm_fault(status);
462 }
463
464 __dead2 __unused
465 static void
arm64_implementation_specific_error(arm_saved_state_t * state,uint64_t esr,vm_offset_t far)466 arm64_implementation_specific_error(arm_saved_state_t *state, uint64_t esr, vm_offset_t far)
467 {
468 #pragma unused (state, esr, far)
469 panic_plain("Unhandled implementation specific error\n");
470 }
471
472 #if CONFIG_KERNEL_INTEGRITY
473 #pragma clang diagnostic push
474 #pragma clang diagnostic ignored "-Wunused-parameter"
475 static void
kernel_integrity_error_handler(uint64_t esr,vm_offset_t far)476 kernel_integrity_error_handler(uint64_t esr, vm_offset_t far)
477 {
478 #if defined(KERNEL_INTEGRITY_WT)
479 #if (DEVELOPMENT || DEBUG)
480 if (ESR_WT_SERROR(esr)) {
481 switch (ESR_WT_REASON(esr)) {
482 case WT_REASON_INTEGRITY_FAIL:
483 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
484 case WT_REASON_BAD_SYSCALL:
485 panic_plain("Kernel integrity, bad syscall.");
486 case WT_REASON_NOT_LOCKED:
487 panic_plain("Kernel integrity, not locked.");
488 case WT_REASON_ALREADY_LOCKED:
489 panic_plain("Kernel integrity, already locked.");
490 case WT_REASON_SW_REQ:
491 panic_plain("Kernel integrity, software request.");
492 case WT_REASON_PT_INVALID:
493 panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
494 "walking 0x%016lx.", far);
495 case WT_REASON_PT_VIOLATION:
496 panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
497 far);
498 case WT_REASON_REG_VIOLATION:
499 panic_plain("Kernel integrity, violation in system register %d.",
500 (unsigned) far);
501 default:
502 panic_plain("Kernel integrity, unknown (esr=0x%08llx).", esr);
503 }
504 }
505 #else
506 if (ESR_WT_SERROR(esr)) {
507 panic_plain("SError esr: 0x%08llx far: 0x%016lx.", esr, far);
508 }
509 #endif
510 #endif
511 }
512 #pragma clang diagnostic pop
513 #endif
514
515 static void
arm64_platform_error(arm_saved_state_t * state,uint64_t esr,vm_offset_t far,platform_error_source_t source)516 arm64_platform_error(arm_saved_state_t *state, uint64_t esr, vm_offset_t far, platform_error_source_t source)
517 {
518 #if CONFIG_KERNEL_INTEGRITY
519 kernel_integrity_error_handler(esr, far);
520 #endif
521
522 (void)source;
523 cpu_data_t *cdp = getCpuDatap();
524
525 if (PE_handle_platform_error(far)) {
526 return;
527 } else if (cdp->platform_error_handler != NULL) {
528 cdp->platform_error_handler(cdp->cpu_id, far);
529 } else {
530 arm64_implementation_specific_error(state, esr, far);
531 }
532 }
533
534 void
panic_with_thread_kernel_state(const char * msg,arm_saved_state_t * ss)535 panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
536 {
537 boolean_t ss_valid;
538
539 ss_valid = is_saved_state64(ss);
540 arm_saved_state64_t *state = saved_state64(ss);
541
542 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);
543
544 // rdar://80659177
545 // Read SoCD tracepoints up to twice — once the first time we call panic and
546 // another time if we encounter a nested panic after that.
547 static int twice = 2;
548 if (twice > 0) {
549 twice--;
550 SOCD_TRACE_XNU(KERNEL_STATE_PANIC,
551 SOCD_TRACE_MODE_STICKY_TRACEPOINT,
552 ADDR(state->pc),
553 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
554 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
555 VALUE(state->far));
556 }
557
558
559 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
560 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n"
561 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n"
562 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n"
563 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n"
564 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n"
565 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n"
566 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n"
567 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n"
568 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%016llx far: 0x%016llx\n",
569 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"),
570 state->x[0], state->x[1], state->x[2], state->x[3],
571 state->x[4], state->x[5], state->x[6], state->x[7],
572 state->x[8], state->x[9], state->x[10], state->x[11],
573 state->x[12], state->x[13], state->x[14], state->x[15],
574 state->x[16], state->x[17], state->x[18], state->x[19],
575 state->x[20], state->x[21], state->x[22], state->x[23],
576 state->x[24], state->x[25], state->x[26], state->x[27],
577 state->x[28], state->fp, state->lr, state->sp,
578 state->pc, state->cpsr, state->esr, state->far);
579 }
580
581 void
sleh_synchronous_sp1(arm_context_t * context,uint64_t esr,vm_offset_t far __unused)582 sleh_synchronous_sp1(arm_context_t *context, uint64_t esr, vm_offset_t far __unused)
583 {
584 esr_exception_class_t class = ESR_EC(esr);
585 arm_saved_state_t * state = &context->ss;
586
587 switch (class) {
588 case ESR_EC_UNCATEGORIZED:
589 {
590 #if (DEVELOPMENT || DEBUG)
591 uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
592 if (IS_ARM_GDB_TRAP(instr)) {
593 DebuggerCall(EXC_BREAKPOINT, state);
594 }
595 OS_FALLTHROUGH; // panic if we return from the debugger
596 #else
597 panic_with_thread_kernel_state("Unexpected debugger trap while SP1 selected", state);
598 #endif /* (DEVELOPMENT || DEBUG) */
599 }
600 default:
601 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
602 }
603 }
604
605
606 __attribute__((noreturn))
607 void
thread_exception_return()608 thread_exception_return()
609 {
610 thread_t thread = current_thread();
611 if (thread->machine.exception_trace_code != 0) {
612 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
613 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
614 thread->machine.exception_trace_code = 0;
615 }
616
617 #if KASAN_TBI
618 kasan_unpoison_curstack(true);
619 #endif /* KASAN_TBI */
620 arm64_thread_exception_return();
621 __builtin_unreachable();
622 }
623
624 /*
625 * check whether task vtimers are running and set thread and CPU BSD AST
626 *
627 * must be called with interrupts masked so updates of fields are atomic
628 * must be emitted inline to avoid generating an FBT probe on the exception path
629 *
630 */
631 __attribute__((__always_inline__))
632 static inline void
task_vtimer_check(thread_t thread)633 task_vtimer_check(thread_t thread)
634 {
635 task_t task = get_threadtask_early(thread);
636
637 if (__improbable(task != NULL && task->vtimers)) {
638 thread_ast_set(thread, AST_BSD);
639 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
640 }
641 }
642
643 #if MACH_ASSERT
644 /**
645 * A version of get_preemption_level() that works in early boot.
646 *
647 * If an exception is raised in early boot before the initial thread has been
648 * set up, then calling get_preemption_level() in the SLEH will trigger an
649 * infinitely-recursing exception. This function handles this edge case.
650 */
651 static inline int
sleh_get_preemption_level(void)652 sleh_get_preemption_level(void)
653 {
654 if (__improbable(current_thread() == NULL)) {
655 return 0;
656 }
657 return get_preemption_level();
658 }
659 #endif // MACH_ASSERT
660
661 static inline bool
is_platform_error(uint64_t esr)662 is_platform_error(uint64_t esr)
663 {
664 esr_exception_class_t class = ESR_EC(esr);
665 uint32_t iss = ESR_ISS(esr);
666 fault_status_t fault_code;
667
668 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
669 fault_code = ISS_DA_FSC(iss);
670 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
671 fault_code = ISS_IA_FSC(iss);
672 } else {
673 return false;
674 }
675
676 return is_parity_error(fault_code) || is_sync_external_abort(fault_code) ||
677 is_table_walk_error(fault_code);
678 }
679
680 void
sleh_synchronous(arm_context_t * context,uint64_t esr,vm_offset_t far,__unused bool did_initiate_panic_lockdown)681 sleh_synchronous(arm_context_t *context, uint64_t esr, vm_offset_t far, __unused bool did_initiate_panic_lockdown)
682 {
683 esr_exception_class_t class = ESR_EC(esr);
684 arm_saved_state_t * state = &context->ss;
685 thread_t thread = current_thread();
686 #if MACH_ASSERT
687 int preemption_level = sleh_get_preemption_level();
688 #endif
689 expected_fault_handler_t expected_fault_handler = NULL;
690 #ifdef CONFIG_XNUPOST
691 expected_fault_handler_t saved_expected_fault_handler = NULL;
692 uintptr_t saved_expected_fault_addr = 0;
693 uintptr_t saved_expected_fault_pc = 0;
694 #endif /* CONFIG_XNUPOST */
695
696 ASSERT_CONTEXT_SANITY(context);
697
698 task_vtimer_check(thread);
699
700 #if CONFIG_DTRACE
701 /*
702 * Handle kernel DTrace probes as early as possible to minimize the likelihood
703 * that this path will itself trigger a DTrace probe, which would lead to infinite
704 * probe recursion.
705 */
706 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
707 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
708 #if CONFIG_SPTM
709 if (__improbable(did_initiate_panic_lockdown)) {
710 panic("Unexpectedly initiated lockdown for DTrace probe?");
711 }
712 #endif
713 return;
714 }
715 #endif
716 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
717
718 #if CONFIG_SPTM
719 // Lockdown should only be initiated for kernel exceptions
720 assert(!(is_user && did_initiate_panic_lockdown));
721 #endif /* CONFIG_SPTM */
722
723 /*
724 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints
725 * that would disclose the behavior of PT_DENY_ATTACH processes.
726 */
727 if (is_user) {
728 /* Sanitize FAR (but only if the exception was taken from userspace) */
729 switch (class) {
730 case ESR_EC_IABORT_EL1:
731 case ESR_EC_IABORT_EL0:
732 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */
733 if (ISS_IA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) {
734 saved_state64(state)->far = 0;
735 }
736 break;
737 case ESR_EC_DABORT_EL1:
738 case ESR_EC_DABORT_EL0:
739 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */
740 if (ISS_DA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) {
741 saved_state64(state)->far = 0;
742 }
743 break;
744 case ESR_EC_WATCHPT_MATCH_EL1:
745 case ESR_EC_WATCHPT_MATCH_EL0:
746 case ESR_EC_PC_ALIGN:
747 break; /* FAR_ELx is valid */
748 default:
749 saved_state64(state)->far = 0;
750 break;
751 }
752
753 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class);
754 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
755 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START,
756 esr, far, get_saved_state_pc(state), 0, 0);
757 } else {
758 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
759 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START,
760 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
761 }
762
763 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) {
764 /*
765 * We no longer support 32-bit, which means no 2-byte
766 * instructions.
767 */
768 if (is_user) {
769 panic("Exception on 2-byte instruction, "
770 "context=%p, esr=%#llx, far=%p",
771 context, esr, (void *)far);
772 } else {
773 panic_with_thread_kernel_state("Exception on 2-byte instruction", state);
774 }
775 }
776
777 #ifdef CONFIG_XNUPOST
778 if (thread->machine.expected_fault_handler != NULL) {
779 bool matching_fault_pc = false;
780 saved_expected_fault_handler = thread->machine.expected_fault_handler;
781 saved_expected_fault_addr = thread->machine.expected_fault_addr;
782 saved_expected_fault_pc = thread->machine.expected_fault_pc;
783
784 thread->machine.expected_fault_handler = NULL;
785 thread->machine.expected_fault_addr = 0;
786 thread->machine.expected_fault_pc = 0;
787
788 #if __has_feature(ptrauth_calls)
789 /*
790 * Compare only the bits of PC which make up the virtual address.
791 * This ignores the upper bits, which may have been corrupted by HW in
792 * platform dependent ways to signal pointer authentication fault.
793 */
794 uint64_t fault_addr_mask = fault_addr_bitmask(0, 64 - T1SZ_BOOT - 1);
795 uint64_t masked_expected_pc = saved_expected_fault_pc & fault_addr_mask;
796 uint64_t masked_saved_pc = get_saved_state_pc(state) & fault_addr_mask;
797 matching_fault_pc = masked_expected_pc == masked_saved_pc;
798 #else
799 matching_fault_pc =
800 (saved_expected_fault_pc == get_saved_state_pc(state));
801 #endif /* ptrauth_call */
802 if (saved_expected_fault_addr == far ||
803 matching_fault_pc) {
804 expected_fault_handler = saved_expected_fault_handler;
805 }
806 }
807 #endif /* CONFIG_XNUPOST */
808
809 if (__improbable(is_platform_error(esr))) {
810 /*
811 * Must gather error info in platform error handler before
812 * thread is preempted to another core/cluster to guarantee
813 * accurate error details
814 */
815
816 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_SYNC);
817 #if CONFIG_SPTM
818 if (__improbable(did_initiate_panic_lockdown)) {
819 panic("Panic lockdown initiated for platform error");
820 }
821 #endif
822 return;
823 }
824
825 if (is_user && class == ESR_EC_DABORT_EL0) {
826 thread_reset_pcs_will_fault(thread);
827 }
828
829 #if CONFIG_SPTM
830 if (__improbable(did_initiate_panic_lockdown && current_thread() != NULL)) {
831 /*
832 * If we initiated panic lockdown, we must disable preemption before
833 * enabling interrupts. While unlikely, preempting the panicked thread
834 * after lockdown has occurred may hang the system if all cores end up
835 * blocked while attempting to return to user space.
836 */
837 disable_preemption();
838 }
839 #endif /* CONFIG_SPTM */
840
841 /* Inherit the interrupt masks from previous context */
842 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) {
843 ml_set_interrupts_enabled(TRUE);
844 }
845
846
847 switch (class) {
848 case ESR_EC_SVC_64:
849 if (!is_saved_state64(state) || !is_user) {
850 panic("Invalid SVC_64 context");
851 }
852
853 handle_svc(state);
854 break;
855
856 case ESR_EC_DABORT_EL0:
857 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler);
858 break;
859
860 case ESR_EC_MSR_TRAP:
861 handle_msr_trap(state, esr);
862 break;
863 /**
864 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for
865 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host
866 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC
867 * handler here needs to be built into the guest kernels to prevent the exception to fall
868 * through.
869 */
870 #if __has_feature(ptrauth_calls)
871 case ESR_EC_PAC_FAIL:
872 #ifdef CONFIG_XNUPOST
873 if (expected_fault_handler != NULL && expected_fault_handler(state)) {
874 break;
875 }
876 #endif /* CONFIG_XNUPOST */
877 handle_pac_fail(state, esr);
878 __builtin_unreachable();
879
880 #endif /* __has_feature(ptrauth_calls) */
881
882 #if HAS_ARM_FEAT_SME
883 case ESR_EC_SME:
884 handle_sme_trap(state, esr);
885 break;
886 #endif /* HAS_ARM_FEAT_SME */
887
888 case ESR_EC_IABORT_EL0:
889 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler);
890 break;
891
892 case ESR_EC_IABORT_EL1:
893 #ifdef CONFIG_XNUPOST
894 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
895 break;
896 }
897 #endif /* CONFIG_XNUPOST */
898
899 panic_with_thread_kernel_state("Kernel instruction fetch abort", state);
900
901 case ESR_EC_PC_ALIGN:
902 handle_pc_align(state);
903 __builtin_unreachable();
904
905 case ESR_EC_DABORT_EL1:
906 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler);
907 break;
908
909 case ESR_EC_UNCATEGORIZED:
910 assert(!ESR_ISS(esr));
911
912 #if CONFIG_XNUPOST
913 if (!is_user && (expected_fault_handler != NULL) && expected_fault_handler(state)) {
914 /*
915 * The fault handler accepted the exception and handled it on its
916 * own. Don't trap to the debugger/panic.
917 */
918 break;
919 }
920 #endif /* CONFIG_XNUPOST */
921 handle_uncategorized(&context->ss);
922 break;
923
924 case ESR_EC_SP_ALIGN:
925 handle_sp_align(state);
926 __builtin_unreachable();
927
928 case ESR_EC_BKPT_AARCH32:
929 handle_breakpoint(state, esr);
930 __builtin_unreachable();
931
932 case ESR_EC_BRK_AARCH64:
933 #ifdef CONFIG_XNUPOST
934 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
935 break;
936 }
937 #endif /* CONFIG_XNUPOST */
938 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
939 handle_kernel_breakpoint(state, esr);
940 break;
941 } else {
942 handle_breakpoint(state, esr);
943 __builtin_unreachable();
944 }
945
946 case ESR_EC_BKPT_REG_MATCH_EL0:
947 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
948 handle_breakpoint(state, esr);
949 }
950 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p",
951 class, state, class, esr, (void *)far);
952 __builtin_unreachable();
953
954 case ESR_EC_BKPT_REG_MATCH_EL1:
955 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state);
956 __builtin_unreachable();
957
958 case ESR_EC_SW_STEP_DEBUG_EL0:
959 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
960 handle_sw_step_debug(state);
961 }
962 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p",
963 class, state, class, esr, (void *)far);
964 __builtin_unreachable();
965
966 case ESR_EC_SW_STEP_DEBUG_EL1:
967 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state);
968 __builtin_unreachable();
969
970 case ESR_EC_WATCHPT_MATCH_EL0:
971 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
972 handle_watchpoint(far);
973 }
974 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p",
975 class, state, class, esr, (void *)far);
976 __builtin_unreachable();
977
978 case ESR_EC_WATCHPT_MATCH_EL1:
979 /*
980 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to
981 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception..
982 */
983 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) {
984 arm_debug_set(NULL);
985 break; /* return to first level handler */
986 }
987 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p",
988 class, state, class, esr, (void *)far);
989 __builtin_unreachable();
990
991 case ESR_EC_TRAP_SIMD_FP:
992 handle_simd_trap(state, esr);
993 __builtin_unreachable();
994
995 case ESR_EC_ILLEGAL_INSTR_SET:
996 panic("Illegal instruction set exception. state=%p class=%u esr=%llu far=%p spsr=0x%x",
997 state, class, esr, (void *)far, get_saved_state_cpsr(state));
998 __builtin_unreachable();
999
1000 case ESR_EC_MCR_MRC_CP15_TRAP:
1001 case ESR_EC_MCRR_MRRC_CP15_TRAP:
1002 case ESR_EC_MCR_MRC_CP14_TRAP:
1003 case ESR_EC_LDC_STC_CP14_TRAP:
1004 case ESR_EC_MCRR_MRRC_CP14_TRAP:
1005 handle_user_trapped_instruction32(state, esr);
1006 __builtin_unreachable();
1007
1008 case ESR_EC_WFI_WFE:
1009 // Use of WFI or WFE instruction when they have been disabled for EL0
1010 handle_wf_trap(state);
1011 __builtin_unreachable();
1012
1013 case ESR_EC_FLOATING_POINT_64:
1014 handle_fp_trap(state, esr);
1015 __builtin_unreachable();
1016 case ESR_EC_BTI_FAIL:
1017 #ifdef CONFIG_XNUPOST
1018 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) {
1019 break;
1020 }
1021 #endif /* CONFIG_XNUPOST */
1022 #ifdef CONFIG_BTI_TELEMETRY
1023 if (bti_telemetry_handle_exception(state)) {
1024 /* Telemetry has accepted and corrected the exception, continue */
1025 break;
1026 }
1027 #endif /* CONFIG_BTI_TELEMETRY */
1028 handle_bti_fail(state, esr);
1029 __builtin_unreachable();
1030
1031 default:
1032 handle_uncategorized(state);
1033 }
1034
1035 #ifdef CONFIG_XNUPOST
1036 if (saved_expected_fault_handler != NULL) {
1037 thread->machine.expected_fault_handler = saved_expected_fault_handler;
1038 thread->machine.expected_fault_addr = saved_expected_fault_addr;
1039 thread->machine.expected_fault_pc = saved_expected_fault_pc;
1040 }
1041 #endif /* CONFIG_XNUPOST */
1042
1043 if (is_user) {
1044 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
1045 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END,
1046 esr, far, get_saved_state_pc(state), 0, 0);
1047 thread->machine.exception_trace_code = 0;
1048 } else {
1049 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
1050 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END,
1051 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0);
1052 }
1053
1054 #if CONFIG_SPTM
1055 if (__improbable(did_initiate_panic_lockdown)) {
1056 #if CONFIG_XNUPOST
1057 bool can_recover = !!(expected_fault_handler);
1058 #else
1059 bool can_recover = false;
1060 #endif /* CONFIG_XNU_POST */
1061
1062 if (can_recover) {
1063 /*
1064 * If we matched an exception handler, this was a simulated lockdown
1065 * and so we can recover. Re-enable preemption if we disabled it.
1066 */
1067 if (current_thread() != NULL) {
1068 enable_preemption();
1069 }
1070 } else {
1071 /*
1072 * fleh already triggered a lockdown but we, for whatever reason,
1073 * didn't end up finding a reason to panic. Catch all panic in this
1074 * case.
1075 * Note that the panic here has no security benefit as the system is
1076 * already hosed, this is merely for telemetry.
1077 */
1078 panic_with_thread_kernel_state("Panic lockdown initiated", state);
1079 }
1080 }
1081 #endif /* CONFIG_SPTM */
1082
1083 #if MACH_ASSERT
1084 if (preemption_level != sleh_get_preemption_level()) {
1085 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
1086 }
1087 #endif
1088
1089 }
1090
1091 /*
1092 * Uncategorized exceptions are a catch-all for general execution errors.
1093 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions.
1094 */
1095 static void
handle_uncategorized(arm_saved_state_t * state)1096 handle_uncategorized(arm_saved_state_t *state)
1097 {
1098 exception_type_t exception = EXC_BAD_INSTRUCTION;
1099 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1100 mach_msg_type_number_t numcodes = 2;
1101 uint32_t instr = 0;
1102
1103 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1104
1105 #if CONFIG_DTRACE
1106
1107 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
1108 /*
1109 * For a 64bit user process, we care about all 4 bytes of the
1110 * instr.
1111 */
1112 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
1113 if (dtrace_user_probe(state) == KERN_SUCCESS) {
1114 return;
1115 }
1116 }
1117 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
1118 /*
1119 * For a 32bit user process, we check for thumb mode, in
1120 * which case we only care about a 2 byte instruction length.
1121 * For non-thumb mode, we care about all 4 bytes of the instructin.
1122 */
1123 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
1124 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
1125 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) {
1126 if (dtrace_user_probe(state) == KERN_SUCCESS) {
1127 return;
1128 }
1129 }
1130 } else {
1131 if ((instr == FASTTRAP_ARM32_INSTR) ||
1132 (instr == FASTTRAP_ARM32_RET_INSTR)) {
1133 if (dtrace_user_probe(state) == KERN_SUCCESS) {
1134 return;
1135 }
1136 }
1137 }
1138 }
1139
1140 #endif /* CONFIG_DTRACE */
1141
1142 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1143 if (IS_ARM_GDB_TRAP(instr)) {
1144 boolean_t interrupt_state;
1145 exception = EXC_BREAKPOINT;
1146
1147 interrupt_state = ml_set_interrupts_enabled(FALSE);
1148
1149 /* Save off the context here (so that the debug logic
1150 * can see the original state of this thread).
1151 */
1152 current_thread()->machine.kpcb = state;
1153
1154 /* Hop into the debugger (typically either due to a
1155 * fatal exception, an explicit panic, or a stackshot
1156 * request.
1157 */
1158 DebuggerCall(exception, state);
1159
1160 current_thread()->machine.kpcb = NULL;
1161 (void) ml_set_interrupts_enabled(interrupt_state);
1162 return;
1163 } else {
1164 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr);
1165 }
1166 }
1167
1168 /*
1169 * Check for GDB breakpoint via illegal opcode.
1170 */
1171 if (IS_ARM_GDB_TRAP(instr)) {
1172 exception = EXC_BREAKPOINT;
1173 codes[0] = EXC_ARM_BREAKPOINT;
1174 codes[1] = instr;
1175 } else {
1176 codes[1] = instr;
1177 }
1178
1179 exception_triage(exception, codes, numcodes);
1180 __builtin_unreachable();
1181 }
1182
1183 #if __has_feature(ptrauth_calls)
1184 static const uint16_t PTRAUTH_TRAP_START = 0xC470;
1185 static inline bool
brk_comment_is_ptrauth(uint16_t comment)1186 brk_comment_is_ptrauth(uint16_t comment)
1187 {
1188 return comment >= PTRAUTH_TRAP_START &&
1189 comment <= PTRAUTH_TRAP_START + ptrauth_key_asdb;
1190 }
1191
1192 static inline const char *
ptrauth_key_to_string(ptrauth_key key)1193 ptrauth_key_to_string(ptrauth_key key)
1194 {
1195 switch (key) {
1196 case ptrauth_key_asia:
1197 return "IA";
1198 case ptrauth_key_asib:
1199 return "IB";
1200 case ptrauth_key_asda:
1201 return "DA";
1202 case ptrauth_key_asdb:
1203 return "DB";
1204 default:
1205 __builtin_unreachable();
1206 }
1207 }
1208
1209 static const char *
ptrauth_handle_brk_trap(void * tstate,uint16_t comment)1210 ptrauth_handle_brk_trap(void *tstate, uint16_t comment)
1211 {
1212 kernel_panic_reason_t pr = PERCPU_GET(panic_reason);
1213 arm_saved_state_t *state = (arm_saved_state_t *)tstate;
1214
1215 ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAP_START);
1216 const char *key_str = ptrauth_key_to_string(key);
1217
1218 snprintf(pr->buf, sizeof(pr->buf),
1219 "Break 0x%04X instruction exception from kernel. "
1220 "Ptrauth failure with %s key resulted in 0x%016llx",
1221 comment, key_str, saved_state64(state)->x[16]);
1222
1223 return pr->buf;
1224 }
1225 #endif /* __has_feature(ptrauth_calls) */
1226
1227 #if HAS_TELEMETRY_KERNEL_BRK
1228 static uint32_t bound_chk_violations_event;
1229
1230 static const char *
xnu_soft_trap_handle_breakpoint(void * tstate,uint16_t comment)1231 xnu_soft_trap_handle_breakpoint(
1232 void *tstate,
1233 uint16_t comment)
1234 {
1235 #if CONFIG_UBSAN_MINIMAL
1236 if (comment == UBSAN_SOFT_TRAP_SIGNED_OF) {
1237 ubsan_handle_brk_trap(tstate, comment);
1238 }
1239 #else
1240 (void)tstate;
1241 #endif
1242
1243 if (comment == CLANG_SOFT_TRAP_BOUND_CHK) {
1244 os_atomic_inc(&bound_chk_violations_event, relaxed);
1245 }
1246 return NULL;
1247 }
1248 #endif /* HAS_TELEMETRY_KERNEL_BRK */
1249
1250 static const char *
xnu_hard_trap_handle_breakpoint(void * tstate,uint16_t comment)1251 xnu_hard_trap_handle_breakpoint(void *tstate, uint16_t comment)
1252 {
1253 kernel_panic_reason_t pr = PERCPU_GET(panic_reason);
1254 arm_saved_state64_t *state = saved_state64(tstate);
1255
1256 switch (comment) {
1257 case XNU_HARD_TRAP_SAFE_UNLINK:
1258 snprintf(pr->buf, sizeof(pr->buf),
1259 "panic: corrupt list around element %p",
1260 (void *)state->x[8]);
1261 return pr->buf;
1262
1263 case XNU_HARD_TRAP_STRING_CHK:
1264 return "panic: string operation caused an overflow";
1265
1266 case XNU_HARD_TRAP_ASSERT_FAILURE:
1267 /*
1268 * Read the implicit assert arguments, see:
1269 * ML_TRAP_REGISTER_1: x8
1270 * ML_TRAP_REGISTER_2: x16
1271 * ML_TRAP_REGISTER_3: x17
1272 */
1273 panic_assert_format(pr->buf, sizeof(pr->buf),
1274 (struct mach_assert_hdr *)state->x[8],
1275 state->x[16], state->x[17]);
1276 return pr->buf;
1277
1278 default:
1279 return NULL;
1280 }
1281 }
1282
1283 #if __has_feature(ptrauth_calls)
1284 KERNEL_BRK_DESCRIPTOR_DEFINE(ptrauth_desc,
1285 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_PTRAUTH,
1286 .base = PTRAUTH_TRAP_START,
1287 .max = PTRAUTH_TRAP_START + ptrauth_key_asdb,
1288 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT,
1289 .handle_breakpoint = ptrauth_handle_brk_trap);
1290 #endif
1291
1292 KERNEL_BRK_DESCRIPTOR_DEFINE(clang_desc,
1293 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_CLANG,
1294 .base = CLANG_ARM_TRAP_START,
1295 .max = CLANG_ARM_TRAP_END,
1296 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT,
1297 .handle_breakpoint = NULL);
1298
1299 KERNEL_BRK_DESCRIPTOR_DEFINE(libcxx_desc,
1300 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_LIBCXX,
1301 .base = LIBCXX_TRAP_START,
1302 .max = LIBCXX_TRAP_END,
1303 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT,
1304 .handle_breakpoint = NULL);
1305
1306 #if HAS_TELEMETRY_KERNEL_BRK
1307 KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_soft_traps_desc,
1308 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_TELEMETRY,
1309 .base = XNU_SOFT_TRAP_START,
1310 .max = XNU_SOFT_TRAP_END,
1311 .options = BRK_TELEMETRY_OPTIONS_RECOVERABLE_DEFAULT(
1312 /* enable_telemetry */ true),
1313 .handle_breakpoint = xnu_soft_trap_handle_breakpoint);
1314 #endif /* HAS_TELEMETRY_KERNEL_BRK */
1315
1316 KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_hard_traps_desc,
1317 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_XNU,
1318 .base = XNU_HARD_TRAP_START,
1319 .max = XNU_HARD_TRAP_END,
1320 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT,
1321 .handle_breakpoint = xnu_hard_trap_handle_breakpoint);
1322
1323 static void
1324 #if !HAS_TELEMETRY_KERNEL_BRK
1325 __attribute__((noreturn))
1326 #endif
handle_kernel_breakpoint(arm_saved_state_t * state,uint64_t esr)1327 handle_kernel_breakpoint(arm_saved_state_t *state, uint64_t esr)
1328 {
1329 uint16_t comment = ISS_BRK_COMMENT(esr);
1330 const struct kernel_brk_descriptor *desc;
1331 const char *msg = NULL;
1332
1333 desc = find_brk_descriptor_by_comment(comment);
1334
1335 if (!desc) {
1336 goto brk_out;
1337 }
1338
1339 #if HAS_TELEMETRY_KERNEL_BRK
1340 if (desc->options.enable_trap_telemetry) {
1341 trap_telemetry_report_exception(
1342 /* trap_type */ desc->type,
1343 /* trap_code */ comment,
1344 /* options */ desc->options.telemetry_options,
1345 /* saved_state */ (void *)state);
1346 }
1347 #endif
1348
1349 if (desc->handle_breakpoint) {
1350 msg = desc->handle_breakpoint(state, comment);
1351 }
1352
1353 #if HAS_TELEMETRY_KERNEL_BRK
1354 /* Still alive? Check if we should recover. */
1355 if (desc->options.recoverable) {
1356 add_saved_state_pc(state, 4);
1357 return;
1358 }
1359 #endif
1360
1361 brk_out:
1362 if (msg == NULL) {
1363 kernel_panic_reason_t pr = PERCPU_GET(panic_reason);
1364
1365 msg = tsnprintf(pr->buf, sizeof(pr->buf),
1366 "Break 0x%04X instruction exception from kernel. "
1367 "Panic (by design)",
1368 comment);
1369 }
1370
1371 panic_with_thread_kernel_state(msg, state);
1372 __builtin_unreachable();
1373 #undef MSG_FMT
1374 }
1375
1376 static void
handle_breakpoint(arm_saved_state_t * state,uint64_t esr __unused)1377 handle_breakpoint(arm_saved_state_t *state, uint64_t esr __unused)
1378 {
1379 exception_type_t exception = EXC_BREAKPOINT;
1380 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT};
1381 mach_msg_type_number_t numcodes = 2;
1382
1383 #if __has_feature(ptrauth_calls)
1384 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 &&
1385 brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) {
1386 exception |= EXC_PTRAUTH_BIT;
1387 }
1388 #endif /* __has_feature(ptrauth_calls) */
1389
1390 codes[1] = get_saved_state_pc(state);
1391 exception_triage(exception, codes, numcodes);
1392 __builtin_unreachable();
1393 }
1394
1395 static void
handle_watchpoint(vm_offset_t fault_addr)1396 handle_watchpoint(vm_offset_t fault_addr)
1397 {
1398 exception_type_t exception = EXC_BREAKPOINT;
1399 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG};
1400 mach_msg_type_number_t numcodes = 2;
1401
1402 codes[1] = fault_addr;
1403 exception_triage(exception, codes, numcodes);
1404 __builtin_unreachable();
1405 }
1406
1407 static void
handle_abort(arm_saved_state_t * state,uint64_t esr,vm_offset_t fault_addr,abort_inspector_t inspect_abort,abort_handler_t handler,expected_fault_handler_t expected_fault_handler)1408 handle_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr,
1409 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
1410 {
1411 fault_status_t fault_code;
1412 vm_prot_t fault_type;
1413
1414 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
1415 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
1416 }
1417
1418 static void
inspect_instruction_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1419 inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1420 {
1421 getCpuDatap()->cpu_stat.instr_ex_cnt++;
1422 *fault_code = ISS_IA_FSC(iss);
1423 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
1424 }
1425
1426 static void
inspect_data_abort(uint32_t iss,fault_status_t * fault_code,vm_prot_t * fault_type)1427 inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
1428 {
1429 getCpuDatap()->cpu_stat.data_ex_cnt++;
1430 *fault_code = ISS_DA_FSC(iss);
1431
1432 /*
1433 * Cache maintenance operations always report faults as write access.
1434 * Change these to read access, unless they report a permission fault.
1435 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
1436 * access to the mapping, but if a cache maintenance operation that only requires
1437 * read access generates a permission fault, then we will not be able to handle
1438 * the fault regardless of whether we treat it as a read or write fault.
1439 */
1440 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
1441 *fault_type = (VM_PROT_READ | VM_PROT_WRITE);
1442 } else {
1443 *fault_type = (VM_PROT_READ);
1444 }
1445 }
1446
1447 #if __has_feature(ptrauth_calls)
1448 static inline uint64_t
fault_addr_bitmask(unsigned int bit_from,unsigned int bit_to)1449 fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
1450 {
1451 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
1452 }
1453
1454 static inline bool
fault_addr_bit(vm_offset_t fault_addr,unsigned int bit)1455 fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
1456 {
1457 return (bool)((fault_addr >> bit) & 1);
1458 }
1459
1460 extern int gARM_FEAT_FPAC;
1461 extern int gARM_FEAT_FPACCOMBINE;
1462 extern int gARM_FEAT_PAuth2;
1463
1464 /**
1465 * Determines whether a fault address taken at EL0 contains a PAC error code
1466 * corresponding to the specified kind of ptrauth key.
1467 */
1468 static bool
user_fault_matches_pac_error_code(vm_offset_t fault_addr,uint64_t pc,bool data_key)1469 user_fault_matches_pac_error_code(vm_offset_t fault_addr, uint64_t pc, bool data_key)
1470 {
1471 if (gARM_FEAT_FPACCOMBINE) {
1472 /*
1473 * CPUs with FPACCOMBINE always raise PAC Fail exceptions during
1474 * PAC failure. If the CPU took any other kind of exception, we
1475 * can rule out PAC as the root cause.
1476 */
1477 return false;
1478 }
1479
1480 if (data_key && gARM_FEAT_FPAC) {
1481 uint32_t instr;
1482 int err = copyin(pc, (char *)&instr, sizeof(instr));
1483 if (!err && !ARM64_INSTR_IS_LDRAx(instr)) {
1484 /*
1485 * On FPAC-enabled devices, PAC failure can only cause
1486 * data aborts during "combined" LDRAx instructions. If
1487 * PAC fails during a discrete AUTxx + LDR/STR
1488 * instruction sequence, then the AUTxx instruction
1489 * raises a PAC Fail exception rather than poisoning its
1490 * output address.
1491 *
1492 * In principle the same logic applies to instruction
1493 * aborts. But we have no way to identify the exact
1494 * instruction that caused the abort, so we can't tell
1495 * if it was a combined branch + auth instruction.
1496 */
1497 return false;
1498 }
1499 }
1500
1501 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY);
1502 bool tbi = data_key || __improbable(instruction_tbi);
1503
1504 if (gARM_FEAT_PAuth2) {
1505 /*
1506 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so
1507 * treat all non-canonical address bits like potential poison bits.
1508 */
1509 uint64_t mask = fault_addr_bitmask(64 - T0SZ_BOOT, 54);
1510 if (!tbi) {
1511 mask |= fault_addr_bitmask(56, 63);
1512 }
1513 return (fault_addr & mask) != 0;
1514 } else {
1515 unsigned int poison_shift;
1516 if (tbi) {
1517 poison_shift = 53;
1518 } else {
1519 poison_shift = 61;
1520 }
1521
1522 /* PAC error codes are always in the form key_number:NOT(key_number) */
1523 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift);
1524 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1);
1525 return poison_bit_1 != poison_bit_2;
1526 }
1527 }
1528 #endif /* __has_feature(ptrauth_calls) */
1529
1530 /**
1531 * Determines whether the userland thread has a JIT region in RW mode, TPRO
1532 * in RW mode, or JCTL_EL0 in pointer signing mode. A fault in any of these trusted
1533 * code paths may indicate an attack on WebKit. Rather than letting a
1534 * potentially-compromised process try to handle the exception, it will be killed
1535 * by the kernel and a crash report will be generated.
1536 */
1537 static bool
user_fault_in_self_restrict_mode(thread_t thread __unused)1538 user_fault_in_self_restrict_mode(thread_t thread __unused)
1539 {
1540
1541 return false;
1542 }
1543
1544 static void
handle_pc_align(arm_saved_state_t * ss)1545 handle_pc_align(arm_saved_state_t *ss)
1546 {
1547 exception_type_t exc;
1548 mach_exception_data_type_t codes[2];
1549 mach_msg_type_number_t numcodes = 2;
1550
1551 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1552 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
1553 }
1554
1555 exc = EXC_BAD_ACCESS;
1556 #if __has_feature(ptrauth_calls)
1557 uint64_t pc = get_saved_state_pc(ss);
1558 if (user_fault_matches_pac_error_code(pc, pc, false)) {
1559 exc |= EXC_PTRAUTH_BIT;
1560 }
1561 #endif /* __has_feature(ptrauth_calls) */
1562
1563 codes[0] = EXC_ARM_DA_ALIGN;
1564 codes[1] = get_saved_state_pc(ss);
1565
1566 exception_triage(exc, codes, numcodes);
1567 __builtin_unreachable();
1568 }
1569
1570 static void
handle_sp_align(arm_saved_state_t * ss)1571 handle_sp_align(arm_saved_state_t *ss)
1572 {
1573 exception_type_t exc;
1574 mach_exception_data_type_t codes[2];
1575 mach_msg_type_number_t numcodes = 2;
1576
1577 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
1578 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
1579 }
1580
1581 exc = EXC_BAD_ACCESS;
1582 #if __has_feature(ptrauth_calls)
1583 if (user_fault_matches_pac_error_code(get_saved_state_sp(ss), get_saved_state_pc(ss), true)) {
1584 exc |= EXC_PTRAUTH_BIT;
1585 }
1586 #endif /* __has_feature(ptrauth_calls) */
1587
1588 codes[0] = EXC_ARM_SP_ALIGN;
1589 codes[1] = get_saved_state_sp(ss);
1590
1591 exception_triage(exc, codes, numcodes);
1592 __builtin_unreachable();
1593 }
1594
1595 static void
handle_wf_trap(arm_saved_state_t * state)1596 handle_wf_trap(arm_saved_state_t *state)
1597 {
1598 exception_type_t exc;
1599 mach_exception_data_type_t codes[2];
1600 mach_msg_type_number_t numcodes = 2;
1601 uint32_t instr = 0;
1602
1603 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1604
1605 exc = EXC_BAD_INSTRUCTION;
1606 codes[0] = EXC_ARM_UNDEFINED;
1607 codes[1] = instr;
1608
1609 exception_triage(exc, codes, numcodes);
1610 __builtin_unreachable();
1611 }
1612
1613 static void
handle_fp_trap(arm_saved_state_t * state,uint64_t esr)1614 handle_fp_trap(arm_saved_state_t *state, uint64_t esr)
1615 {
1616 exception_type_t exc = EXC_ARITHMETIC;
1617 mach_exception_data_type_t codes[2];
1618 mach_msg_type_number_t numcodes = 2;
1619 uint32_t instr = 0;
1620
1621 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
1622 panic_with_thread_kernel_state("Floating point exception from kernel", state);
1623 }
1624
1625 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1626 codes[1] = instr;
1627
1628 /* The floating point trap flags are only valid if TFV is set. */
1629 if (!fp_exceptions_enabled) {
1630 exc = EXC_BAD_INSTRUCTION;
1631 codes[0] = EXC_ARM_UNDEFINED;
1632 } else if (!(esr & ISS_FP_TFV)) {
1633 codes[0] = EXC_ARM_FP_UNDEFINED;
1634 } else if (esr & ISS_FP_UFF) {
1635 codes[0] = EXC_ARM_FP_UF;
1636 } else if (esr & ISS_FP_OFF) {
1637 codes[0] = EXC_ARM_FP_OF;
1638 } else if (esr & ISS_FP_IOF) {
1639 codes[0] = EXC_ARM_FP_IO;
1640 } else if (esr & ISS_FP_DZF) {
1641 codes[0] = EXC_ARM_FP_DZ;
1642 } else if (esr & ISS_FP_IDF) {
1643 codes[0] = EXC_ARM_FP_ID;
1644 } else if (esr & ISS_FP_IXF) {
1645 codes[0] = EXC_ARM_FP_IX;
1646 } else {
1647 panic("Unrecognized floating point exception, state=%p, esr=%#llx", state, esr);
1648 }
1649
1650 exception_triage(exc, codes, numcodes);
1651 __builtin_unreachable();
1652 }
1653
1654
1655
1656 /*
1657 * handle_alignment_fault_from_user:
1658 * state: Saved state
1659 *
1660 * Attempts to deal with an alignment fault from userspace (possibly by
1661 * emulating the faulting instruction). If emulation failed due to an
1662 * unservicable fault, the ESR for that fault will be stored in the
1663 * recovery_esr field of the thread by the exception code.
1664 *
1665 * Returns:
1666 * -1: Emulation failed (emulation of state/instr not supported)
1667 * 0: Successfully emulated the instruction
1668 * EFAULT: Emulation failed (probably due to permissions)
1669 * EINVAL: Emulation failed (probably due to a bad address)
1670 */
1671
1672
1673 static int
handle_alignment_fault_from_user(arm_saved_state_t * state,kern_return_t * vmfr)1674 handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
1675 {
1676 int ret = -1;
1677
1678 #pragma unused (state)
1679 #pragma unused (vmfr)
1680
1681 return ret;
1682 }
1683
1684
1685
1686 #if HAS_ARM_FEAT_SME
1687 static void
handle_sme_trap(arm_saved_state_t * state,uint64_t esr)1688 handle_sme_trap(arm_saved_state_t *state, uint64_t esr)
1689 {
1690 exception_type_t exc = EXC_BAD_INSTRUCTION;
1691 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
1692 mach_msg_type_number_t numcodes = 2;
1693
1694 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1695 panic("SME exception from kernel, state=%p, esr=%#llx", state, esr);
1696 }
1697 if (!arm_sme_version()) {
1698 /*
1699 * If SME is disabled in software but userspace executes an SME
1700 * instruction anyway, then the CPU will still raise an
1701 * SME-specific trap. Triage it as if the CPU raised an
1702 * undefined-instruction trap.
1703 */
1704 exception_triage(exc, codes, numcodes);
1705 __builtin_unreachable();
1706 }
1707
1708 if (ISS_SME_SMTC(ESR_ISS(esr)) == ISS_SME_SMTC_CAPCR) {
1709 thread_t thread = current_thread();
1710 switch (machine_thread_sme_state_alloc(thread)) {
1711 case KERN_SUCCESS:
1712 return;
1713
1714
1715 default:
1716 panic("Failed to allocate SME state for thread %p", thread);
1717 }
1718 }
1719
1720 uint32_t instr;
1721 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
1722 codes[1] = instr;
1723
1724 exception_triage(exc, codes, numcodes);
1725 __builtin_unreachable();
1726 }
1727 #endif /* HAS_ARM_FEAT_SME */
1728
1729 static void
handle_sw_step_debug(arm_saved_state_t * state)1730 handle_sw_step_debug(arm_saved_state_t *state)
1731 {
1732 thread_t thread = current_thread();
1733 exception_type_t exc;
1734 mach_exception_data_type_t codes[2];
1735 mach_msg_type_number_t numcodes = 2;
1736
1737 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
1738 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
1739 }
1740
1741 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
1742 if (thread->machine.DebugData != NULL) {
1743 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
1744 } else {
1745 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
1746 }
1747
1748 mask_user_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);
1749
1750 // Special encoding for gdb single step event on ARM
1751 exc = EXC_BREAKPOINT;
1752 codes[0] = 1;
1753 codes[1] = 0;
1754
1755 exception_triage(exc, codes, numcodes);
1756 __builtin_unreachable();
1757 }
1758
1759 #if MACH_ASSERT
1760 TUNABLE_WRITEABLE(int, panic_on_jit_guard, "panic_on_jit_guard", 0);
1761 #endif /* MACH_ASSERT */
1762
1763
1764 static void
handle_user_abort(arm_saved_state_t * state,uint64_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)1765 handle_user_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr,
1766 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
1767 {
1768 exception_type_t exc = EXC_BAD_ACCESS;
1769 mach_exception_data_type_t codes[2];
1770 mach_msg_type_number_t numcodes = 2;
1771 thread_t thread = current_thread();
1772
1773 (void)esr;
1774 (void)expected_fault_handler;
1775
1776 if (__improbable(!SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state)))) {
1777 panic_with_thread_kernel_state("User abort from non-interruptible context", state);
1778 }
1779
1780 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
1781
1782 if (!is_servicible_fault(fault_code) &&
1783 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1784 thread_reset_pcs_done_faulting(thread);
1785 }
1786
1787 if (is_vm_fault(fault_code)) {
1788 vm_map_t map = thread->map;
1789 vm_offset_t vm_fault_addr = fault_addr;
1790 kern_return_t result = KERN_FAILURE;
1791
1792 assert(map != kernel_map);
1793
1794 if (!(fault_type & VM_PROT_EXECUTE)) {
1795 vm_fault_addr = VM_USER_STRIP_TBI(fault_addr);
1796 }
1797
1798 /* check to see if it is just a pmap ref/modify fault */
1799 if (!is_translation_fault(fault_code)) {
1800 result = arm_fast_fault(map->pmap,
1801 vm_fault_addr,
1802 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE);
1803 }
1804 if (result != KERN_SUCCESS) {
1805
1806 {
1807 /* We have to fault the page in */
1808 result = vm_fault(map, vm_fault_addr, fault_type,
1809 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE,
1810 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
1811 }
1812 }
1813 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
1814 thread_reset_pcs_done_faulting(thread);
1815 }
1816 if (result == KERN_SUCCESS || result == KERN_ABORTED) {
1817 return;
1818 }
1819
1820 /*
1821 * vm_fault() should never return KERN_FAILURE for page faults from user space.
1822 * If it does, we're leaking preemption disables somewhere in the kernel.
1823 */
1824 if (__improbable(result == KERN_FAILURE)) {
1825 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread);
1826 }
1827
1828 codes[0] = result;
1829 } else if (is_alignment_fault(fault_code)) {
1830 kern_return_t vmfkr = KERN_SUCCESS;
1831 thread->machine.recover_esr = 0;
1832 thread->machine.recover_far = 0;
1833 int result = handle_alignment_fault_from_user(state, &vmfkr);
1834 if (result == 0) {
1835 /* Successfully emulated, or instruction
1836 * copyin() for decode/emulation failed.
1837 * Continue, or redrive instruction.
1838 */
1839 thread_exception_return();
1840 } else if (((result == EFAULT) || (result == EINVAL)) &&
1841 (thread->machine.recover_esr == 0)) {
1842 /*
1843 * If we didn't actually take a fault, but got one of
1844 * these errors, then we failed basic sanity checks of
1845 * the fault address. Treat this as an invalid
1846 * address.
1847 */
1848 codes[0] = KERN_INVALID_ADDRESS;
1849 } else if ((result == EFAULT) &&
1850 (thread->machine.recover_esr)) {
1851 /*
1852 * Since alignment aborts are prioritized
1853 * ahead of translation aborts, the misaligned
1854 * atomic emulation flow may have triggered a
1855 * VM pagefault, which the VM could not resolve.
1856 * Report the VM fault error in codes[]
1857 */
1858
1859 codes[0] = vmfkr;
1860 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr);
1861 /* Cause ESR_EC to reflect an EL0 abort */
1862 thread->machine.recover_esr &= ~ESR_EC_MASK;
1863 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT);
1864 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr);
1865 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far);
1866 fault_addr = thread->machine.recover_far;
1867 } else {
1868 /* This was just an unsupported alignment
1869 * exception. Misaligned atomic emulation
1870 * timeouts fall in this category.
1871 */
1872 codes[0] = EXC_ARM_DA_ALIGN;
1873 }
1874 } else if (is_parity_error(fault_code)) {
1875 #if defined(APPLE_ARM64_ARCH_FAMILY)
1876 /*
1877 * Platform errors are handled in sleh_sync before interrupts are enabled.
1878 */
1879 #else
1880 panic("User parity error.");
1881 #endif
1882 } else {
1883 codes[0] = KERN_FAILURE;
1884 }
1885
1886 #if CODE_SIGNING_MONITOR
1887 /*
1888 * If the code reaches here, it means we weren't able to resolve the fault and we're
1889 * going to be sending the task an exception. On systems which have the code signing
1890 * monitor enabled, an execute fault which cannot be handled must result in sending
1891 * a SIGKILL to the task.
1892 */
1893 if (is_vm_fault(fault_code) && (fault_type & VM_PROT_EXECUTE)) {
1894 csm_code_signing_violation(current_proc(), fault_addr);
1895 }
1896 #endif
1897
1898 codes[1] = fault_addr;
1899 #if __has_feature(ptrauth_calls)
1900 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0);
1901 if (user_fault_matches_pac_error_code(fault_addr, get_saved_state_pc(state), is_data_abort)) {
1902 exc |= EXC_PTRAUTH_BIT;
1903 }
1904 #endif /* __has_feature(ptrauth_calls) */
1905
1906 if (user_fault_in_self_restrict_mode(thread) &&
1907 task_is_jit_exception_fatal(get_threadtask(thread))) {
1908 int flags = PX_KTRIAGE;
1909 exception_info_t info = {
1910 .os_reason = OS_REASON_SELF_RESTRICT,
1911 .exception_type = exc,
1912 .mx_code = codes[0],
1913 .mx_subcode = codes[1]
1914 };
1915
1916 #if MACH_ASSERT
1917 printf("\nGUARD_REASON_JIT exc %d codes=<0x%llx,0x%llx> syscalls %d task %p thread %p va 0x%lx code 0x%x type 0x%x esr 0x%llx\n",
1918 exc, codes[0], codes[1], thread->syscalls_unix, current_task(), thread, fault_addr, fault_code, fault_type, esr);
1919 if (panic_on_jit_guard &&
1920 current_task()->thread_count == 1 &&
1921 thread->syscalls_unix < 24) {
1922 panic("GUARD_REASON_JIT exc %d codes=<0x%llx,0x%llx> syscalls %d task %p thread %p va 0x%lx code 0x%x type 0x%x esr 0x%llx state %p j %d t %d s user 0x%llx (0x%llx) jb 0x%llx (0x%llx)",
1923 exc, codes[0], codes[1], thread->syscalls_unix, current_task(), thread, fault_addr, fault_code, fault_type, esr, state,
1924 0, 0, 0ull, 0ull,
1925 0ull, 0ull
1926 );
1927 }
1928 #endif /* MACH_ASSERT */
1929
1930 exit_with_mach_exception(current_proc(), info, flags);
1931 }
1932
1933
1934 exception_triage(exc, codes, numcodes);
1935 __builtin_unreachable();
1936 }
1937
1938 /**
1939 * Panic because the kernel abort handler tried to apply a recovery handler that
1940 * isn't inside copyio_recover_table[].
1941 *
1942 * @param state original saved-state
1943 * @param recover invalid recovery handler
1944 */
1945 __attribute__((noreturn, used))
1946 static void
panic_on_invalid_recovery_handler(arm_saved_state_t * state,struct copyio_recovery_entry * recover)1947 panic_on_invalid_recovery_handler(arm_saved_state_t *state, struct copyio_recovery_entry *recover)
1948 {
1949 panic("attempt to set invalid recovery handler %p on kernel saved-state %p", recover, state);
1950 }
1951
1952 /**
1953 * Update a thread saved-state to store an error code in x0 and branch to a
1954 * copyio recovery handler.
1955 *
1956 * @param state original saved-state
1957 * @param esr ESR_ELx value for the fault taken
1958 * @param fault_addr FAR_ELx value for the fault taken
1959 * @param thread target thread
1960 * @param recover destination copyio recovery handler
1961 * @param x0 error code to populate into x0
1962 */
1963 static void
handle_kernel_abort_recover_with_error_code(arm_saved_state_t * state,uint64_t esr,vm_offset_t fault_addr,thread_t thread,struct copyio_recovery_entry * _Nonnull recover,uint64_t x0)1964 handle_kernel_abort_recover_with_error_code(
1965 arm_saved_state_t *state,
1966 uint64_t esr,
1967 vm_offset_t fault_addr,
1968 thread_t thread,
1969 struct copyio_recovery_entry *_Nonnull recover,
1970 uint64_t x0)
1971 {
1972 thread->machine.recover_esr = esr;
1973 thread->machine.recover_far = fault_addr;
1974 saved_state64(state)->x[0] = x0;
1975 #if defined(HAS_APPLE_PAC)
1976 MANIPULATE_SIGNED_THREAD_STATE(state,
1977 "adrp x6, _copyio_recover_table_end@page \n"
1978 "add x6, x6, _copyio_recover_table_end@pageoff \n"
1979 "cmp %[recover], x6 \n"
1980 "b.lt 1f \n"
1981 "bl _panic_on_invalid_recovery_handler \n"
1982 "brk #0 \n"
1983 "1: \n"
1984 "adrp x6, _copyio_recover_table@page \n"
1985 "add x6, x6, _copyio_recover_table@pageoff \n"
1986 "subs x7, %[recover], x6 \n"
1987 "b.pl 1f \n"
1988 "bl _panic_on_invalid_recovery_handler \n"
1989 "brk #0 \n"
1990 "1: \n"
1991 "udiv x8, x7, %[SIZEOF_RECOVER] \n"
1992 "mul x8, x8, %[SIZEOF_RECOVER] \n"
1993 "cmp x7, x8 \n"
1994 "b.eq 1f \n"
1995 "bl _panic_on_invalid_recovery_handler \n"
1996 "brk #0 \n"
1997 "1: \n"
1998 "ldr x1, [%[recover], %[CRE_RECOVERY]] \n"
1999 "add x1, x1, x6 \n"
2000 "str x1, [x0, %[SS64_PC]] \n",
2001 [recover] "r"(recover),
2002 [SIZEOF_RECOVER] "r"((sizeof(*recover))),
2003 [CRE_RECOVERY] "i"(offsetof(struct copyio_recovery_entry, cre_recovery))
2004 );
2005 #else
2006 ptrdiff_t recover_offset = (uintptr_t)recover - (uintptr_t)copyio_recover_table;
2007 if ((uintptr_t)recover < (uintptr_t)copyio_recover_table ||
2008 (uintptr_t)recover >= (uintptr_t)copyio_recover_table_end ||
2009 (recover_offset % sizeof(*recover)) != 0) {
2010 panic_on_invalid_recovery_handler(state, recover);
2011 }
2012 saved_state64(state)->pc = copyio_recovery_addr(recover->cre_recovery);
2013 #endif
2014 }
2015
2016 static inline void
handle_kernel_abort_recover(arm_saved_state_t * state,uint64_t esr,vm_offset_t fault_addr,thread_t thread,struct copyio_recovery_entry * _Nonnull recover)2017 handle_kernel_abort_recover(
2018 arm_saved_state_t *state,
2019 uint64_t esr,
2020 vm_offset_t fault_addr,
2021 thread_t thread,
2022 struct copyio_recovery_entry *_Nonnull recover)
2023 {
2024 handle_kernel_abort_recover_with_error_code(state, esr, fault_addr, thread, recover, EFAULT);
2025 }
2026
2027
2028 static void
handle_kernel_abort(arm_saved_state_t * state,uint64_t esr,vm_offset_t fault_addr,fault_status_t fault_code,vm_prot_t fault_type,expected_fault_handler_t expected_fault_handler)2029 handle_kernel_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr,
2030 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
2031 {
2032 thread_t thread = current_thread();
2033 struct copyio_recovery_entry *recover = find_copyio_recovery_entry(
2034 get_saved_state_pc(state));
2035
2036 #ifndef CONFIG_XNUPOST
2037 (void)expected_fault_handler;
2038 #endif /* CONFIG_XNUPOST */
2039
2040 #if CONFIG_DTRACE
2041 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
2042 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
2043 /*
2044 * Point to next instruction, or recovery handler if set.
2045 */
2046 if (recover) {
2047 handle_kernel_abort_recover(state, esr, VM_USER_STRIP_PTR(fault_addr), thread, recover);
2048 } else {
2049 add_saved_state_pc(state, 4);
2050 }
2051 return;
2052 } else {
2053 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
2054 }
2055 }
2056 #endif
2057
2058 if (is_vm_fault(fault_code)) {
2059 kern_return_t result = KERN_FAILURE;
2060 vm_map_t map;
2061 int interruptible;
2062
2063 #ifdef CONFIG_XNUPOST
2064 if (expected_fault_handler && expected_fault_handler(state)) {
2065 return;
2066 }
2067 #endif /* CONFIG_XNUPOST */
2068
2069 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) {
2070 /*
2071 * If no recovery handler is supplied, always drive the fault against
2072 * the kernel map. If the fault was taken against a userspace VA, indicating
2073 * an unprotected access to user address space, vm_fault() should fail and
2074 * ultimately lead to a panic here.
2075 */
2076 map = kernel_map;
2077 interruptible = THREAD_UNINT;
2078
2079 #if CONFIG_KERNEL_TAGGING
2080 /*
2081 * If kernel tagging is enabled, canonicalize the address here, so that we have a
2082 * chance to find it in the VM ranges. Do not mess with exec fault cases.
2083 */
2084 if (!((fault_type) & VM_PROT_EXECUTE)) {
2085 fault_addr = vm_memtag_canonicalize(map, fault_addr);
2086 }
2087 #endif /* CONFIG_KERNEL_TAGGING */
2088 } else {
2089 map = thread->map;
2090
2091 /**
2092 * In the case that the recovery handler is set (e.g., during copyio
2093 * and dtrace probes), we don't want the vm_fault() operation to be
2094 * aborted early. Those code paths can't handle restarting the
2095 * vm_fault() operation so don't allow it to return early without
2096 * creating the wanted mapping.
2097 */
2098 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE;
2099
2100 }
2101
2102 /*
2103 * Ensure no faults in the physical aperture. This could happen if
2104 * a page table is incorrectly allocated from the read only region
2105 * when running with KTRR.
2106 */
2107 if (__improbable(fault_addr >= physmap_base) && (fault_addr < physmap_end)) {
2108 panic_with_thread_kernel_state("Unexpected fault in kernel physical aperture", state);
2109 }
2110 if (__improbable(fault_addr >= gVirtBase && fault_addr < static_memory_end)) {
2111 panic_with_thread_kernel_state("Unexpected fault in kernel static region", state);
2112 }
2113
2114 /* check to see if it is just a pmap ref/modify fault */
2115 if (!is_translation_fault(fault_code)) {
2116 result = arm_fast_fault(map->pmap,
2117 fault_addr,
2118 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE);
2119 if (result == KERN_SUCCESS) {
2120 return;
2121 }
2122 }
2123
2124 /**
2125 * vm_fault() can be called with preemption disabled (and indeed this is expected for
2126 * certain copyio() scenarios), but can't safely be called with interrupts disabled once
2127 * the system has gone multi-threaded. Other than some early-boot situations such as
2128 * startup kext loading, kernel paging operations should never be triggered by
2129 * non-interruptible code in the first place, so a fault from such a context will
2130 * ultimately produce a kernel data abort panic anyway. In these cases, skip calling
2131 * vm_fault() to avoid masking the real kernel panic with a failed VM locking assertion.
2132 */
2133 if (__probable(SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state)) ||
2134 startup_phase < STARTUP_SUB_EARLY_BOOT ||
2135 current_cpu_datap()->cpu_hibernate)) {
2136 if (result != KERN_PROTECTION_FAILURE) {
2137 // VM will query this property when deciding to throttle this fault, we don't want to
2138 // throttle kernel faults for copyio faults. The presence of a recovery entry is used as a
2139 // proxy for being in copyio code.
2140 bool const was_recover = thread->recover;
2141 thread->recover = was_recover || recover;
2142
2143 /*
2144 * We have to "fault" the page in.
2145 */
2146 result = vm_fault(map, fault_addr, fault_type,
2147 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible,
2148 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0);
2149
2150 thread->recover = was_recover;
2151 }
2152
2153 if (result == KERN_SUCCESS) {
2154 return;
2155 }
2156 }
2157
2158 /*
2159 * If we have a recover handler, invoke it now.
2160 */
2161 if (recover) {
2162 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
2163 return;
2164 }
2165
2166 panic_fault_address = fault_addr;
2167 } else if (is_alignment_fault(fault_code)) {
2168 if (recover) {
2169 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover);
2170 return;
2171 }
2172 panic_with_thread_kernel_state("Unaligned kernel data abort.", state);
2173 } else if (is_parity_error(fault_code)) {
2174 #if defined(APPLE_ARM64_ARCH_FAMILY)
2175 /*
2176 * Platform errors are handled in sleh_sync before interrupts are enabled.
2177 */
2178 #else
2179 panic_with_thread_kernel_state("Kernel parity error.", state);
2180 #endif
2181 } else {
2182 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code);
2183 }
2184
2185 panic_with_thread_kernel_state("Kernel data abort.", state);
2186 }
2187
2188 extern void syscall_trace(struct arm_saved_state * regs);
2189
2190 static void
handle_svc(arm_saved_state_t * state)2191 handle_svc(arm_saved_state_t *state)
2192 {
2193 int trap_no = get_saved_state_svc_number(state);
2194 thread_t thread = current_thread();
2195 struct proc *p;
2196
2197 #define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */
2198
2199 #define TRACE_SYSCALL 1
2200 #if TRACE_SYSCALL
2201 syscall_trace(state);
2202 #endif
2203
2204 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */
2205
2206 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
2207 platform_syscall(state);
2208 panic("Returned from platform_syscall()?");
2209 }
2210
2211 current_cached_proc_cred_update();
2212
2213 if (trap_no < 0) {
2214 switch (trap_no) {
2215 case MACH_ARM_TRAP_ABSTIME:
2216 handle_mach_absolute_time_trap(state);
2217 return;
2218 case MACH_ARM_TRAP_CONTTIME:
2219 handle_mach_continuous_time_trap(state);
2220 return;
2221 }
2222
2223 /* Counting perhaps better in the handler, but this is how it's been done */
2224 thread->syscalls_mach++;
2225 mach_syscall(state);
2226 } else {
2227 /* Counting perhaps better in the handler, but this is how it's been done */
2228 thread->syscalls_unix++;
2229 p = get_bsdthreadtask_info(thread);
2230
2231 assert(p);
2232
2233 unix_syscall(state, thread, p);
2234 }
2235 }
2236
2237 static void
handle_mach_absolute_time_trap(arm_saved_state_t * state)2238 handle_mach_absolute_time_trap(arm_saved_state_t *state)
2239 {
2240 uint64_t now = mach_absolute_time();
2241 saved_state64(state)->x[0] = now;
2242 }
2243
2244 static void
handle_mach_continuous_time_trap(arm_saved_state_t * state)2245 handle_mach_continuous_time_trap(arm_saved_state_t *state)
2246 {
2247 uint64_t now = mach_continuous_time();
2248 saved_state64(state)->x[0] = now;
2249 }
2250
2251
2252 __attribute__((noreturn))
2253 static void
handle_msr_trap(arm_saved_state_t * state,uint64_t esr)2254 handle_msr_trap(arm_saved_state_t *state, uint64_t esr)
2255 {
2256 exception_type_t exception = EXC_BAD_INSTRUCTION;
2257 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
2258 mach_msg_type_number_t numcodes = 2;
2259 uint32_t instr = 0;
2260
2261 if (!is_saved_state64(state)) {
2262 panic("MSR/MRS trap (ESR 0x%llx) from 32-bit state", esr);
2263 }
2264
2265 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2266 panic("MSR/MRS trap (ESR 0x%llx) from kernel", esr);
2267 }
2268
2269 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2270 codes[1] = instr;
2271
2272 exception_triage(exception, codes, numcodes);
2273 __builtin_unreachable();
2274 }
2275
2276 #if __has_feature(ptrauth_calls)
2277 static void
stringify_gpr(unsigned int r,char reg[4])2278 stringify_gpr(unsigned int r, char reg[4])
2279 {
2280 switch (r) {
2281 case 29:
2282 strncpy(reg, "fp", 4);
2283 return;
2284
2285 case 30:
2286 strncpy(reg, "lr", 4);
2287 return;
2288
2289 case 31:
2290 strncpy(reg, "xzr", 4);
2291 return;
2292
2293 default:
2294 snprintf(reg, 4, "x%u", r);
2295 return;
2296 }
2297 }
2298
2299 static void
autxx_instruction_extract_reg(uint32_t instr,char reg[4])2300 autxx_instruction_extract_reg(uint32_t instr, char reg[4])
2301 {
2302 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
2303 stringify_gpr(rd, reg);
2304 }
2305
2306 static const char *
autix_system_instruction_extract_reg(uint32_t instr)2307 autix_system_instruction_extract_reg(uint32_t instr)
2308 {
2309 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
2310 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
2311 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
2312 return "x17";
2313 } else {
2314 return "lr";
2315 }
2316 }
2317
2318 static void
bxrax_instruction_extract_reg(uint32_t instr,char reg[4])2319 bxrax_instruction_extract_reg(uint32_t instr, char reg[4])
2320 {
2321 unsigned int rn = ARM64_INSTR_BxRAx_RN_GET(instr);
2322 stringify_gpr(rn, reg);
2323 }
2324
2325 static void
handle_pac_fail(arm_saved_state_t * state,uint64_t esr)2326 handle_pac_fail(arm_saved_state_t *state, uint64_t esr)
2327 {
2328 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
2329 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL};
2330 mach_msg_type_number_t numcodes = 2;
2331 uint32_t instr = 0;
2332
2333 if (!is_saved_state64(state)) {
2334 panic("PAC failure (ESR 0x%llx) from 32-bit state", esr);
2335 }
2336
2337 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2338
2339 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2340 #define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
2341 #define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
2342 #define BXRAX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while branching to %s"
2343 #define RETAX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while returning"
2344 #define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
2345 #define MAX_PAC_MSG_FMT BXRAX_MSG_FMT
2346
2347 char msg[strlen(MAX_PAC_MSG_FMT)
2348 - strlen("%s") + strlen("IA")
2349 - strlen("%s") + strlen("xzr")
2350 + 1];
2351 ptrauth_key key = (ptrauth_key)(esr & 0x3);
2352 const char *key_str = ptrauth_key_to_string(key);
2353
2354 if (ARM64_INSTR_IS_AUTxx(instr)) {
2355 char reg[4];
2356 autxx_instruction_extract_reg(instr, reg);
2357 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
2358 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) {
2359 const char *reg = autix_system_instruction_extract_reg(instr);
2360 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg);
2361 } else if (ARM64_INSTR_IS_BxRAx(instr)) {
2362 char reg[4];
2363 bxrax_instruction_extract_reg(instr, reg);
2364 snprintf(msg, sizeof(msg), BXRAX_MSG_FMT, key_str, reg);
2365 } else if (ARM64_INSTR_IS_RETAx(instr)) {
2366 snprintf(msg, sizeof(msg), RETAX_MSG_FMT, key_str);
2367 } else {
2368 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str);
2369 }
2370 panic_with_thread_kernel_state(msg, state);
2371 }
2372
2373 codes[1] = instr;
2374
2375 exception_triage(exception, codes, numcodes);
2376 __builtin_unreachable();
2377 }
2378 #endif /* __has_feature(ptrauth_calls) */
2379
2380 __attribute__((noreturn))
2381 static void
handle_bti_fail(arm_saved_state_t * state,uint64_t esr)2382 handle_bti_fail(arm_saved_state_t *state, uint64_t esr)
2383 {
2384 uint32_t btype = (uint32_t) esr & ISS_BTI_BTYPE_MASK;
2385
2386 if (!is_saved_state64(state)) {
2387 /* BTI is an ARMv8 feature, this should not be possible */
2388 panic("BTI failure for 32-bit state? (ESR=0x%llx)", esr);
2389 }
2390
2391 /*
2392 * We currently only expect BTI to be enabled for kernel pages, so panic if
2393 * we detect otherwise.
2394 */
2395 if (!PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2396 panic("Unexpected non-kernel BTI failure? (ESR=0x%llx)", esr);
2397 }
2398
2399 #define BTI_FAIL_PTR_FMT "%04x"
2400 #define BTI_FAIL_MSG_FMT "Kernel BTI failure (BTYPE=0x" BTI_FAIL_PTR_FMT ")"
2401 /* Replace the pointer format with the length of the pointer message+NULL */
2402 char msg[strlen(BTI_FAIL_MSG_FMT) - strlen(BTI_FAIL_PTR_FMT) + 8 + 1];
2403 snprintf(msg, sizeof(msg), BTI_FAIL_MSG_FMT, btype);
2404 panic_with_thread_kernel_state(msg, state);
2405 __builtin_unreachable();
2406 }
2407
2408 static void
handle_user_trapped_instruction32(arm_saved_state_t * state,uint64_t esr)2409 handle_user_trapped_instruction32(arm_saved_state_t *state, uint64_t esr)
2410 {
2411 exception_type_t exception = EXC_BAD_INSTRUCTION;
2412 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
2413 mach_msg_type_number_t numcodes = 2;
2414 uint32_t instr;
2415
2416 if (is_saved_state64(state)) {
2417 panic("ESR (0x%llx) for instruction trapped from U32, but saved state is 64-bit.", esr);
2418 }
2419
2420 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2421 panic("ESR (0x%llx) for instruction trapped from U32, actually came from kernel?", esr);
2422 }
2423
2424 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2425 codes[1] = instr;
2426
2427 exception_triage(exception, codes, numcodes);
2428 __builtin_unreachable();
2429 }
2430
2431 static void
handle_simd_trap(arm_saved_state_t * state,uint64_t esr)2432 handle_simd_trap(arm_saved_state_t *state, uint64_t esr)
2433 {
2434 exception_type_t exception = EXC_BAD_INSTRUCTION;
2435 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
2436 mach_msg_type_number_t numcodes = 2;
2437 uint32_t instr = 0;
2438
2439 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
2440 panic("ESR (0x%llx) for SIMD trap from userland, actually came from kernel?", esr);
2441 }
2442
2443 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
2444 codes[1] = instr;
2445
2446 exception_triage(exception, codes, numcodes);
2447 __builtin_unreachable();
2448 }
2449
2450 void
sleh_irq(arm_saved_state_t * state)2451 sleh_irq(arm_saved_state_t *state)
2452 {
2453 cpu_data_t * cdp __unused = getCpuDatap();
2454 #if MACH_ASSERT
2455 int preemption_level = sleh_get_preemption_level();
2456 #endif
2457
2458
2459 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);
2460
2461 #if USE_APPLEARMSMP
2462 PE_handle_ext_interrupt();
2463 #else
2464 /* Run the registered interrupt handler. */
2465 cdp->interrupt_handler(cdp->interrupt_target,
2466 cdp->interrupt_refCon,
2467 cdp->interrupt_nub,
2468 cdp->interrupt_source);
2469 #endif
2470
2471 entropy_collect();
2472
2473
2474 sleh_interrupt_handler_epilogue();
2475 #if MACH_ASSERT
2476 if (preemption_level != sleh_get_preemption_level()) {
2477 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
2478 }
2479 #endif
2480 }
2481
2482 void
sleh_fiq(arm_saved_state_t * state)2483 sleh_fiq(arm_saved_state_t *state)
2484 {
2485 unsigned int type = DBG_INTR_TYPE_UNKNOWN;
2486 #if MACH_ASSERT
2487 int preemption_level = sleh_get_preemption_level();
2488 #endif
2489
2490 #if MONOTONIC_FIQ
2491 uint64_t pmcr0 = 0, upmsr = 0;
2492 #endif /* MONOTONIC_FIQ */
2493
2494 #if defined(HAS_IPI)
2495 boolean_t is_ipi = FALSE;
2496 uint64_t ipi_sr = 0;
2497
2498 if (gFastIPI) {
2499 MRS(ipi_sr, "S3_5_C15_C1_1");
2500
2501 if (ipi_sr & ARM64_IPISR_IPI_PENDING) {
2502 is_ipi = TRUE;
2503 }
2504 }
2505
2506 if (is_ipi) {
2507 type = DBG_INTR_TYPE_IPI;
2508 } else
2509 #endif /* defined(HAS_IPI) */
2510 if (ml_get_timer_pending()) {
2511 type = DBG_INTR_TYPE_TIMER;
2512 }
2513 #if MONOTONIC_FIQ
2514 /* Consult the PMI sysregs last, after IPI/timer
2515 * classification.
2516 */
2517 else if (mt_pmi_pending(&pmcr0, &upmsr)) {
2518 type = DBG_INTR_TYPE_PMI;
2519 }
2520 #endif /* MONOTONIC_FIQ */
2521
2522 sleh_interrupt_handler_prologue(state, type);
2523
2524 #if APPLEVIRTUALPLATFORM
2525 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1");
2526 #endif
2527
2528 #if defined(HAS_IPI)
2529 if (type == DBG_INTR_TYPE_IPI) {
2530 /*
2531 * Order is important here: we must ack the IPI by writing IPI_SR
2532 * before we call cpu_signal_handler(). Otherwise, there will be
2533 * a window between the completion of pending-signal processing in
2534 * cpu_signal_handler() and the ack during which a newly-issued
2535 * IPI to this CPU may be lost. ISB is required to ensure the msr
2536 * is retired before execution of cpu_signal_handler().
2537 */
2538 MSR("S3_5_C15_C1_1", ARM64_IPISR_IPI_PENDING);
2539 __builtin_arm_isb(ISB_SY);
2540 cpu_signal_handler();
2541 } else
2542 #endif /* defined(HAS_IPI) */
2543 #if MONOTONIC_FIQ
2544 if (type == DBG_INTR_TYPE_PMI) {
2545 INTERRUPT_MASKED_DEBUG_START(mt_fiq, DBG_INTR_TYPE_PMI);
2546 mt_fiq(getCpuDatap(), pmcr0, upmsr);
2547 INTERRUPT_MASKED_DEBUG_END();
2548 } else
2549 #endif /* MONOTONIC_FIQ */
2550 {
2551 /*
2552 * We don't know that this is a timer, but we don't have insight into
2553 * the other interrupts that go down this path.
2554 */
2555
2556 cpu_data_t *cdp = getCpuDatap();
2557
2558 cdp->cpu_decrementer = -1; /* Large */
2559
2560 /*
2561 * ARM64_TODO: whether we're coming from userland is ignored right now.
2562 * We can easily thread it through, but not bothering for the
2563 * moment (AArch32 doesn't either).
2564 */
2565 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_TIMER);
2566 rtclock_intr(TRUE);
2567 INTERRUPT_MASKED_DEBUG_END();
2568 }
2569
2570 #if APPLEVIRTUALPLATFORM
2571 if (iar != GIC_SPURIOUS_IRQ) {
2572 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar);
2573 __builtin_arm_isb(ISB_SY);
2574 }
2575 #endif
2576
2577 sleh_interrupt_handler_epilogue();
2578 #if MACH_ASSERT
2579 if (preemption_level != sleh_get_preemption_level()) {
2580 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level());
2581 }
2582 #endif
2583 }
2584
2585 void
sleh_serror(arm_context_t * context,uint64_t esr,vm_offset_t far)2586 sleh_serror(arm_context_t *context, uint64_t esr, vm_offset_t far)
2587 {
2588 task_vtimer_check(current_thread());
2589
2590 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
2591 esr, VM_KERNEL_ADDRHIDE(far));
2592 arm_saved_state_t *state = &context->ss;
2593 #if MACH_ASSERT
2594 int preemption_level = sleh_get_preemption_level();
2595 #endif
2596
2597 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
2598 /* Sanitize FAR (only if we came from userspace) */
2599 saved_state64(state)->far = 0;
2600 }
2601
2602 ASSERT_CONTEXT_SANITY(context);
2603 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC);
2604 #if MACH_ASSERT
2605 if (preemption_level != sleh_get_preemption_level()) {
2606 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
2607 }
2608 #endif
2609 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
2610 esr, VM_KERNEL_ADDRHIDE(far));
2611 }
2612
2613 void
mach_syscall_trace_exit(unsigned int retval,unsigned int call_number)2614 mach_syscall_trace_exit(unsigned int retval,
2615 unsigned int call_number)
2616 {
2617 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2618 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
2619 DBG_FUNC_END, retval, 0, 0, 0, 0);
2620 }
2621
2622 __attribute__((noreturn))
2623 void
thread_syscall_return(kern_return_t error)2624 thread_syscall_return(kern_return_t error)
2625 {
2626 thread_t thread;
2627 struct arm_saved_state *state;
2628
2629 thread = current_thread();
2630 state = get_user_regs(thread);
2631
2632 assert(is_saved_state64(state));
2633 saved_state64(state)->x[0] = error;
2634
2635 #if MACH_ASSERT
2636 kern_allocation_name_t
2637 prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
2638 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
2639 #endif /* MACH_ASSERT */
2640
2641 if (kdebug_enable) {
2642 /* Invert syscall number (negative for a mach syscall) */
2643 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
2644 }
2645
2646 thread_exception_return();
2647 }
2648
2649 void
syscall_trace(struct arm_saved_state * regs __unused)2650 syscall_trace(
2651 struct arm_saved_state * regs __unused)
2652 {
2653 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */
2654 }
2655
2656 static void
sleh_interrupt_handler_prologue(arm_saved_state_t * state,unsigned int type)2657 sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
2658 {
2659 const bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));
2660
2661 if (is_user == true) {
2662 /* Sanitize FAR (only if the interrupt occurred while the CPU was in usermode) */
2663 saved_state64(state)->far = 0;
2664 }
2665
2666 recount_enter_interrupt();
2667
2668 task_vtimer_check(current_thread());
2669
2670 uint64_t pc = is_user ? get_saved_state_pc(state) :
2671 VM_KERNEL_UNSLIDE(get_saved_state_pc(state));
2672
2673 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
2674 0, pc, is_user, type);
2675 }
2676
2677 static void
sleh_interrupt_handler_epilogue(void)2678 sleh_interrupt_handler_epilogue(void)
2679 {
2680 #if KPERF
2681 kperf_interrupt();
2682 #endif /* KPERF */
2683 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
2684 recount_leave_interrupt();
2685 }
2686
2687 void
sleh_invalid_stack(arm_context_t * context,uint64_t esr __unused,vm_offset_t far __unused)2688 sleh_invalid_stack(arm_context_t *context, uint64_t esr __unused, vm_offset_t far __unused)
2689 {
2690 thread_t thread = current_thread();
2691 vm_offset_t kernel_stack_bottom, sp;
2692
2693 sp = get_saved_state_sp(&context->ss);
2694 vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr;
2695 kernel_stack_bottom = round_page(kstackptr) - KERNEL_STACK_SIZE;
2696
2697 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
2698 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
2699 }
2700
2701 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption).", &context->ss);
2702 }
2703
2704
2705 #if MACH_ASSERT
2706 static int trap_handled;
2707 static const char *
handle_recoverable_kernel_trap(__unused void * tstate,uint16_t comment)2708 handle_recoverable_kernel_trap(
2709 __unused void *tstate,
2710 uint16_t comment)
2711 {
2712 assert(comment == TEST_RECOVERABLE_SOFT_TRAP);
2713
2714 printf("Recoverable trap handled.\n");
2715 trap_handled = 1;
2716
2717 return NULL;
2718 }
2719
2720 KERNEL_BRK_DESCRIPTOR_DEFINE(test_desc,
2721 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_TEST,
2722 .base = TEST_RECOVERABLE_SOFT_TRAP,
2723 .max = TEST_RECOVERABLE_SOFT_TRAP,
2724 .options = BRK_TELEMETRY_OPTIONS_RECOVERABLE_DEFAULT(
2725 /* enable_telemetry */ false),
2726 .handle_breakpoint = handle_recoverable_kernel_trap);
2727
2728 static int
recoverable_kernel_trap_test(__unused int64_t in,int64_t * out)2729 recoverable_kernel_trap_test(__unused int64_t in, int64_t *out)
2730 {
2731 ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP);
2732
2733 *out = trap_handled;
2734 return 0;
2735 }
2736
2737 SYSCTL_TEST_REGISTER(recoverable_kernel_trap, recoverable_kernel_trap_test);
2738
2739 #endif
2740
2741 #if CONFIG_SPTM
2742 /**
2743 * Evaluate the panic lockdown policy for a synchronous EL1 SP0 exception
2744 *
2745 * Returns true if panic lockdown should be initiated (but does not itself do
2746 * so)
2747 */
2748 __SECURITY_STACK_DISALLOWED_PUSH
2749 bool
sleh_panic_lockdown_should_initiate_el1_sp0_sync(uint64_t esr,uint64_t elr,uint64_t far,uint64_t spsr)2750 sleh_panic_lockdown_should_initiate_el1_sp0_sync(uint64_t esr, uint64_t elr,
2751 uint64_t far, uint64_t spsr)
2752 {
2753 const esr_exception_class_t class = ESR_EC(esr);
2754 const bool any_exceptions_masked = spsr & DAIF_STANDARD_DISABLE;
2755
2756 switch (class) {
2757 case ESR_EC_PC_ALIGN: /* PC misaligned (should never happen) */
2758 case ESR_EC_IABORT_EL1: /* Potential iPAC failure (poisoned PC) */
2759 case ESR_EC_PAC_FAIL: { /* FPAC fail */
2760 return true;
2761 }
2762
2763 case ESR_EC_BRK_AARCH64: {
2764 /*
2765 * Breakpoints are used on non-FPAC systems to signal some PAC failures
2766 */
2767 #if HAS_TELEMETRY_KERNEL_BRK
2768 const struct kernel_brk_descriptor *desc;
2769 desc = find_brk_descriptor_by_comment(ISS_BRK_COMMENT(esr));
2770 if (desc && desc->options.recoverable) {
2771 /*
2772 * We matched a breakpoint and it's recoverable, skip lockdown.
2773 */
2774 return false;
2775 }
2776 #endif /* HAS_TELEMETRY_KERNEL_BRK */
2777
2778 /*
2779 * If we don't support telemetry breakpoints and/or didn't match a
2780 * recoverable breakpoint, the exception is fatal.
2781 */
2782 return true;
2783 }
2784
2785 case ESR_EC_DABORT_EL1: {
2786 const struct copyio_recovery_entry *cre =
2787 find_copyio_recovery_entry(elr);
2788 if (cre) {
2789
2790 /*
2791 * copyio faults are recoverable regardless of whether or not
2792 * exceptions are masked.
2793 */
2794 return false;
2795 }
2796
2797
2798 /*
2799 * Heuristic: if FAR != XPAC(FAR), the pointer was likely corrupted
2800 * due to PAC.
2801 */
2802 const uint64_t far_stripped =
2803 (uint64_t)ptrauth_strip((void *)far, ptrauth_key_asda);
2804
2805 if (far != far_stripped) {
2806 /* potential dPAC failure (poisoined address) */
2807 return true;
2808 }
2809
2810 if (any_exceptions_masked && startup_phase >= STARTUP_SUB_LOCKDOWN) {
2811 /*
2812 * Any data abort taken with exceptions masked is fatal if we're
2813 * past early boot.
2814 */
2815 return true;
2816 }
2817
2818 return false;
2819 }
2820
2821 case ESR_EC_UNCATEGORIZED: {
2822 /* Undefined instruction (GDBTRAP for stackshots, etc.) */
2823 return false;
2824 }
2825
2826 case ESR_EC_BTI_FAIL: {
2827 /* Kernel BTI exceptions are recoverable only in telemetry mode */
2828 #ifdef CONFIG_BTI_TELEMETRY
2829 return false;
2830 #else
2831 return true;
2832 #endif /* CONFIG_BTI_TELEMETRY */
2833 }
2834
2835 default: {
2836 if (!any_exceptions_masked) {
2837 /*
2838 * When exceptions are not masked, we default-allow exceptions.
2839 */
2840 return false;
2841 }
2842
2843 if (startup_phase < STARTUP_SUB_LOCKDOWN) {
2844 /*
2845 * Ignore early boot exceptions even if exceptions are masked.
2846 */
2847 return false;
2848 }
2849
2850 /* Default-deny all others when exceptions are masked */
2851 return true;
2852 }
2853 }
2854 }
2855 __SECURITY_STACK_DISALLOWED_POP
2856 #endif /* CONFIG_SPTM */
2857