1 /*
2 * Copyright (c) 2007-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <debug.h>
30
31 #include <types.h>
32
33 #include <mach/mach_types.h>
34 #include <mach/thread_status.h>
35 #include <mach/vm_types.h>
36
37 #include <kern/kern_types.h>
38 #include <kern/task.h>
39 #include <kern/thread.h>
40 #include <kern/misc_protos.h>
41 #include <kern/mach_param.h>
42 #include <kern/spl.h>
43 #include <kern/machine.h>
44 #include <kern/kpc.h>
45 #include <kern/monotonic.h>
46
47 #include <machine/atomic.h>
48 #include <arm64/proc_reg.h>
49 #include <arm64/machine_machdep.h>
50 #include <arm/cpu_data_internal.h>
51 #include <arm/machdep_call.h>
52 #include <arm/misc_protos.h>
53 #include <arm/cpuid.h>
54 #include <arm/cpu_capabilities_public.h>
55
56 #include <vm/vm_map_xnu.h>
57 #include <vm/vm_protos.h>
58
59 #include <sys/kdebug.h>
60
61
62 #include <san/kcov_stksz.h>
63
64 #include <IOKit/IOBSD.h>
65
66 #include <pexpert/arm64/apple_arm64_cpu.h>
67 #include <pexpert/pexpert.h>
68
69 // fixme: rdar://114299113 tracks resolving the supportlib issue with hwtrace features
70
71 extern int debug_task;
72
73 /* zone for debug_state area */
74 ZONE_DEFINE_TYPE(ads_zone, "arm debug state", arm_debug_state_t, ZC_NONE);
75 ZONE_DEFINE_TYPE(user_ss_zone, "user save state", arm_context_t, ZC_NONE);
76 #if HAVE_MACHINE_THREAD_MATRIX_STATE
77 static SECURITY_READ_ONLY_LATE(zone_t) matrix_ss_zone;
78 #endif
79
80
81 #if HAS_ARM_FEAT_SME
82 static SECURITY_READ_ONLY_LATE(uint16_t) sme_svl_b;
83 SECURITY_READ_ONLY_LATE(arm_state_hdr_t) sme_state_hdr;
84 #endif
85
86 void
arm_get_matrix_cpu_state(struct arm_matrix_cpu_state * cpu_state)87 arm_get_matrix_cpu_state(struct arm_matrix_cpu_state *cpu_state)
88 {
89 #if HAS_ARM_FEAT_SME
90 cpu_state->have_sme = arm_sme_version() > 0;
91 if (cpu_state->have_sme) {
92 cpu_state->za_is_enabled = !!(__builtin_arm_rsr64("SVCR") & SVCR_ZA);
93 } else {
94 cpu_state->za_is_enabled = false;
95 }
96 #endif /* HAS_ARM_FEAT_SME */
97
98 #if !HAS_ARM_FEAT_SME
99 #pragma unused(cpu_state)
100 #endif
101 }
102
103 /*
104 * Routine: consider_machine_collect
105 *
106 */
107 void
consider_machine_collect(void)108 consider_machine_collect(void)
109 {
110 pmap_gc();
111 }
112
113 /*
114 * Routine: consider_machine_adjust
115 *
116 */
117 void
consider_machine_adjust(void)118 consider_machine_adjust(void)
119 {
120 }
121
122
123
124 #if HAS_ARM_FEAT_SME
125 static inline bool
machine_thread_has_valid_za(const arm_sme_saved_state_t * _Nullable sme_ss)126 machine_thread_has_valid_za(const arm_sme_saved_state_t *_Nullable sme_ss)
127 {
128 return sme_ss && (sme_ss->svcr & SVCR_ZA);
129 }
130
131 arm_sme_saved_state_t *
machine_thread_get_sme_state(thread_t thread)132 machine_thread_get_sme_state(thread_t thread)
133 {
134 arm_state_hdr_t *hdr = thread->machine.umatrix_hdr;
135 if (hdr) {
136 assert(hdr->flavor == ARM_SME_SAVED_STATE);
137 return thread->machine.usme;
138 }
139
140 return NULL;
141 }
142
143 static void
machine_save_sme_context(thread_t old,arm_sme_saved_state_t * old_sme_ss,const struct arm_matrix_cpu_state * cpu_state)144 machine_save_sme_context(thread_t old, arm_sme_saved_state_t *old_sme_ss, const struct arm_matrix_cpu_state *cpu_state)
145 {
146 /*
147 * Note: we're deliberately not saving old_sme_ss->svcr, since it
148 * already happened on kernel entry. Likewise we're not restoring the
149 * SM bit from new_sme_ss->svcr, since we don't want streaming SVE mode
150 * active while we're in kernel space; we'll put it back on kernel exit.
151 */
152
153 old->machine.tpidr2_el0 = __builtin_arm_rsr64("TPIDR2_EL0");
154
155
156 if (cpu_state->za_is_enabled) {
157 arm_save_sme_za_zt0(&old_sme_ss->context, old_sme_ss->svl_b);
158 }
159 }
160
161 static void
machine_restore_sme_context(thread_t new,const arm_sme_saved_state_t * new_sme_ss,const struct arm_matrix_cpu_state * cpu_state)162 machine_restore_sme_context(thread_t new, const arm_sme_saved_state_t *new_sme_ss, const struct arm_matrix_cpu_state *cpu_state)
163 {
164 __builtin_arm_wsr64("TPIDR2_EL0", new->machine.tpidr2_el0);
165
166 if (new_sme_ss) {
167 if (machine_thread_has_valid_za(new_sme_ss)) {
168 if (!cpu_state->za_is_enabled) {
169 asm volatile ("smstart za");
170 }
171 arm_load_sme_za_zt0(&new_sme_ss->context, new_sme_ss->svl_b);
172 } else if (cpu_state->za_is_enabled) {
173 asm volatile ("smstop za");
174 }
175 }
176 }
177
178 static void
machine_clear_sme_context(const struct arm_matrix_cpu_state * cpu_state)179 machine_clear_sme_context(const struct arm_matrix_cpu_state *cpu_state)
180 {
181 if (cpu_state->za_is_enabled) {
182 asm volatile ("smstop za");
183 }
184 }
185 #endif /* HAS_ARM_FEAT_SME */
186
187
188 #if HAVE_MACHINE_THREAD_MATRIX_STATE
189 static void
machine_trap_el0_matrix_instructions(bool enable_el0_trap,const struct arm_matrix_cpu_state * cpu_state)190 machine_trap_el0_matrix_instructions(bool enable_el0_trap,
191 const struct arm_matrix_cpu_state *cpu_state)
192 {
193 #pragma unused(cpu_state)
194
195 #if HAS_ARM_FEAT_SME
196 if (arm_sme_version()) {
197 arm_sme_trap_at_el0(enable_el0_trap);
198 }
199 #endif
200 }
201
202 static void
machine_switch_matrix_context(thread_t old,thread_t new)203 machine_switch_matrix_context(thread_t old, thread_t new)
204 {
205 struct arm_matrix_cpu_state cpu_state;
206 arm_get_matrix_cpu_state(&cpu_state);
207 bool enable_el0_trap = true;
208
209
210 #if HAS_ARM_FEAT_SME
211 arm_sme_saved_state_t *old_sme_ss = machine_thread_get_sme_state(old);
212 const arm_sme_saved_state_t *new_sme_ss = machine_thread_get_sme_state(new);
213
214 if (cpu_state.have_sme) {
215 machine_save_sme_context(old, old_sme_ss, &cpu_state);
216 }
217 #endif /* HAS_ARM_FEAT_SME */
218
219
220 #if HAS_ARM_FEAT_SME
221 if (cpu_state.have_sme && !new_sme_ss) {
222 machine_clear_sme_context(&cpu_state);
223 }
224 #endif /* HAS_ARM_FEAT_SME */
225
226
227 #if HAS_ARM_FEAT_SME
228 if (cpu_state.have_sme) {
229 machine_restore_sme_context(new, new_sme_ss, &cpu_state);
230 if (new_sme_ss) {
231 enable_el0_trap = false;
232 }
233 }
234 #endif /* HAS_ARM_FEAT_SME */
235
236
237 machine_trap_el0_matrix_instructions(enable_el0_trap, &cpu_state);
238
239 }
240
241
242 #endif /* HAVE_MACHINE_THREAD_MATRIX_STATE */
243
244
245
246
247 static inline void
machine_thread_switch_cpu_data(thread_t old,thread_t new)248 machine_thread_switch_cpu_data(thread_t old, thread_t new)
249 {
250 /*
251 * We build with -fno-strict-aliasing, so the load through temporaries
252 * is required so that this generates a single load / store pair.
253 */
254 cpu_data_t *datap = old->machine.CpuDatap;
255 vm_offset_t base = old->machine.pcpu_data_base_and_cpu_number;
256
257 /* TODO: Should this be ordered? */
258
259 old->machine.CpuDatap = NULL;
260 old->machine.pcpu_data_base_and_cpu_number = 0;
261
262 new->machine.CpuDatap = datap;
263 new->machine.pcpu_data_base_and_cpu_number = base;
264 }
265
266 /**
267 * routine: machine_switch_pmap_and_extended_context
268 *
269 * Helper function used by machine_switch_context and machine_stack_handoff to switch the
270 * extended context and switch the pmap if necessary.
271 *
272 */
273
274 static inline void
machine_switch_pmap_and_extended_context(thread_t old,thread_t new)275 machine_switch_pmap_and_extended_context(thread_t old, thread_t new)
276 {
277 pmap_t new_pmap;
278
279
280
281
282
283 #if HAVE_MACHINE_THREAD_MATRIX_STATE
284 machine_switch_matrix_context(old, new);
285 #endif
286
287
288
289
290 new_pmap = new->map->pmap;
291 bool pmap_changed = old->map->pmap != new_pmap;
292 bool sec_override_changed =
293 #if HAS_MTE
294 ml_thread_get_sec_override(old) != ml_thread_get_sec_override(new);
295 #else
296 false;
297 #endif
298
299 if (pmap_changed || sec_override_changed) {
300 pmap_switch(new_pmap, new);
301 } else {
302 /*
303 * If the thread is preempted while performing cache or TLB maintenance,
304 * it may be migrated to a different CPU between the completion of the relevant
305 * maintenance instruction and the synchronizing DSB. ARM requires that the
306 * synchronizing DSB must be issued *on the PE that issued the maintenance instruction*
307 * in order to guarantee completion of the instruction and visibility of its effects.
308 * Issue DSB here to enforce that guarantee. We only do this for the case in which
309 * the pmap isn't changing, as we expect pmap_switch() to issue DSB when it updates
310 * TTBR0. Note also that cache maintenance may be performed in userspace, so we
311 * cannot further limit this operation e.g. by setting a per-thread flag to indicate
312 * a pending kernel TLB or cache maintenance instruction.
313 */
314 __builtin_arm_dsb(DSB_ISH);
315
316 /*
317 * An ISB is needed for similar userspace reasons to the DSB above. Unlike the DSB
318 * case, the context synchronization needs to happen on the CPU the 'old' thread will
319 * later be scheduled on. We can rely on the fact that when 'old' is later scheduled,
320 * whatever thread it is replacing will go through this function as 'old' and will
321 * issue this ISB on its behalf.
322 */
323 arm_context_switch_requires_sync();
324 }
325
326
327 machine_thread_switch_cpu_data(old, new);
328 }
329
330 /*
331 * Routine: machine_switch_context
332 *
333 */
334 thread_t
machine_switch_context(thread_t old,thread_continue_t continuation,thread_t new)335 machine_switch_context(thread_t old,
336 thread_continue_t continuation,
337 thread_t new)
338 {
339 thread_t retval;
340
341 #if __ARM_PAN_AVAILABLE__
342 if (__improbable(__builtin_arm_rsr("pan") == 0)) {
343 panic("context switch with PAN disabled");
344 }
345 #endif
346
347 #define machine_switch_context_kprintf(x...) \
348 /* kprintf("machine_switch_context: " x) */
349
350 if (old == new) {
351 panic("machine_switch_context");
352 }
353
354 #if CONFIG_CPU_COUNTERS
355 kpc_off_cpu(old);
356 #endif /* CONFIG_CPU_COUNTERS */
357
358 machine_switch_pmap_and_extended_context(old, new);
359
360 machine_switch_context_kprintf("old= %x contination = %x new = %x\n", old, continuation, new);
361
362 retval = Switch_context(old, continuation, new);
363 assert(retval != NULL);
364
365 return retval;
366 }
367
368 boolean_t
machine_thread_on_core(thread_t thread)369 machine_thread_on_core(thread_t thread)
370 {
371 return thread->machine.CpuDatap != NULL;
372 }
373
374 boolean_t
machine_thread_on_core_allow_invalid(thread_t thread)375 machine_thread_on_core_allow_invalid(thread_t thread)
376 {
377 #if HAS_MTE
378 #define _copyin_fn _copyin_atomic64_allow_invalid_kernel_tag
379 #else
380 #define _copyin_fn _copyin_atomic64
381 #endif
382
383 extern int _copyin_fn(const char *src, uint64_t *dst);
384 uint64_t addr;
385
386 /*
387 * Utilize that the thread zone is sequestered which means
388 * that this kernel-to-kernel copyin can't read data
389 * from anything but a thread, zeroed or freed memory.
390 */
391 assert(get_preemption_level() > 0);
392 if (thread == THREAD_NULL) {
393 return false;
394 }
395 thread_require(thread);
396 if (_copyin_fn((void *)&thread->machine.CpuDatap, &addr) == 0) {
397 return addr != 0;
398 }
399 return false;
400
401 #undef _copyin_fn
402 }
403
404
405 /*
406 * Routine: machine_thread_create
407 *
408 */
409 void
machine_thread_create(thread_t thread,task_t task,bool first_thread)410 machine_thread_create(thread_t thread, task_t task, bool first_thread)
411 {
412 #define machine_thread_create_kprintf(x...) \
413 /* kprintf("machine_thread_create: " x) */
414
415 machine_thread_create_kprintf("thread = %x\n", thread);
416
417 if (!first_thread) {
418 thread->machine.CpuDatap = (cpu_data_t *)0;
419 // setting this offset will cause trying to use it to panic
420 thread->machine.pcpu_data_base_and_cpu_number =
421 ml_make_pcpu_base_and_cpu_number(VM_MIN_KERNEL_ADDRESS, 0);
422 }
423 thread->machine.arm_machine_flags = 0;
424 thread->machine.preemption_count = 0;
425 thread->machine.cthread_self = 0;
426 thread->machine.kpcb = NULL;
427 thread->machine.exception_trace_code = 0;
428 #if defined(HAS_APPLE_PAC)
429 thread->machine.rop_pid = task->rop_pid;
430 thread->machine.jop_pid = task->jop_pid;
431 if (task->disable_user_jop) {
432 thread->machine.arm_machine_flags |= ARM_MACHINE_THREAD_DISABLE_USER_JOP;
433 }
434 #endif
435
436
437
438
439 if (task != kernel_task) {
440 /* If this isn't a kernel thread, we'll have userspace state. */
441 arm_context_t *contextData = zalloc_flags(user_ss_zone,
442 Z_WAITOK | Z_NOFAIL);
443
444 #if __has_feature(ptrauth_calls)
445 uint64_t intr = ml_pac_safe_interrupts_disable();
446 zone_require(user_ss_zone, contextData);
447 #endif
448 thread->machine.contextData = contextData;
449 thread->machine.upcb = &contextData->ss;
450 thread->machine.uNeon = &contextData->ns;
451 #if __has_feature(ptrauth_calls)
452 ml_pac_safe_interrupts_restore(intr);
453 #endif
454
455 if (task_has_64Bit_data(task)) {
456 thread->machine.upcb->ash.flavor = ARM_SAVED_STATE64;
457 thread->machine.upcb->ash.count = ARM_SAVED_STATE64_COUNT;
458 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64;
459 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT;
460
461 } else {
462 thread->machine.upcb->ash.flavor = ARM_SAVED_STATE32;
463 thread->machine.upcb->ash.count = ARM_SAVED_STATE32_COUNT;
464 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
465 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
466 }
467 } else {
468 thread->machine.upcb = NULL;
469 thread->machine.uNeon = NULL;
470 thread->machine.contextData = NULL;
471 }
472
473 #if HAVE_MACHINE_THREAD_MATRIX_STATE
474 thread->machine.umatrix_hdr = NULL;
475 #endif
476
477
478 #if HAS_ARM_FEAT_SME
479 thread->machine.tpidr2_el0 = 0;
480 #endif
481
482 bzero(&thread->machine.perfctrl_state, sizeof(thread->machine.perfctrl_state));
483 machine_thread_state_initialize(thread);
484 }
485
486 /*
487 * Routine: machine_thread_process_signature
488 *
489 * Called to allow code signature dependent adjustments to the thread
490 * state. Note that this is usually called twice for the main thread:
491 * Once at thread creation by thread_create, when the signature is
492 * potentially not attached yet (which is usually the case for the
493 * first/main thread of a task), and once after the task's signature
494 * has actually been attached.
495 *
496 */
497 kern_return_t
machine_thread_process_signature(thread_t __unused thread,task_t __unused task)498 machine_thread_process_signature(thread_t __unused thread, task_t __unused task)
499 {
500 kern_return_t result = KERN_SUCCESS;
501
502 /*
503 * Reset to default state.
504 *
505 * In general, this function must not assume anything about the
506 * previous signature dependent thread state.
507 *
508 * At least at the time of writing this, threads don't transition
509 * to different code signatures, so each thread this function
510 * operates on is "fresh" in the sense that
511 * machine_thread_process_signature() has either not even been
512 * called on it yet, or only been called as part of thread
513 * creation when there was no signature yet.
514 *
515 * But for easier reasoning, and to prevent future bugs, this
516 * function should always recalculate all signature-dependent
517 * thread state, as if the signature could actually change from an
518 * actual signature to another.
519 */
520 #if !__ARM_KERNEL_PROTECT__
521 thread->machine.arm_machine_flags &= ~(ARM_MACHINE_THREAD_PRESERVE_X18);
522 #endif /* !__ARM_KERNEL_PROTECT__ */
523 thread->machine.arm_machine_flags &= ~(ARM_MACHINE_THREAD_USES_1GHZ_TIMBASE);
524
525 /*
526 * Set signature dependent state.
527 */
528 if (task != kernel_task && task_has_64Bit_data(task)) {
529 #if !__ARM_KERNEL_PROTECT__
530 #if CONFIG_ROSETTA
531 if (task_is_translated(task)) {
532 /* Note that for x86_64 translation specifically, the
533 * context switch path implicitly switches x18 regardless
534 * of this flag. */
535 thread->machine.arm_machine_flags |= ARM_MACHINE_THREAD_PRESERVE_X18;
536 }
537 #endif /* CONFIG_ROSETTA */
538
539 if (task->preserve_x18) {
540 thread->machine.arm_machine_flags |= ARM_MACHINE_THREAD_PRESERVE_X18;
541 }
542 #endif /* !__ARM_KERNEL_PROTECT__ */
543
544 if (task->uses_1ghz_timebase) {
545 thread->machine.arm_machine_flags |= ARM_MACHINE_THREAD_USES_1GHZ_TIMBASE;
546 }
547 } else {
548 #if !__ARM_KERNEL_PROTECT__
549 /*
550 * For informational value only, context switch only trashes
551 * x18 for user threads. (Except for devices with
552 * __ARM_KERNEL_PROTECT__, which make real destructive use of
553 * x18.)
554 */
555 thread->machine.arm_machine_flags |= ARM_MACHINE_THREAD_PRESERVE_X18;
556 #endif /* !__ARM_KERNEL_PROTECT__ */
557 thread->machine.arm_machine_flags |= ARM_MACHINE_THREAD_USES_1GHZ_TIMBASE;
558 }
559
560 /**
561 * Make sure the machine flags are observed before the thread becomes available
562 * to run in user mode, especially in the posix_spawn() path.
563 */
564 os_atomic_thread_fence(release);
565 return result;
566 }
567
568 /*
569 * Routine: machine_thread_destroy
570 *
571 */
572 void
machine_thread_destroy(thread_t thread)573 machine_thread_destroy(thread_t thread)
574 {
575 arm_context_t *thread_user_ss;
576
577 if (thread->machine.contextData) {
578 /* Disassociate the user save state from the thread before we free it. */
579 thread_user_ss = thread->machine.contextData;
580 thread->machine.upcb = NULL;
581 thread->machine.uNeon = NULL;
582 thread->machine.contextData = NULL;
583
584 #if HAVE_MACHINE_THREAD_MATRIX_STATE
585 if (thread->machine.umatrix_hdr) {
586 zfree(matrix_ss_zone, thread->machine.umatrix_hdr);
587 thread->machine.umatrix_hdr = NULL;
588 }
589 #endif /* HAVE_MACHINE_THREAD_MATRIX_STATE */
590
591 zfree(user_ss_zone, thread_user_ss);
592 }
593
594 if (thread->machine.DebugData != NULL) {
595 if (thread->machine.DebugData == getCpuDatap()->cpu_user_debug) {
596 arm_debug_set(NULL);
597 }
598
599 if (os_ref_release(&thread->machine.DebugData->ref) == 0) {
600 zfree(ads_zone, thread->machine.DebugData);
601 }
602 }
603 }
604
605
606 #if HAS_ARM_FEAT_SME
607 static arm_sme_saved_state_t *
zalloc_sme_saved_state(void)608 zalloc_sme_saved_state(void)
609 {
610 arm_sme_saved_state_t *sme_ss = zalloc_flags(matrix_ss_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL);
611 sme_ss->hdr.flavor = ARM_SME_SAVED_STATE;
612 sme_ss->hdr.count = arm_sme_saved_state_count(sme_svl_b);
613 sme_ss->svl_b = sme_svl_b;
614 return sme_ss;
615 }
616
617 kern_return_t
machine_thread_sme_state_alloc(thread_t thread)618 machine_thread_sme_state_alloc(thread_t thread)
619 {
620 assert(arm_sme_version());
621
622
623 if (thread->machine.usme) {
624 panic("thread %p already has SME saved state %p",
625 thread, thread->machine.usme);
626 }
627
628 arm_sme_saved_state_t *sme_ss = zalloc_sme_saved_state();
629 disable_preemption();
630
631 arm_sme_trap_at_el0(false);
632 __builtin_arm_isb(ISB_SY);
633 thread->machine.usme = sme_ss;
634
635 enable_preemption();
636
637 return KERN_SUCCESS;
638 }
639
640 static void
machine_thread_sme_state_dup(const arm_sme_saved_state_t * src_sme_ss,thread_t target)641 machine_thread_sme_state_dup(const arm_sme_saved_state_t *src_sme_ss, thread_t target)
642 {
643 arm_sme_saved_state_t *sme_ss = zalloc_sme_saved_state();
644 assert(sme_ss->svl_b == src_sme_ss->svl_b);
645
646 arm_sme_context_t *context = &sme_ss->context;
647 uint16_t svl_b = sme_ss->svl_b;
648
649 sme_ss->svcr = src_sme_ss->svcr;
650 /* Z and P are saved on kernel entry. ZA and ZT0 may be stale. */
651 if (sme_ss->svcr & SVCR_SM) {
652 const arm_sme_context_t *src_context = &src_sme_ss->context;
653 memcpy(arm_sme_z(context), const_arm_sme_z(src_context), arm_sme_z_size(svl_b));
654 memcpy(arm_sme_p(context, svl_b), const_arm_sme_p(src_context, svl_b), arm_sme_p_size(svl_b));
655 }
656 if (sme_ss->svcr & SVCR_ZA) {
657 arm_save_sme_za_zt0(context, svl_b);
658 }
659
660 target->machine.usme = sme_ss;
661 }
662 #endif /* HAS_ARM_FEAT_SME */
663
664 #if HAVE_MACHINE_THREAD_MATRIX_STATE
665 void
machine_thread_matrix_state_dup(thread_t target)666 machine_thread_matrix_state_dup(thread_t target)
667 {
668 assert(!target->machine.umatrix_hdr);
669 thread_t thread = current_thread();
670
671 #if HAS_ARM_FEAT_SME
672 /*
673 * TPIDR2_EL0 is accessible even when there's no active matrix state.
674 * xnu normally only spills it during context-switch events, so
675 * current_thread()->machine.tpidr2_el0 may not hold the current value.
676 */
677 if (arm_sme_version()) {
678 target->machine.tpidr2_el0 = __builtin_arm_rsr64("TPIDR2_EL0");
679 }
680
681 const arm_sme_saved_state_t *sme_ss = machine_thread_get_sme_state(thread);
682 if (sme_ss) {
683 machine_thread_sme_state_dup(sme_ss, target);
684 return;
685 }
686 #endif
687
688 }
689 #endif /* HAVE_MACHINE_THREAD_MATRIX_STATE */
690
691 /*
692 * Routine: machine_thread_init
693 *
694 */
695 void
machine_thread_init(void)696 machine_thread_init(void)
697 {
698 #if HAVE_MACHINE_THREAD_MATRIX_STATE
699 vm_size_t matrix_ss_size = 0;
700 zone_create_flags_t matrix_ss_flags = ZC_NONE;
701
702
703 #if HAS_ARM_FEAT_SME
704 if (arm_sme_version()) {
705 sme_svl_b = arm_sme_svl_b();
706 sme_state_hdr.flavor = ARM_SME_SAVED_STATE;
707 sme_state_hdr.count = arm_sme_saved_state_count(sme_svl_b);
708 matrix_ss_size = MAX(matrix_ss_size, sme_state_hdr.count * sizeof(unsigned int));
709 }
710 #endif /* HAS_ARM_FEAT_SME */
711
712 if (matrix_ss_size) {
713 matrix_ss_zone = zone_create_ext("matrix saved state", matrix_ss_size, matrix_ss_flags, ZONE_ID_ANY, NULL);
714 }
715 #endif /* HAVE_MACHINE_THREAD_MATRIX_STATE */
716 }
717
718 /*
719 * Routine: machine_thread_template_init
720 *
721 */
722 void
machine_thread_template_init(thread_t __unused thr_template)723 machine_thread_template_init(thread_t __unused thr_template)
724 {
725 /* Nothing to do on this platform. */
726 }
727
728 /*
729 * Routine: get_useraddr
730 *
731 */
732 user_addr_t
get_useraddr()733 get_useraddr()
734 {
735 return get_saved_state_pc(current_thread()->machine.upcb);
736 }
737
738 /*
739 * Routine: machine_stack_detach
740 *
741 */
742 vm_offset_t
machine_stack_detach(thread_t thread)743 machine_stack_detach(thread_t thread)
744 {
745 vm_offset_t stack;
746
747 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH),
748 (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0);
749
750 stack = thread->kernel_stack;
751 #if CONFIG_STKSZ
752 kcov_stksz_set_thread_stack(thread, stack);
753 #endif
754 thread->kernel_stack = 0;
755 thread->machine.kstackptr = NULL;
756
757 return stack;
758 }
759
760
761 /*
762 * Routine: machine_stack_attach
763 *
764 */
765 void
machine_stack_attach(thread_t thread,vm_offset_t stack)766 machine_stack_attach(thread_t thread,
767 vm_offset_t stack)
768 {
769 struct arm_kernel_context *context;
770 struct arm_kernel_saved_state *savestate;
771 struct arm_kernel_neon_saved_state *neon_savestate;
772 uint32_t current_el;
773
774 #define machine_stack_attach_kprintf(x...) \
775 /* kprintf("machine_stack_attach: " x) */
776
777 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH),
778 (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0);
779
780 thread->kernel_stack = stack;
781 #if CONFIG_STKSZ
782 kcov_stksz_set_thread_stack(thread, 0);
783 #endif
784 void *kstackptr = (void *)(stack + kernel_stack_size - sizeof(struct thread_kernel_state));
785 thread->machine.kstackptr = kstackptr;
786 thread_initialize_kernel_state(thread);
787
788 machine_stack_attach_kprintf("kstackptr: %lx\n", (vm_address_t)kstackptr);
789
790 current_el = (uint32_t) __builtin_arm_rsr64("CurrentEL");
791 context = &((thread_kernel_state_t) thread->machine.kstackptr)->machine;
792 savestate = &context->ss;
793 savestate->fp = 0;
794 savestate->sp = (uint64_t)kstackptr;
795 savestate->pc_was_in_userspace = false;
796 #if defined(HAS_APPLE_PAC)
797 /* Sign the initial kernel stack saved state */
798 uint64_t intr = ml_pac_safe_interrupts_disable();
799 asm volatile (
800 "adrp x17, _thread_continue@page" "\n"
801 "add x17, x17, _thread_continue@pageoff" "\n"
802 "ldr x16, [%[ss], %[SS64_SP]]" "\n"
803 "pacia1716" "\n"
804 "str x17, [%[ss], %[SS64_LR]]" "\n"
805 :
806 : [ss] "r"(&context->ss),
807 [SS64_SP] "i"(offsetof(struct arm_kernel_saved_state, sp)),
808 [SS64_LR] "i"(offsetof(struct arm_kernel_saved_state, lr))
809 : "x16", "x17"
810 );
811 ml_pac_safe_interrupts_restore(intr);
812 #else
813 savestate->lr = (uintptr_t)thread_continue;
814 #endif /* defined(HAS_APPLE_PAC) */
815 neon_savestate = &context->ns;
816 neon_savestate->fpcr = FPCR_DEFAULT;
817 machine_stack_attach_kprintf("thread = %p pc = %llx, sp = %llx\n", thread, savestate->lr, savestate->sp);
818 }
819
820
821 /*
822 * Routine: machine_stack_handoff
823 *
824 */
825 void
machine_stack_handoff(thread_t old,thread_t new)826 machine_stack_handoff(thread_t old,
827 thread_t new)
828 {
829 vm_offset_t stack;
830
831 #if __ARM_PAN_AVAILABLE__
832 if (__improbable(__builtin_arm_rsr("pan") == 0)) {
833 panic("stack handoff with PAN disabled");
834 }
835 #endif
836
837 #if CONFIG_CPU_COUNTERS
838 kpc_off_cpu(old);
839 #endif /* CONFIG_CPU_COUNTERS */
840
841 stack = machine_stack_detach(old);
842 #if CONFIG_STKSZ
843 kcov_stksz_set_thread_stack(new, 0);
844 #endif
845 new->kernel_stack = stack;
846 void *kstackptr = (void *)(stack + kernel_stack_size - sizeof(struct thread_kernel_state));
847 new->machine.kstackptr = kstackptr;
848 if (stack == old->reserved_stack) {
849 assert(new->reserved_stack);
850 old->reserved_stack = new->reserved_stack;
851 #if KASAN_TBI
852 kasan_unpoison_stack(old->reserved_stack, kernel_stack_size);
853 #endif /* KASAN_TBI */
854 new->reserved_stack = stack;
855 }
856
857 machine_switch_pmap_and_extended_context(old, new);
858
859 machine_set_current_thread(new);
860 thread_initialize_kernel_state(new);
861 }
862
863
864 /*
865 * Routine: call_continuation
866 *
867 */
868 void
call_continuation(thread_continue_t continuation,void * parameter,wait_result_t wresult,boolean_t enable_interrupts)869 call_continuation(thread_continue_t continuation,
870 void *parameter,
871 wait_result_t wresult,
872 boolean_t enable_interrupts)
873 {
874 #define call_continuation_kprintf(x...) \
875 /* kprintf("call_continuation_kprintf:" x) */
876
877 call_continuation_kprintf("thread = %p continuation = %p, stack = %lx\n",
878 current_thread(), continuation, current_thread()->machine.kstackptr);
879 Call_continuation(continuation, parameter, wresult, enable_interrupts);
880 }
881
882 #define SET_DBGBCRn(n, value, accum) \
883 __asm__ volatile( \
884 "msr DBGBCR" #n "_EL1, %[val]\n" \
885 "orr %[result], %[result], %[val]\n" \
886 : [result] "+r"(accum) : [val] "r"((value)))
887
888 #define SET_DBGBVRn(n, value) \
889 __asm__ volatile("msr DBGBVR" #n "_EL1, %0" : : "r"(value))
890
891 #define SET_DBGWCRn(n, value, accum) \
892 __asm__ volatile( \
893 "msr DBGWCR" #n "_EL1, %[val]\n" \
894 "orr %[result], %[result], %[val]\n" \
895 : [result] "+r"(accum) : [val] "r"((value)))
896
897 #define SET_DBGWVRn(n, value) \
898 __asm__ volatile("msr DBGWVR" #n "_EL1, %0" : : "r"(value))
899
900 void
arm_debug_set32(arm_debug_state_t * debug_state)901 arm_debug_set32(arm_debug_state_t *debug_state)
902 {
903 struct cpu_data * cpu_data_ptr;
904 arm_debug_info_t * debug_info = arm_debug_info();
905 boolean_t intr;
906 arm_debug_state_t off_state;
907 arm_debug_state_t *cpu_debug;
908 uint64_t all_ctrls = 0;
909
910 // Non-developers should never need to have hardware break/watchpoints
911 // set on their phones.
912 extern bool developer_mode_state(void);
913 if (!developer_mode_state()) {
914 return;
915 }
916
917 intr = ml_set_interrupts_enabled(FALSE);
918 cpu_data_ptr = getCpuDatap();
919 cpu_debug = cpu_data_ptr->cpu_user_debug;
920
921 /*
922 * Retain and set new per-cpu state.
923 * Reference count does not matter when turning off debug state.
924 */
925 if (debug_state == NULL) {
926 bzero(&off_state, sizeof(off_state));
927 cpu_data_ptr->cpu_user_debug = NULL;
928 debug_state = &off_state;
929 } else {
930 os_ref_retain(&debug_state->ref);
931 cpu_data_ptr->cpu_user_debug = debug_state;
932 }
933
934 /* Release previous debug state. */
935 if (cpu_debug != NULL) {
936 if (os_ref_release(&cpu_debug->ref) == 0) {
937 zfree(ads_zone, cpu_debug);
938 }
939 }
940
941 switch (debug_info->num_breakpoint_pairs) {
942 case 16:
943 SET_DBGBVRn(15, (uint64_t)debug_state->uds.ds32.bvr[15]);
944 SET_DBGBCRn(15, (uint64_t)debug_state->uds.ds32.bcr[15], all_ctrls);
945 OS_FALLTHROUGH;
946 case 15:
947 SET_DBGBVRn(14, (uint64_t)debug_state->uds.ds32.bvr[14]);
948 SET_DBGBCRn(14, (uint64_t)debug_state->uds.ds32.bcr[14], all_ctrls);
949 OS_FALLTHROUGH;
950 case 14:
951 SET_DBGBVRn(13, (uint64_t)debug_state->uds.ds32.bvr[13]);
952 SET_DBGBCRn(13, (uint64_t)debug_state->uds.ds32.bcr[13], all_ctrls);
953 OS_FALLTHROUGH;
954 case 13:
955 SET_DBGBVRn(12, (uint64_t)debug_state->uds.ds32.bvr[12]);
956 SET_DBGBCRn(12, (uint64_t)debug_state->uds.ds32.bcr[12], all_ctrls);
957 OS_FALLTHROUGH;
958 case 12:
959 SET_DBGBVRn(11, (uint64_t)debug_state->uds.ds32.bvr[11]);
960 SET_DBGBCRn(11, (uint64_t)debug_state->uds.ds32.bcr[11], all_ctrls);
961 OS_FALLTHROUGH;
962 case 11:
963 SET_DBGBVRn(10, (uint64_t)debug_state->uds.ds32.bvr[10]);
964 SET_DBGBCRn(10, (uint64_t)debug_state->uds.ds32.bcr[10], all_ctrls);
965 OS_FALLTHROUGH;
966 case 10:
967 SET_DBGBVRn(9, (uint64_t)debug_state->uds.ds32.bvr[9]);
968 SET_DBGBCRn(9, (uint64_t)debug_state->uds.ds32.bcr[9], all_ctrls);
969 OS_FALLTHROUGH;
970 case 9:
971 SET_DBGBVRn(8, (uint64_t)debug_state->uds.ds32.bvr[8]);
972 SET_DBGBCRn(8, (uint64_t)debug_state->uds.ds32.bcr[8], all_ctrls);
973 OS_FALLTHROUGH;
974 case 8:
975 SET_DBGBVRn(7, (uint64_t)debug_state->uds.ds32.bvr[7]);
976 SET_DBGBCRn(7, (uint64_t)debug_state->uds.ds32.bcr[7], all_ctrls);
977 OS_FALLTHROUGH;
978 case 7:
979 SET_DBGBVRn(6, (uint64_t)debug_state->uds.ds32.bvr[6]);
980 SET_DBGBCRn(6, (uint64_t)debug_state->uds.ds32.bcr[6], all_ctrls);
981 OS_FALLTHROUGH;
982 case 6:
983 SET_DBGBVRn(5, (uint64_t)debug_state->uds.ds32.bvr[5]);
984 SET_DBGBCRn(5, (uint64_t)debug_state->uds.ds32.bcr[5], all_ctrls);
985 OS_FALLTHROUGH;
986 case 5:
987 SET_DBGBVRn(4, (uint64_t)debug_state->uds.ds32.bvr[4]);
988 SET_DBGBCRn(4, (uint64_t)debug_state->uds.ds32.bcr[4], all_ctrls);
989 OS_FALLTHROUGH;
990 case 4:
991 SET_DBGBVRn(3, (uint64_t)debug_state->uds.ds32.bvr[3]);
992 SET_DBGBCRn(3, (uint64_t)debug_state->uds.ds32.bcr[3], all_ctrls);
993 OS_FALLTHROUGH;
994 case 3:
995 SET_DBGBVRn(2, (uint64_t)debug_state->uds.ds32.bvr[2]);
996 SET_DBGBCRn(2, (uint64_t)debug_state->uds.ds32.bcr[2], all_ctrls);
997 OS_FALLTHROUGH;
998 case 2:
999 SET_DBGBVRn(1, (uint64_t)debug_state->uds.ds32.bvr[1]);
1000 SET_DBGBCRn(1, (uint64_t)debug_state->uds.ds32.bcr[1], all_ctrls);
1001 OS_FALLTHROUGH;
1002 case 1:
1003 SET_DBGBVRn(0, (uint64_t)debug_state->uds.ds32.bvr[0]);
1004 SET_DBGBCRn(0, (uint64_t)debug_state->uds.ds32.bcr[0], all_ctrls);
1005 OS_FALLTHROUGH;
1006 default:
1007 break;
1008 }
1009
1010 switch (debug_info->num_watchpoint_pairs) {
1011 case 16:
1012 SET_DBGWVRn(15, (uint64_t)debug_state->uds.ds32.wvr[15]);
1013 SET_DBGWCRn(15, (uint64_t)debug_state->uds.ds32.wcr[15], all_ctrls);
1014 OS_FALLTHROUGH;
1015 case 15:
1016 SET_DBGWVRn(14, (uint64_t)debug_state->uds.ds32.wvr[14]);
1017 SET_DBGWCRn(14, (uint64_t)debug_state->uds.ds32.wcr[14], all_ctrls);
1018 OS_FALLTHROUGH;
1019 case 14:
1020 SET_DBGWVRn(13, (uint64_t)debug_state->uds.ds32.wvr[13]);
1021 SET_DBGWCRn(13, (uint64_t)debug_state->uds.ds32.wcr[13], all_ctrls);
1022 OS_FALLTHROUGH;
1023 case 13:
1024 SET_DBGWVRn(12, (uint64_t)debug_state->uds.ds32.wvr[12]);
1025 SET_DBGWCRn(12, (uint64_t)debug_state->uds.ds32.wcr[12], all_ctrls);
1026 OS_FALLTHROUGH;
1027 case 12:
1028 SET_DBGWVRn(11, (uint64_t)debug_state->uds.ds32.wvr[11]);
1029 SET_DBGWCRn(11, (uint64_t)debug_state->uds.ds32.wcr[11], all_ctrls);
1030 OS_FALLTHROUGH;
1031 case 11:
1032 SET_DBGWVRn(10, (uint64_t)debug_state->uds.ds32.wvr[10]);
1033 SET_DBGWCRn(10, (uint64_t)debug_state->uds.ds32.wcr[10], all_ctrls);
1034 OS_FALLTHROUGH;
1035 case 10:
1036 SET_DBGWVRn(9, (uint64_t)debug_state->uds.ds32.wvr[9]);
1037 SET_DBGWCRn(9, (uint64_t)debug_state->uds.ds32.wcr[9], all_ctrls);
1038 OS_FALLTHROUGH;
1039 case 9:
1040 SET_DBGWVRn(8, (uint64_t)debug_state->uds.ds32.wvr[8]);
1041 SET_DBGWCRn(8, (uint64_t)debug_state->uds.ds32.wcr[8], all_ctrls);
1042 OS_FALLTHROUGH;
1043 case 8:
1044 SET_DBGWVRn(7, (uint64_t)debug_state->uds.ds32.wvr[7]);
1045 SET_DBGWCRn(7, (uint64_t)debug_state->uds.ds32.wcr[7], all_ctrls);
1046 OS_FALLTHROUGH;
1047 case 7:
1048 SET_DBGWVRn(6, (uint64_t)debug_state->uds.ds32.wvr[6]);
1049 SET_DBGWCRn(6, (uint64_t)debug_state->uds.ds32.wcr[6], all_ctrls);
1050 OS_FALLTHROUGH;
1051 case 6:
1052 SET_DBGWVRn(5, (uint64_t)debug_state->uds.ds32.wvr[5]);
1053 SET_DBGWCRn(5, (uint64_t)debug_state->uds.ds32.wcr[5], all_ctrls);
1054 OS_FALLTHROUGH;
1055 case 5:
1056 SET_DBGWVRn(4, (uint64_t)debug_state->uds.ds32.wvr[4]);
1057 SET_DBGWCRn(4, (uint64_t)debug_state->uds.ds32.wcr[4], all_ctrls);
1058 OS_FALLTHROUGH;
1059 case 4:
1060 SET_DBGWVRn(3, (uint64_t)debug_state->uds.ds32.wvr[3]);
1061 SET_DBGWCRn(3, (uint64_t)debug_state->uds.ds32.wcr[3], all_ctrls);
1062 OS_FALLTHROUGH;
1063 case 3:
1064 SET_DBGWVRn(2, (uint64_t)debug_state->uds.ds32.wvr[2]);
1065 SET_DBGWCRn(2, (uint64_t)debug_state->uds.ds32.wcr[2], all_ctrls);
1066 OS_FALLTHROUGH;
1067 case 2:
1068 SET_DBGWVRn(1, (uint64_t)debug_state->uds.ds32.wvr[1]);
1069 SET_DBGWCRn(1, (uint64_t)debug_state->uds.ds32.wcr[1], all_ctrls);
1070 OS_FALLTHROUGH;
1071 case 1:
1072 SET_DBGWVRn(0, (uint64_t)debug_state->uds.ds32.wvr[0]);
1073 SET_DBGWCRn(0, (uint64_t)debug_state->uds.ds32.wcr[0], all_ctrls);
1074 OS_FALLTHROUGH;
1075 default:
1076 break;
1077 }
1078
1079 #if defined(CONFIG_KERNEL_INTEGRITY)
1080 if ((all_ctrls & (ARM_DBG_CR_MODE_CONTROL_PRIVILEGED | ARM_DBG_CR_HIGHER_MODE_ENABLE)) != 0) {
1081 panic("sorry, self-hosted debug is not supported: 0x%llx", all_ctrls);
1082 }
1083 #endif
1084
1085 /*
1086 * Breakpoint/Watchpoint Enable
1087 */
1088 if (all_ctrls != 0) {
1089 update_mdscr(0, 0x8000); // MDSCR_EL1[MDE]
1090 } else {
1091 update_mdscr(0x8000, 0);
1092 }
1093
1094 /*
1095 * Software debug single step enable
1096 */
1097 if (debug_state->uds.ds32.mdscr_el1 & 0x1) {
1098 update_mdscr(0, 1); // MDSCR_EL1[SS]
1099
1100 mask_user_saved_state_cpsr(current_thread()->machine.upcb, PSR64_SS, 0);
1101 } else {
1102 update_mdscr(0x1, 0);
1103 }
1104
1105 __builtin_arm_isb(ISB_SY);
1106 (void) ml_set_interrupts_enabled(intr);
1107 }
1108
1109 void
arm_debug_set64(arm_debug_state_t * debug_state)1110 arm_debug_set64(arm_debug_state_t *debug_state)
1111 {
1112 struct cpu_data * cpu_data_ptr;
1113 arm_debug_info_t * debug_info = arm_debug_info();
1114 boolean_t intr;
1115 arm_debug_state_t off_state;
1116 arm_debug_state_t *cpu_debug;
1117 uint64_t all_ctrls = 0;
1118
1119 // Non-developers should never need to have hardware break/watchpoints
1120 // set on their phones.
1121 extern bool developer_mode_state(void);
1122 if (!developer_mode_state()) {
1123 return;
1124 }
1125
1126 intr = ml_set_interrupts_enabled(FALSE);
1127 cpu_data_ptr = getCpuDatap();
1128 cpu_debug = cpu_data_ptr->cpu_user_debug;
1129
1130 /*
1131 * Retain and set new per-cpu state.
1132 * Reference count does not matter when turning off debug state.
1133 */
1134 if (debug_state == NULL) {
1135 bzero(&off_state, sizeof(off_state));
1136 cpu_data_ptr->cpu_user_debug = NULL;
1137 debug_state = &off_state;
1138 } else {
1139 os_ref_retain(&debug_state->ref);
1140 cpu_data_ptr->cpu_user_debug = debug_state;
1141 }
1142
1143 /* Release previous debug state. */
1144 if (cpu_debug != NULL) {
1145 if (os_ref_release(&cpu_debug->ref) == 0) {
1146 zfree(ads_zone, cpu_debug);
1147 }
1148 }
1149
1150 switch (debug_info->num_breakpoint_pairs) {
1151 case 16:
1152 SET_DBGBVRn(15, debug_state->uds.ds64.bvr[15]);
1153 SET_DBGBCRn(15, (uint64_t)debug_state->uds.ds64.bcr[15], all_ctrls);
1154 OS_FALLTHROUGH;
1155 case 15:
1156 SET_DBGBVRn(14, debug_state->uds.ds64.bvr[14]);
1157 SET_DBGBCRn(14, (uint64_t)debug_state->uds.ds64.bcr[14], all_ctrls);
1158 OS_FALLTHROUGH;
1159 case 14:
1160 SET_DBGBVRn(13, debug_state->uds.ds64.bvr[13]);
1161 SET_DBGBCRn(13, (uint64_t)debug_state->uds.ds64.bcr[13], all_ctrls);
1162 OS_FALLTHROUGH;
1163 case 13:
1164 SET_DBGBVRn(12, debug_state->uds.ds64.bvr[12]);
1165 SET_DBGBCRn(12, (uint64_t)debug_state->uds.ds64.bcr[12], all_ctrls);
1166 OS_FALLTHROUGH;
1167 case 12:
1168 SET_DBGBVRn(11, debug_state->uds.ds64.bvr[11]);
1169 SET_DBGBCRn(11, (uint64_t)debug_state->uds.ds64.bcr[11], all_ctrls);
1170 OS_FALLTHROUGH;
1171 case 11:
1172 SET_DBGBVRn(10, debug_state->uds.ds64.bvr[10]);
1173 SET_DBGBCRn(10, (uint64_t)debug_state->uds.ds64.bcr[10], all_ctrls);
1174 OS_FALLTHROUGH;
1175 case 10:
1176 SET_DBGBVRn(9, debug_state->uds.ds64.bvr[9]);
1177 SET_DBGBCRn(9, (uint64_t)debug_state->uds.ds64.bcr[9], all_ctrls);
1178 OS_FALLTHROUGH;
1179 case 9:
1180 SET_DBGBVRn(8, debug_state->uds.ds64.bvr[8]);
1181 SET_DBGBCRn(8, (uint64_t)debug_state->uds.ds64.bcr[8], all_ctrls);
1182 OS_FALLTHROUGH;
1183 case 8:
1184 SET_DBGBVRn(7, debug_state->uds.ds64.bvr[7]);
1185 SET_DBGBCRn(7, (uint64_t)debug_state->uds.ds64.bcr[7], all_ctrls);
1186 OS_FALLTHROUGH;
1187 case 7:
1188 SET_DBGBVRn(6, debug_state->uds.ds64.bvr[6]);
1189 SET_DBGBCRn(6, (uint64_t)debug_state->uds.ds64.bcr[6], all_ctrls);
1190 OS_FALLTHROUGH;
1191 case 6:
1192 SET_DBGBVRn(5, debug_state->uds.ds64.bvr[5]);
1193 SET_DBGBCRn(5, (uint64_t)debug_state->uds.ds64.bcr[5], all_ctrls);
1194 OS_FALLTHROUGH;
1195 case 5:
1196 SET_DBGBVRn(4, debug_state->uds.ds64.bvr[4]);
1197 SET_DBGBCRn(4, (uint64_t)debug_state->uds.ds64.bcr[4], all_ctrls);
1198 OS_FALLTHROUGH;
1199 case 4:
1200 SET_DBGBVRn(3, debug_state->uds.ds64.bvr[3]);
1201 SET_DBGBCRn(3, (uint64_t)debug_state->uds.ds64.bcr[3], all_ctrls);
1202 OS_FALLTHROUGH;
1203 case 3:
1204 SET_DBGBVRn(2, debug_state->uds.ds64.bvr[2]);
1205 SET_DBGBCRn(2, (uint64_t)debug_state->uds.ds64.bcr[2], all_ctrls);
1206 OS_FALLTHROUGH;
1207 case 2:
1208 SET_DBGBVRn(1, debug_state->uds.ds64.bvr[1]);
1209 SET_DBGBCRn(1, (uint64_t)debug_state->uds.ds64.bcr[1], all_ctrls);
1210 OS_FALLTHROUGH;
1211 case 1:
1212 SET_DBGBVRn(0, debug_state->uds.ds64.bvr[0]);
1213 SET_DBGBCRn(0, (uint64_t)debug_state->uds.ds64.bcr[0], all_ctrls);
1214 OS_FALLTHROUGH;
1215 default:
1216 break;
1217 }
1218
1219 switch (debug_info->num_watchpoint_pairs) {
1220 case 16:
1221 SET_DBGWVRn(15, debug_state->uds.ds64.wvr[15]);
1222 SET_DBGWCRn(15, (uint64_t)debug_state->uds.ds64.wcr[15], all_ctrls);
1223 OS_FALLTHROUGH;
1224 case 15:
1225 SET_DBGWVRn(14, debug_state->uds.ds64.wvr[14]);
1226 SET_DBGWCRn(14, (uint64_t)debug_state->uds.ds64.wcr[14], all_ctrls);
1227 OS_FALLTHROUGH;
1228 case 14:
1229 SET_DBGWVRn(13, debug_state->uds.ds64.wvr[13]);
1230 SET_DBGWCRn(13, (uint64_t)debug_state->uds.ds64.wcr[13], all_ctrls);
1231 OS_FALLTHROUGH;
1232 case 13:
1233 SET_DBGWVRn(12, debug_state->uds.ds64.wvr[12]);
1234 SET_DBGWCRn(12, (uint64_t)debug_state->uds.ds64.wcr[12], all_ctrls);
1235 OS_FALLTHROUGH;
1236 case 12:
1237 SET_DBGWVRn(11, debug_state->uds.ds64.wvr[11]);
1238 SET_DBGWCRn(11, (uint64_t)debug_state->uds.ds64.wcr[11], all_ctrls);
1239 OS_FALLTHROUGH;
1240 case 11:
1241 SET_DBGWVRn(10, debug_state->uds.ds64.wvr[10]);
1242 SET_DBGWCRn(10, (uint64_t)debug_state->uds.ds64.wcr[10], all_ctrls);
1243 OS_FALLTHROUGH;
1244 case 10:
1245 SET_DBGWVRn(9, debug_state->uds.ds64.wvr[9]);
1246 SET_DBGWCRn(9, (uint64_t)debug_state->uds.ds64.wcr[9], all_ctrls);
1247 OS_FALLTHROUGH;
1248 case 9:
1249 SET_DBGWVRn(8, debug_state->uds.ds64.wvr[8]);
1250 SET_DBGWCRn(8, (uint64_t)debug_state->uds.ds64.wcr[8], all_ctrls);
1251 OS_FALLTHROUGH;
1252 case 8:
1253 SET_DBGWVRn(7, debug_state->uds.ds64.wvr[7]);
1254 SET_DBGWCRn(7, (uint64_t)debug_state->uds.ds64.wcr[7], all_ctrls);
1255 OS_FALLTHROUGH;
1256 case 7:
1257 SET_DBGWVRn(6, debug_state->uds.ds64.wvr[6]);
1258 SET_DBGWCRn(6, (uint64_t)debug_state->uds.ds64.wcr[6], all_ctrls);
1259 OS_FALLTHROUGH;
1260 case 6:
1261 SET_DBGWVRn(5, debug_state->uds.ds64.wvr[5]);
1262 SET_DBGWCRn(5, (uint64_t)debug_state->uds.ds64.wcr[5], all_ctrls);
1263 OS_FALLTHROUGH;
1264 case 5:
1265 SET_DBGWVRn(4, debug_state->uds.ds64.wvr[4]);
1266 SET_DBGWCRn(4, (uint64_t)debug_state->uds.ds64.wcr[4], all_ctrls);
1267 OS_FALLTHROUGH;
1268 case 4:
1269 SET_DBGWVRn(3, debug_state->uds.ds64.wvr[3]);
1270 SET_DBGWCRn(3, (uint64_t)debug_state->uds.ds64.wcr[3], all_ctrls);
1271 OS_FALLTHROUGH;
1272 case 3:
1273 SET_DBGWVRn(2, debug_state->uds.ds64.wvr[2]);
1274 SET_DBGWCRn(2, (uint64_t)debug_state->uds.ds64.wcr[2], all_ctrls);
1275 OS_FALLTHROUGH;
1276 case 2:
1277 SET_DBGWVRn(1, debug_state->uds.ds64.wvr[1]);
1278 SET_DBGWCRn(1, (uint64_t)debug_state->uds.ds64.wcr[1], all_ctrls);
1279 OS_FALLTHROUGH;
1280 case 1:
1281 SET_DBGWVRn(0, debug_state->uds.ds64.wvr[0]);
1282 SET_DBGWCRn(0, (uint64_t)debug_state->uds.ds64.wcr[0], all_ctrls);
1283 OS_FALLTHROUGH;
1284 default:
1285 break;
1286 }
1287
1288 #if defined(CONFIG_KERNEL_INTEGRITY)
1289 if ((all_ctrls & (ARM_DBG_CR_MODE_CONTROL_PRIVILEGED | ARM_DBG_CR_HIGHER_MODE_ENABLE)) != 0) {
1290 panic("sorry, self-hosted debug is not supported: 0x%llx", all_ctrls);
1291 }
1292 #endif
1293
1294 /*
1295 * Breakpoint/Watchpoint Enable
1296 */
1297 if (all_ctrls != 0) {
1298 update_mdscr(0, 0x8000); // MDSCR_EL1[MDE]
1299 } else {
1300 update_mdscr(0x8000, 0);
1301 }
1302
1303 /*
1304 * Software debug single step enable
1305 */
1306 if (debug_state->uds.ds64.mdscr_el1 & 0x1) {
1307
1308 update_mdscr(0, 1); // MDSCR_EL1[SS]
1309
1310 mask_user_saved_state_cpsr(current_thread()->machine.upcb, PSR64_SS, 0);
1311 } else {
1312 update_mdscr(0x1, 0);
1313 }
1314
1315 __builtin_arm_isb(ISB_SY);
1316 (void) ml_set_interrupts_enabled(intr);
1317 }
1318
1319 void
arm_debug_set(arm_debug_state_t * debug_state)1320 arm_debug_set(arm_debug_state_t *debug_state)
1321 {
1322 if (debug_state) {
1323 switch (debug_state->dsh.flavor) {
1324 case ARM_DEBUG_STATE32:
1325 arm_debug_set32(debug_state);
1326 break;
1327 case ARM_DEBUG_STATE64:
1328 arm_debug_set64(debug_state);
1329 break;
1330 default:
1331 panic("arm_debug_set");
1332 break;
1333 }
1334 } else {
1335 if (thread_is_64bit_data(current_thread())) {
1336 arm_debug_set64(debug_state);
1337 } else {
1338 arm_debug_set32(debug_state);
1339 }
1340 }
1341 }
1342
1343 #define VM_MAX_ADDRESS32 ((vm_address_t) 0x80000000)
1344 boolean_t
debug_legacy_state_is_valid(arm_legacy_debug_state_t * debug_state)1345 debug_legacy_state_is_valid(arm_legacy_debug_state_t *debug_state)
1346 {
1347 arm_debug_info_t *debug_info = arm_debug_info();
1348 uint32_t i;
1349 for (i = 0; i < debug_info->num_breakpoint_pairs; i++) {
1350 if (0 != debug_state->bcr[i] && VM_MAX_ADDRESS32 <= debug_state->bvr[i]) {
1351 return FALSE;
1352 }
1353 }
1354
1355 for (i = 0; i < debug_info->num_watchpoint_pairs; i++) {
1356 if (0 != debug_state->wcr[i] && VM_MAX_ADDRESS32 <= debug_state->wvr[i]) {
1357 return FALSE;
1358 }
1359 }
1360 return TRUE;
1361 }
1362
1363 boolean_t
debug_state_is_valid32(arm_debug_state32_t * debug_state)1364 debug_state_is_valid32(arm_debug_state32_t *debug_state)
1365 {
1366 arm_debug_info_t *debug_info = arm_debug_info();
1367 uint32_t i;
1368 for (i = 0; i < debug_info->num_breakpoint_pairs; i++) {
1369 if (0 != debug_state->bcr[i] && VM_MAX_ADDRESS32 <= debug_state->bvr[i]) {
1370 return FALSE;
1371 }
1372 }
1373
1374 for (i = 0; i < debug_info->num_watchpoint_pairs; i++) {
1375 if (0 != debug_state->wcr[i] && VM_MAX_ADDRESS32 <= debug_state->wvr[i]) {
1376 return FALSE;
1377 }
1378 }
1379 return TRUE;
1380 }
1381
1382 boolean_t
debug_state_is_valid64(arm_debug_state64_t * debug_state)1383 debug_state_is_valid64(arm_debug_state64_t *debug_state)
1384 {
1385 arm_debug_info_t *debug_info = arm_debug_info();
1386 uint32_t i;
1387 for (i = 0; i < debug_info->num_breakpoint_pairs; i++) {
1388 if (0 != debug_state->bcr[i] && MACH_VM_MAX_ADDRESS <= debug_state->bvr[i]) {
1389 return FALSE;
1390 }
1391 }
1392
1393 for (i = 0; i < debug_info->num_watchpoint_pairs; i++) {
1394 if (0 != debug_state->wcr[i] && MACH_VM_MAX_ADDRESS <= debug_state->wvr[i]) {
1395 return FALSE;
1396 }
1397 }
1398 return TRUE;
1399 }
1400
1401 /*
1402 * Duplicate one arm_debug_state_t to another. "all" parameter
1403 * is ignored in the case of ARM -- Is this the right assumption?
1404 */
1405 void
copy_legacy_debug_state(arm_legacy_debug_state_t * src,arm_legacy_debug_state_t * target,__unused boolean_t all)1406 copy_legacy_debug_state(arm_legacy_debug_state_t * src,
1407 arm_legacy_debug_state_t * target,
1408 __unused boolean_t all)
1409 {
1410 bcopy(src, target, sizeof(arm_legacy_debug_state_t));
1411 }
1412
1413 void
copy_debug_state32(arm_debug_state32_t * src,arm_debug_state32_t * target,__unused boolean_t all)1414 copy_debug_state32(arm_debug_state32_t * src,
1415 arm_debug_state32_t * target,
1416 __unused boolean_t all)
1417 {
1418 bcopy(src, target, sizeof(arm_debug_state32_t));
1419 }
1420
1421 void
copy_debug_state64(arm_debug_state64_t * src,arm_debug_state64_t * target,__unused boolean_t all)1422 copy_debug_state64(arm_debug_state64_t * src,
1423 arm_debug_state64_t * target,
1424 __unused boolean_t all)
1425 {
1426 bcopy(src, target, sizeof(arm_debug_state64_t));
1427 }
1428
1429 kern_return_t
machine_thread_set_tsd_base(thread_t thread,mach_vm_offset_t tsd_base)1430 machine_thread_set_tsd_base(thread_t thread,
1431 mach_vm_offset_t tsd_base)
1432 {
1433 if (get_threadtask(thread) == kernel_task) {
1434 return KERN_INVALID_ARGUMENT;
1435 }
1436
1437 if (thread_is_64bit_addr(thread)) {
1438 if (tsd_base > vm_map_max(thread->map)) {
1439 tsd_base = 0ULL;
1440 }
1441 } else {
1442 if (tsd_base > UINT32_MAX) {
1443 tsd_base = 0ULL;
1444 }
1445 }
1446
1447 thread->machine.cthread_self = tsd_base;
1448
1449 /* For current thread, make the TSD base active immediately */
1450 if (thread == current_thread()) {
1451 mp_disable_preemption();
1452 set_tpidrro(tsd_base);
1453 mp_enable_preemption();
1454 }
1455
1456 return KERN_SUCCESS;
1457 }
1458
1459 void
machine_tecs(__unused thread_t thr)1460 machine_tecs(__unused thread_t thr)
1461 {
1462 }
1463
1464 int
machine_csv(__unused cpuvn_e cve)1465 machine_csv(__unused cpuvn_e cve)
1466 {
1467 return 0;
1468 }
1469
1470 void
arm_context_switch_requires_sync()1471 arm_context_switch_requires_sync()
1472 {
1473 current_cpu_datap()->sync_on_cswitch = 1;
1474 }
1475
1476 void
arm_context_switch_sync()1477 arm_context_switch_sync()
1478 {
1479 if (__improbable(current_cpu_datap()->sync_on_cswitch != 0)) {
1480 __builtin_arm_isb(ISB_SY);
1481 current_cpu_datap()->sync_on_cswitch = 0;
1482 }
1483 }
1484
1485 #if __has_feature(ptrauth_calls)
1486 boolean_t
arm_user_jop_disabled(void)1487 arm_user_jop_disabled(void)
1488 {
1489 return FALSE;
1490 }
1491 #endif /* __has_feature(ptrauth_calls) */
1492