1 /*
2 * Copyright (c) 2007-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <debug.h>
29 #include <mach/mach_types.h>
30 #include <mach/kern_return.h>
31 #include <mach/thread_status.h>
32 #include <kern/thread.h>
33 #include <kern/kalloc.h>
34 #include <arm/vmparam.h>
35 #include <arm/cpu_data_internal.h>
36 #include <arm/misc_protos.h>
37 #include <arm64/machine_machdep.h>
38 #include <arm64/proc_reg.h>
39 #include <sys/random.h>
40 #if __has_feature(ptrauth_calls)
41 #include <ptrauth.h>
42 #endif
43
44 #include <libkern/coreanalytics/coreanalytics.h>
45
46
47 struct arm_vfpv2_state {
48 __uint32_t __r[32];
49 __uint32_t __fpscr;
50 };
51
52 typedef struct arm_vfpv2_state arm_vfpv2_state_t;
53
54 #define ARM_VFPV2_STATE_COUNT \
55 ((mach_msg_type_number_t)(sizeof (arm_vfpv2_state_t)/sizeof(uint32_t)))
56
57 /*
58 * Forward definitions
59 */
60 void thread_set_child(thread_t child, int pid);
61 void thread_set_parent(thread_t parent, int pid);
62 static void free_debug_state(thread_t thread);
63 user_addr_t thread_get_sigreturn_token(thread_t thread);
64 uint32_t thread_get_sigreturn_diversifier(thread_t thread);
65
66 /*
67 * Maps state flavor to number of words in the state:
68 */
69 /* __private_extern__ */
70 unsigned int _MachineStateCount[] = {
71 [ARM_UNIFIED_THREAD_STATE] = ARM_UNIFIED_THREAD_STATE_COUNT,
72 [ARM_VFP_STATE] = ARM_VFP_STATE_COUNT,
73 [ARM_EXCEPTION_STATE] = ARM_EXCEPTION_STATE_COUNT,
74 [ARM_DEBUG_STATE] = ARM_DEBUG_STATE_COUNT,
75 [ARM_THREAD_STATE64] = ARM_THREAD_STATE64_COUNT,
76 [ARM_EXCEPTION_STATE64] = ARM_EXCEPTION_STATE64_COUNT,
77 [ARM_THREAD_STATE32] = ARM_THREAD_STATE32_COUNT,
78 [ARM_DEBUG_STATE32] = ARM_DEBUG_STATE32_COUNT,
79 [ARM_DEBUG_STATE64] = ARM_DEBUG_STATE64_COUNT,
80 [ARM_NEON_STATE] = ARM_NEON_STATE_COUNT,
81 [ARM_NEON_STATE64] = ARM_NEON_STATE64_COUNT,
82 [ARM_PAGEIN_STATE] = ARM_PAGEIN_STATE_COUNT,
83 };
84
85 extern zone_t ads_zone;
86
87 #if __arm64__
88 /*
89 * Copy values from saved_state to ts64.
90 */
91 void
saved_state_to_thread_state64(const arm_saved_state_t * saved_state,arm_thread_state64_t * ts64)92 saved_state_to_thread_state64(const arm_saved_state_t * saved_state,
93 arm_thread_state64_t * ts64)
94 {
95 uint32_t i;
96
97 assert(is_saved_state64(saved_state));
98
99 ts64->fp = get_saved_state_fp(saved_state);
100 ts64->lr = get_saved_state_lr(saved_state);
101 ts64->sp = get_saved_state_sp(saved_state);
102 ts64->pc = get_saved_state_pc(saved_state);
103 ts64->cpsr = get_saved_state_cpsr(saved_state);
104 for (i = 0; i < 29; i++) {
105 ts64->x[i] = get_saved_state_reg(saved_state, i);
106 }
107 }
108
109 /*
110 * Copy values from ts64 to saved_state.
111 *
112 * For safety, CPSR is sanitized as follows:
113 *
114 * - ts64->cpsr.{N,Z,C,V} are copied as-is into saved_state->cpsr
115 * - ts64->cpsr.M is ignored, and saved_state->cpsr.M is reset to EL0
116 * - All other saved_state->cpsr bits are preserved as-is
117 */
118 void
thread_state64_to_saved_state(const arm_thread_state64_t * ts64,arm_saved_state_t * saved_state)119 thread_state64_to_saved_state(const arm_thread_state64_t * ts64,
120 arm_saved_state_t * saved_state)
121 {
122 uint32_t i;
123 #if __has_feature(ptrauth_calls)
124 uint64_t intr = ml_pac_safe_interrupts_disable();
125 #endif /* __has_feature(ptrauth_calls) */
126
127 assert(is_saved_state64(saved_state));
128
129 const uint32_t CPSR_COPY_MASK = PSR64_USER_MASK;
130 const uint32_t CPSR_ZERO_MASK = PSR64_MODE_MASK;
131 const uint32_t CPSR_PRESERVE_MASK = ~(CPSR_COPY_MASK | CPSR_ZERO_MASK);
132 #if __has_feature(ptrauth_calls)
133 /* BEGIN IGNORE CODESTYLE */
134 MANIPULATE_SIGNED_THREAD_STATE(saved_state,
135 "and w2, w2, %w[preserve_mask]" "\n"
136 "mov w6, %w[cpsr]" "\n"
137 "and w6, w6, %w[copy_mask]" "\n"
138 "orr w2, w2, w6" "\n"
139 "str w2, [x0, %[SS64_CPSR]]" "\n",
140 [cpsr] "r"(ts64->cpsr),
141 [preserve_mask] "i"(CPSR_PRESERVE_MASK),
142 [copy_mask] "i"(CPSR_COPY_MASK)
143 );
144 /* END IGNORE CODESTYLE */
145 /*
146 * Make writes to ts64->cpsr visible first, since it's useful as a
147 * canary to detect thread-state corruption.
148 */
149 __builtin_arm_dmb(DMB_ST);
150 #else
151 uint32_t new_cpsr = get_saved_state_cpsr(saved_state);
152 new_cpsr &= CPSR_PRESERVE_MASK;
153 new_cpsr |= (ts64->cpsr & CPSR_COPY_MASK);
154 set_saved_state_cpsr(saved_state, new_cpsr);
155 #endif /* __has_feature(ptrauth_calls) */
156 set_saved_state_fp(saved_state, ts64->fp);
157 set_saved_state_lr(saved_state, ts64->lr);
158 set_saved_state_sp(saved_state, ts64->sp);
159 set_saved_state_pc(saved_state, ts64->pc);
160 for (i = 0; i < 29; i++) {
161 set_saved_state_reg(saved_state, i, ts64->x[i]);
162 }
163
164 #if __has_feature(ptrauth_calls)
165 ml_pac_safe_interrupts_restore(intr);
166 #endif /* __has_feature(ptrauth_calls) */
167 }
168
169 #endif /* __arm64__ */
170
171 static kern_return_t
handle_get_arm32_thread_state(thread_state_t tstate,mach_msg_type_number_t * count,const arm_saved_state_t * saved_state)172 handle_get_arm32_thread_state(thread_state_t tstate,
173 mach_msg_type_number_t * count,
174 const arm_saved_state_t * saved_state)
175 {
176 if (*count < ARM_THREAD_STATE32_COUNT) {
177 return KERN_INVALID_ARGUMENT;
178 }
179 if (!is_saved_state32(saved_state)) {
180 return KERN_INVALID_ARGUMENT;
181 }
182
183 (void)saved_state_to_thread_state32(saved_state, (arm_thread_state32_t *)tstate);
184 *count = ARM_THREAD_STATE32_COUNT;
185 return KERN_SUCCESS;
186 }
187
188 static kern_return_t
handle_get_arm64_thread_state(thread_state_t tstate,mach_msg_type_number_t * count,const arm_saved_state_t * saved_state)189 handle_get_arm64_thread_state(thread_state_t tstate,
190 mach_msg_type_number_t * count,
191 const arm_saved_state_t * saved_state)
192 {
193 if (*count < ARM_THREAD_STATE64_COUNT) {
194 return KERN_INVALID_ARGUMENT;
195 }
196 if (!is_saved_state64(saved_state)) {
197 return KERN_INVALID_ARGUMENT;
198 }
199
200 (void)saved_state_to_thread_state64(saved_state, (arm_thread_state64_t *)tstate);
201 *count = ARM_THREAD_STATE64_COUNT;
202 return KERN_SUCCESS;
203 }
204
205
206 static kern_return_t
handle_get_arm_thread_state(thread_state_t tstate,mach_msg_type_number_t * count,const arm_saved_state_t * saved_state)207 handle_get_arm_thread_state(thread_state_t tstate,
208 mach_msg_type_number_t * count,
209 const arm_saved_state_t * saved_state)
210 {
211 /* In an arm64 world, this flavor can be used to retrieve the thread
212 * state of a 32-bit or 64-bit thread into a unified structure, but we
213 * need to support legacy clients who are only aware of 32-bit, so
214 * check the count to see what the client is expecting.
215 */
216 if (*count < ARM_UNIFIED_THREAD_STATE_COUNT) {
217 return handle_get_arm32_thread_state(tstate, count, saved_state);
218 }
219
220 arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *) tstate;
221 bzero(unified_state, sizeof(*unified_state));
222 #if __arm64__
223 if (is_saved_state64(saved_state)) {
224 unified_state->ash.flavor = ARM_THREAD_STATE64;
225 unified_state->ash.count = ARM_THREAD_STATE64_COUNT;
226 (void)saved_state_to_thread_state64(saved_state, thread_state64(unified_state));
227 } else
228 #endif
229 {
230 unified_state->ash.flavor = ARM_THREAD_STATE32;
231 unified_state->ash.count = ARM_THREAD_STATE32_COUNT;
232 (void)saved_state_to_thread_state32(saved_state, thread_state32(unified_state));
233 }
234 *count = ARM_UNIFIED_THREAD_STATE_COUNT;
235 return KERN_SUCCESS;
236 }
237
238
239 static kern_return_t
handle_set_arm32_thread_state(const thread_state_t tstate,mach_msg_type_number_t count,arm_saved_state_t * saved_state)240 handle_set_arm32_thread_state(const thread_state_t tstate,
241 mach_msg_type_number_t count,
242 arm_saved_state_t * saved_state)
243 {
244 if (count != ARM_THREAD_STATE32_COUNT) {
245 return KERN_INVALID_ARGUMENT;
246 }
247
248 (void)thread_state32_to_saved_state((const arm_thread_state32_t *)tstate, saved_state);
249 return KERN_SUCCESS;
250 }
251
252 static kern_return_t
handle_set_arm64_thread_state(const thread_state_t tstate,mach_msg_type_number_t count,arm_saved_state_t * saved_state)253 handle_set_arm64_thread_state(const thread_state_t tstate,
254 mach_msg_type_number_t count,
255 arm_saved_state_t * saved_state)
256 {
257 if (count != ARM_THREAD_STATE64_COUNT) {
258 return KERN_INVALID_ARGUMENT;
259 }
260
261 (void)thread_state64_to_saved_state((const arm_thread_state64_t *)tstate, saved_state);
262 return KERN_SUCCESS;
263 }
264
265
266 static kern_return_t
handle_set_arm_thread_state(const thread_state_t tstate,mach_msg_type_number_t count,arm_saved_state_t * saved_state)267 handle_set_arm_thread_state(const thread_state_t tstate,
268 mach_msg_type_number_t count,
269 arm_saved_state_t * saved_state)
270 {
271 /* In an arm64 world, this flavor can be used to set the thread state of a
272 * 32-bit or 64-bit thread from a unified structure, but we need to support
273 * legacy clients who are only aware of 32-bit, so check the count to see
274 * what the client is expecting.
275 */
276 if (count < ARM_UNIFIED_THREAD_STATE_COUNT) {
277 if (!is_saved_state32(saved_state)) {
278 return KERN_INVALID_ARGUMENT;
279 }
280 return handle_set_arm32_thread_state(tstate, count, saved_state);
281 }
282
283 const arm_unified_thread_state_t *unified_state = (const arm_unified_thread_state_t *) tstate;
284 #if __arm64__
285 if (is_thread_state64(unified_state)) {
286 if (!is_saved_state64(saved_state)) {
287 return KERN_INVALID_ARGUMENT;
288 }
289 (void)thread_state64_to_saved_state(const_thread_state64(unified_state), saved_state);
290 } else
291 #endif
292 {
293 if (!is_saved_state32(saved_state)) {
294 return KERN_INVALID_ARGUMENT;
295 }
296 (void)thread_state32_to_saved_state(const_thread_state32(unified_state), saved_state);
297 }
298
299 return KERN_SUCCESS;
300 }
301
302
303 #if __has_feature(ptrauth_calls)
304
305 static inline uint32_t
thread_generate_sigreturn_token(void * ptr,thread_t thread)306 thread_generate_sigreturn_token(
307 void *ptr,
308 thread_t thread)
309 {
310 user64_addr_t token = (user64_addr_t)ptr;
311 token ^= (user64_addr_t)thread_get_sigreturn_token(thread);
312 token = (user64_addr_t)pmap_sign_user_ptr((void*)token,
313 ptrauth_key_process_independent_data, ptrauth_string_discriminator("nonce"),
314 thread->machine.jop_pid);
315 token >>= 32;
316 return (uint32_t)token;
317 }
318 #endif //__has_feature(ptrauth_calls)
319
320 /*
321 * Translate thread state arguments to userspace representation
322 */
323
324 kern_return_t
machine_thread_state_convert_to_user(thread_t thread,thread_flavor_t flavor,thread_state_t tstate,mach_msg_type_number_t * count,thread_set_status_flags_t tssf_flags)325 machine_thread_state_convert_to_user(
326 thread_t thread,
327 thread_flavor_t flavor,
328 thread_state_t tstate,
329 mach_msg_type_number_t *count,
330 thread_set_status_flags_t tssf_flags)
331 {
332 #if __has_feature(ptrauth_calls)
333 arm_thread_state64_t *ts64;
334 bool preserve_flags = !!(tssf_flags & TSSF_PRESERVE_FLAGS);
335 bool stash_sigreturn_token = !!(tssf_flags & TSSF_STASH_SIGRETURN_TOKEN);
336 bool random_div = !!(tssf_flags & TSSF_RANDOM_USER_DIV);
337 bool thread_div = !!(tssf_flags & TSSF_THREAD_USER_DIV);
338 uint32_t old_flags;
339 bool kernel_signed_pc = true;
340 bool kernel_signed_lr = true;
341 uint32_t userland_diversifier = 0;
342
343 switch (flavor) {
344 case ARM_THREAD_STATE:
345 {
346 arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *)tstate;
347
348 if (*count < ARM_UNIFIED_THREAD_STATE_COUNT || !is_thread_state64(unified_state)) {
349 return KERN_SUCCESS;
350 }
351 ts64 = thread_state64(unified_state);
352 break;
353 }
354 case ARM_THREAD_STATE64:
355 {
356 if (*count < ARM_THREAD_STATE64_COUNT) {
357 return KERN_SUCCESS;
358 }
359 ts64 = (arm_thread_state64_t *)tstate;
360 break;
361 }
362 default:
363 return KERN_SUCCESS;
364 }
365
366 // Note that kernel threads never have disable_user_jop set
367 if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
368 !thread_is_64bit_addr(current_thread()) ||
369 (thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) || !thread_is_64bit_addr(thread)
370 ) {
371 ts64->flags = __DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH;
372 return KERN_SUCCESS;
373 }
374
375 old_flags = ts64->flags;
376 ts64->flags = 0;
377 if (ts64->lr) {
378 // lr might contain an IB-signed return address (strip is a no-op on unsigned addresses)
379 uintptr_t stripped_lr = (uintptr_t)ptrauth_strip((void *)ts64->lr,
380 ptrauth_key_return_address);
381 if (ts64->lr != stripped_lr) {
382 // Need to allow already-signed lr value to round-trip as is
383 ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
384 }
385 // Note that an IB-signed return address that happens to have a 0 signature value
386 // will round-trip correctly even if IA-signed again below (and IA-authd later)
387 }
388
389 if (arm_user_jop_disabled()) {
390 return KERN_SUCCESS;
391 }
392
393 if (preserve_flags) {
394 assert(random_div == false);
395 assert(thread_div == false);
396
397 /* Restore the diversifier and other opaque flags */
398 ts64->flags |= (old_flags & __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
399 userland_diversifier = old_flags & __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK;
400 if (!(old_flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC)) {
401 kernel_signed_pc = false;
402 }
403 if (!(old_flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR)) {
404 kernel_signed_lr = false;
405 }
406 } else {
407 /* Set a non zero userland diversifier */
408 if (random_div) {
409 do {
410 read_random(&userland_diversifier, sizeof(userland_diversifier));
411 userland_diversifier &=
412 __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK;
413 } while (userland_diversifier == 0);
414 } else if (thread_div) {
415 userland_diversifier = thread_get_sigreturn_diversifier(thread) &
416 __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK;
417 }
418 ts64->flags |= userland_diversifier;
419 }
420
421 if (kernel_signed_pc) {
422 ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC;
423 }
424
425 if (kernel_signed_lr) {
426 ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR;
427 }
428
429
430 if (ts64->pc) {
431 uint64_t discriminator = ptrauth_string_discriminator("pc");
432 if (!kernel_signed_pc && userland_diversifier != 0) {
433 discriminator = ptrauth_blend_discriminator((void *)(long)userland_diversifier,
434 ptrauth_string_discriminator("pc"));
435 }
436
437 ts64->pc = (uintptr_t)pmap_sign_user_ptr((void*)ts64->pc,
438 ptrauth_key_process_independent_code, discriminator,
439 thread->machine.jop_pid);
440 }
441 if (ts64->lr && !(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR)) {
442 uint64_t discriminator = ptrauth_string_discriminator("lr");
443 if (!kernel_signed_lr && userland_diversifier != 0) {
444 discriminator = ptrauth_blend_discriminator((void *)(long)userland_diversifier,
445 ptrauth_string_discriminator("lr"));
446 }
447
448 ts64->lr = (uintptr_t)pmap_sign_user_ptr((void*)ts64->lr,
449 ptrauth_key_process_independent_code, discriminator,
450 thread->machine.jop_pid);
451 }
452 if (ts64->sp) {
453 ts64->sp = (uintptr_t)pmap_sign_user_ptr((void*)ts64->sp,
454 ptrauth_key_process_independent_data, ptrauth_string_discriminator("sp"),
455 thread->machine.jop_pid);
456 }
457 if (ts64->fp) {
458 ts64->fp = (uintptr_t)pmap_sign_user_ptr((void*)ts64->fp,
459 ptrauth_key_process_independent_data, ptrauth_string_discriminator("fp"),
460 thread->machine.jop_pid);
461 }
462
463 /* Stash the sigreturn token */
464 if (stash_sigreturn_token) {
465 if (kernel_signed_pc) {
466 uint32_t token = thread_generate_sigreturn_token((void *)ts64->pc, thread);
467 __DARWIN_ARM_THREAD_STATE64_SET_SIGRETURN_TOKEN(ts64, token,
468 __DARWIN_ARM_THREAD_STATE64_SIGRETURN_PC_MASK);
469 }
470
471 if (kernel_signed_lr) {
472 uint32_t token = thread_generate_sigreturn_token((void *)ts64->lr, thread);
473 __DARWIN_ARM_THREAD_STATE64_SET_SIGRETURN_TOKEN(ts64, token,
474 __DARWIN_ARM_THREAD_STATE64_SIGRETURN_LR_MASK);
475 }
476 }
477
478 return KERN_SUCCESS;
479 #else
480 // No conversion to userspace representation on this platform
481 (void)thread; (void)flavor; (void)tstate; (void)count; (void)tssf_flags;
482 return KERN_SUCCESS;
483 #endif /* __has_feature(ptrauth_calls) */
484 }
485
486 #if __has_feature(ptrauth_calls)
487 extern char * proc_name_address(void *p);
488
489 CA_EVENT(pac_thread_state_exception_event,
490 CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name);
491
492 static void
machine_thread_state_check_pac_state(arm_thread_state64_t * ts64,arm_thread_state64_t * old_ts64)493 machine_thread_state_check_pac_state(
494 arm_thread_state64_t *ts64,
495 arm_thread_state64_t *old_ts64)
496 {
497 bool send_event = false;
498 task_t task = current_task();
499 void *proc = get_bsdtask_info(task);
500 char *proc_name = (char *) "unknown";
501
502 if (((ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC) &&
503 ts64->pc != old_ts64->pc) || (!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) &&
504 (ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR) && (ts64->lr != old_ts64->lr ||
505 (old_ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR)))) {
506 send_event = true;
507 }
508
509 if (!send_event) {
510 return;
511 }
512
513 proc_name = proc_name_address(proc);
514 ca_event_t ca_event = CA_EVENT_ALLOCATE(pac_thread_state_exception_event);
515 CA_EVENT_TYPE(pac_thread_state_exception_event) * pexc_event = ca_event->data;
516 strlcpy(pexc_event->proc_name, proc_name, CA_PROCNAME_LEN);
517 CA_EVENT_SEND(ca_event);
518 }
519
520 CA_EVENT(pac_thread_state_sigreturn_event,
521 CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name);
522
523 static bool
machine_thread_state_check_sigreturn_token(arm_thread_state64_t * ts64,thread_t thread)524 machine_thread_state_check_sigreturn_token(
525 arm_thread_state64_t *ts64,
526 thread_t thread)
527 {
528 task_t task = current_task();
529 void *proc = get_bsdtask_info(task);
530 char *proc_name = (char *) "unknown";
531 bool token_matched = true;
532 bool kernel_signed_pc = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC);
533 bool kernel_signed_lr = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR);
534
535 if (kernel_signed_pc) {
536 /* Compute the sigreturn token */
537 uint32_t token = thread_generate_sigreturn_token((void *)ts64->pc, thread);
538 if (!__DARWIN_ARM_THREAD_STATE64_CHECK_SIGRETURN_TOKEN(ts64, token,
539 __DARWIN_ARM_THREAD_STATE64_SIGRETURN_PC_MASK)) {
540 token_matched = false;
541 }
542 }
543
544 if (kernel_signed_lr) {
545 /* Compute the sigreturn token */
546 uint32_t token = thread_generate_sigreturn_token((void *)ts64->lr, thread);
547 if (!__DARWIN_ARM_THREAD_STATE64_CHECK_SIGRETURN_TOKEN(ts64, token,
548 __DARWIN_ARM_THREAD_STATE64_SIGRETURN_LR_MASK)) {
549 token_matched = false;
550 }
551 }
552
553 if (token_matched) {
554 return true;
555 }
556
557 proc_name = proc_name_address(proc);
558 ca_event_t ca_event = CA_EVENT_ALLOCATE(pac_thread_state_sigreturn_event);
559 CA_EVENT_TYPE(pac_thread_state_sigreturn_event) * psig_event = ca_event->data;
560 strlcpy(psig_event->proc_name, proc_name, CA_PROCNAME_LEN);
561 CA_EVENT_SEND(ca_event);
562 return false;
563 }
564
565 #endif
566
567 /*
568 * Translate thread state arguments from userspace representation
569 */
570
571 kern_return_t
machine_thread_state_convert_from_user(thread_t thread,thread_flavor_t flavor,thread_state_t tstate,mach_msg_type_number_t count,thread_state_t old_tstate,mach_msg_type_number_t old_count,thread_set_status_flags_t tssf_flags)572 machine_thread_state_convert_from_user(
573 thread_t thread,
574 thread_flavor_t flavor,
575 thread_state_t tstate,
576 mach_msg_type_number_t count,
577 thread_state_t old_tstate,
578 mach_msg_type_number_t old_count,
579 thread_set_status_flags_t tssf_flags)
580 {
581 #if __has_feature(ptrauth_calls)
582 arm_thread_state64_t *ts64;
583 arm_thread_state64_t *old_ts64 = NULL;
584 void *userland_diversifier = NULL;
585 bool kernel_signed_pc;
586 bool kernel_signed_lr;
587 bool random_div = !!(tssf_flags & TSSF_RANDOM_USER_DIV);
588 bool thread_div = !!(tssf_flags & TSSF_THREAD_USER_DIV);
589
590 switch (flavor) {
591 case ARM_THREAD_STATE:
592 {
593 arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *)tstate;
594
595 if (count < ARM_UNIFIED_THREAD_STATE_COUNT || !is_thread_state64(unified_state)) {
596 return KERN_SUCCESS;
597 }
598 ts64 = thread_state64(unified_state);
599
600 arm_unified_thread_state_t *old_unified_state = (arm_unified_thread_state_t *)old_tstate;
601 if (old_unified_state && old_count >= ARM_UNIFIED_THREAD_STATE_COUNT) {
602 old_ts64 = thread_state64(old_unified_state);
603 }
604 break;
605 }
606 case ARM_THREAD_STATE64:
607 {
608 if (count != ARM_THREAD_STATE64_COUNT) {
609 return KERN_SUCCESS;
610 }
611 ts64 = (arm_thread_state64_t *)tstate;
612
613 if (old_count == ARM_THREAD_STATE64_COUNT) {
614 old_ts64 = (arm_thread_state64_t *)old_tstate;
615 }
616 break;
617 }
618 default:
619 return KERN_SUCCESS;
620 }
621
622 // Note that kernel threads never have disable_user_jop set
623 if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
624 !thread_is_64bit_addr(current_thread())) {
625 if ((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
626 !thread_is_64bit_addr(thread)) {
627 ts64->flags = __DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH;
628 return KERN_SUCCESS;
629 }
630 // A JOP-disabled process must not set thread state on a JOP-enabled process
631 return KERN_PROTECTION_FAILURE;
632 }
633
634 if (ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH) {
635 if ((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
636 !thread_is_64bit_addr(thread)
637 ) {
638 return KERN_SUCCESS;
639 }
640 // Disallow setting unsigned thread state on JOP-enabled processes.
641 // Ignore flag and treat thread state arguments as signed, ptrauth
642 // poisoning will cause resulting thread state to be invalid
643 ts64->flags &= ~__DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH;
644 }
645
646 if (ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) {
647 // lr might contain an IB-signed return address (strip is a no-op on unsigned addresses)
648 uintptr_t stripped_lr = (uintptr_t)ptrauth_strip((void *)ts64->lr,
649 ptrauth_key_return_address);
650 if (ts64->lr == stripped_lr) {
651 // Don't allow unsigned pointer to be passed through as is. Ignore flag and
652 // treat as IA-signed below (where auth failure may poison the value).
653 ts64->flags &= ~__DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
654 }
655 // Note that an IB-signed return address that happens to have a 0 signature value
656 // will also have been IA-signed (without this flag being set) and so will IA-auth
657 // correctly below.
658 }
659
660 if (arm_user_jop_disabled()) {
661 return KERN_SUCCESS;
662 }
663
664 kernel_signed_pc = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC);
665 kernel_signed_lr = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR);
666 /*
667 * Replace pc/lr with old state if allow only
668 * user ptr flag is passed and ptrs are marked
669 * kernel signed.
670 */
671 if ((tssf_flags & TSSF_CHECK_USER_FLAGS) &&
672 (kernel_signed_pc || kernel_signed_lr)) {
673 if (old_ts64 && old_count == count) {
674 /* Send a CA event if the thread state does not match */
675 machine_thread_state_check_pac_state(ts64, old_ts64);
676
677 /* Check if user ptrs needs to be replaced */
678 if ((tssf_flags & TSSF_ALLOW_ONLY_USER_PTRS) &&
679 kernel_signed_pc) {
680 ts64->pc = old_ts64->pc;
681 }
682
683 if ((tssf_flags & TSSF_ALLOW_ONLY_USER_PTRS) &&
684 !(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) &&
685 kernel_signed_lr) {
686 ts64->lr = old_ts64->lr;
687 if (old_ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) {
688 ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
689 } else {
690 ts64->flags &= ~__DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
691 }
692 }
693 }
694 }
695
696 /* Validate sigreturn token */
697 if (tssf_flags & TSSF_CHECK_SIGRETURN_TOKEN) {
698 bool token_matched = machine_thread_state_check_sigreturn_token(ts64, thread);
699 if ((tssf_flags & TSSF_ALLOW_ONLY_MATCHING_TOKEN) && !token_matched) {
700 return KERN_PROTECTION_FAILURE;
701 }
702 }
703
704 /* Get the userland diversifier */
705 if (random_div && old_ts64 && old_count == count) {
706 /* Get the random diversifier from the old thread state */
707 userland_diversifier = (void *)(long)(old_ts64->flags &
708 __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
709 } else if (thread_div) {
710 userland_diversifier = (void *)(long)(thread_get_sigreturn_diversifier(thread) &
711 __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
712 }
713
714 if (ts64->pc) {
715 uint64_t discriminator = ptrauth_string_discriminator("pc");
716 if (!kernel_signed_pc && userland_diversifier != 0) {
717 discriminator = ptrauth_blend_discriminator(userland_diversifier,
718 ptrauth_string_discriminator("pc"));
719 }
720 ts64->pc = (uintptr_t)pmap_auth_user_ptr((void*)ts64->pc,
721 ptrauth_key_process_independent_code, discriminator,
722 thread->machine.jop_pid);
723 }
724 if (ts64->lr && !(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR)) {
725 uint64_t discriminator = ptrauth_string_discriminator("lr");
726 if (!kernel_signed_lr && userland_diversifier != 0) {
727 discriminator = ptrauth_blend_discriminator(userland_diversifier,
728 ptrauth_string_discriminator("lr"));
729 }
730 ts64->lr = (uintptr_t)pmap_auth_user_ptr((void*)ts64->lr,
731 ptrauth_key_process_independent_code, discriminator,
732 thread->machine.jop_pid);
733 }
734 if (ts64->sp) {
735 ts64->sp = (uintptr_t)pmap_auth_user_ptr((void*)ts64->sp,
736 ptrauth_key_process_independent_data, ptrauth_string_discriminator("sp"),
737 thread->machine.jop_pid);
738 }
739 if (ts64->fp) {
740 ts64->fp = (uintptr_t)pmap_auth_user_ptr((void*)ts64->fp,
741 ptrauth_key_process_independent_data, ptrauth_string_discriminator("fp"),
742 thread->machine.jop_pid);
743 }
744
745 return KERN_SUCCESS;
746 #else
747 // No conversion from userspace representation on this platform
748 (void)thread; (void)flavor; (void)tstate; (void)count;
749 (void)old_tstate; (void)old_count; (void)tssf_flags;
750 return KERN_SUCCESS;
751 #endif /* __has_feature(ptrauth_calls) */
752 }
753
754 /*
755 * Translate signal context data pointer to userspace representation
756 */
757
758 kern_return_t
machine_thread_siguctx_pointer_convert_to_user(thread_t thread,user_addr_t * uctxp)759 machine_thread_siguctx_pointer_convert_to_user(
760 thread_t thread,
761 user_addr_t *uctxp)
762 {
763 #if __has_feature(ptrauth_calls)
764 if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
765 !thread_is_64bit_addr(current_thread())) {
766 assert((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) || !thread_is_64bit_addr(thread));
767 return KERN_SUCCESS;
768 }
769
770 if (arm_user_jop_disabled()) {
771 return KERN_SUCCESS;
772 }
773
774 if (*uctxp) {
775 *uctxp = (uintptr_t)pmap_sign_user_ptr((void*)*uctxp,
776 ptrauth_key_process_independent_data, ptrauth_string_discriminator("uctx"),
777 thread->machine.jop_pid);
778 }
779
780 return KERN_SUCCESS;
781 #else
782 // No conversion to userspace representation on this platform
783 (void)thread; (void)uctxp;
784 return KERN_SUCCESS;
785 #endif /* __has_feature(ptrauth_calls) */
786 }
787
788 /*
789 * Translate array of function pointer syscall arguments from userspace representation
790 */
791
792 kern_return_t
machine_thread_function_pointers_convert_from_user(thread_t thread,user_addr_t * fptrs,uint32_t count)793 machine_thread_function_pointers_convert_from_user(
794 thread_t thread,
795 user_addr_t *fptrs,
796 uint32_t count)
797 {
798 #if __has_feature(ptrauth_calls)
799 if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
800 !thread_is_64bit_addr(current_thread())) {
801 assert((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
802 !thread_is_64bit_addr(thread));
803 return KERN_SUCCESS;
804 }
805
806 if (arm_user_jop_disabled()) {
807 return KERN_SUCCESS;
808 }
809
810 while (count--) {
811 if (*fptrs) {
812 *fptrs = (uintptr_t)pmap_auth_user_ptr((void*)*fptrs,
813 ptrauth_key_function_pointer, 0, thread->machine.jop_pid);
814 }
815 fptrs++;
816 }
817
818 return KERN_SUCCESS;
819 #else
820 // No conversion from userspace representation on this platform
821 (void)thread; (void)fptrs; (void)count;
822 return KERN_SUCCESS;
823 #endif /* __has_feature(ptrauth_calls) */
824 }
825
826 /*
827 * Routine: machine_thread_get_state
828 *
829 */
830 kern_return_t
machine_thread_get_state(thread_t thread,thread_flavor_t flavor,thread_state_t tstate,mach_msg_type_number_t * count)831 machine_thread_get_state(thread_t thread,
832 thread_flavor_t flavor,
833 thread_state_t tstate,
834 mach_msg_type_number_t * count)
835 {
836 switch (flavor) {
837 case THREAD_STATE_FLAVOR_LIST:
838 if (*count < 4) {
839 return KERN_INVALID_ARGUMENT;
840 }
841
842 tstate[0] = ARM_THREAD_STATE;
843 tstate[1] = ARM_VFP_STATE;
844 tstate[2] = ARM_EXCEPTION_STATE;
845 tstate[3] = ARM_DEBUG_STATE;
846 *count = 4;
847 break;
848
849 case THREAD_STATE_FLAVOR_LIST_NEW:
850 if (*count < 4) {
851 return KERN_INVALID_ARGUMENT;
852 }
853
854 tstate[0] = ARM_THREAD_STATE;
855 tstate[1] = ARM_VFP_STATE;
856 tstate[2] = thread_is_64bit_data(thread) ? ARM_EXCEPTION_STATE64 : ARM_EXCEPTION_STATE;
857 tstate[3] = thread_is_64bit_data(thread) ? ARM_DEBUG_STATE64 : ARM_DEBUG_STATE32;
858 *count = 4;
859 break;
860
861 case THREAD_STATE_FLAVOR_LIST_10_15:
862 if (*count < 5) {
863 return KERN_INVALID_ARGUMENT;
864 }
865
866 tstate[0] = ARM_THREAD_STATE;
867 tstate[1] = ARM_VFP_STATE;
868 tstate[2] = thread_is_64bit_data(thread) ? ARM_EXCEPTION_STATE64 : ARM_EXCEPTION_STATE;
869 tstate[3] = thread_is_64bit_data(thread) ? ARM_DEBUG_STATE64 : ARM_DEBUG_STATE32;
870 tstate[4] = ARM_PAGEIN_STATE;
871 *count = 5;
872 break;
873
874 case ARM_THREAD_STATE:
875 {
876 kern_return_t rn = handle_get_arm_thread_state(tstate, count, thread->machine.upcb);
877 if (rn) {
878 return rn;
879 }
880 break;
881 }
882 case ARM_THREAD_STATE32:
883 {
884 if (thread_is_64bit_data(thread)) {
885 return KERN_INVALID_ARGUMENT;
886 }
887
888 kern_return_t rn = handle_get_arm32_thread_state(tstate, count, thread->machine.upcb);
889 if (rn) {
890 return rn;
891 }
892 break;
893 }
894 #if __arm64__
895 case ARM_THREAD_STATE64:
896 {
897 if (!thread_is_64bit_data(thread)) {
898 return KERN_INVALID_ARGUMENT;
899 }
900
901 const arm_saved_state_t *current_state = thread->machine.upcb;
902
903 kern_return_t rn = handle_get_arm64_thread_state(tstate, count,
904 current_state);
905 if (rn) {
906 return rn;
907 }
908
909 break;
910 }
911 #endif
912 case ARM_EXCEPTION_STATE:{
913 struct arm_exception_state *state;
914 struct arm_saved_state32 *saved_state;
915
916 if (*count < ARM_EXCEPTION_STATE_COUNT) {
917 return KERN_INVALID_ARGUMENT;
918 }
919 if (thread_is_64bit_data(thread)) {
920 return KERN_INVALID_ARGUMENT;
921 }
922
923 state = (struct arm_exception_state *) tstate;
924 saved_state = saved_state32(thread->machine.upcb);
925
926 state->exception = saved_state->exception;
927 state->fsr = saved_state->esr;
928 state->far = saved_state->far;
929
930 *count = ARM_EXCEPTION_STATE_COUNT;
931 break;
932 }
933 case ARM_EXCEPTION_STATE64:{
934 struct arm_exception_state64 *state;
935 struct arm_saved_state64 *saved_state;
936
937 if (*count < ARM_EXCEPTION_STATE64_COUNT) {
938 return KERN_INVALID_ARGUMENT;
939 }
940 if (!thread_is_64bit_data(thread)) {
941 return KERN_INVALID_ARGUMENT;
942 }
943
944 state = (struct arm_exception_state64 *) tstate;
945 saved_state = saved_state64(thread->machine.upcb);
946
947 state->exception = saved_state->exception;
948 state->far = saved_state->far;
949 state->esr = saved_state->esr;
950
951 *count = ARM_EXCEPTION_STATE64_COUNT;
952 break;
953 }
954 case ARM_DEBUG_STATE:{
955 arm_legacy_debug_state_t *state;
956 arm_debug_state32_t *thread_state;
957
958 if (*count < ARM_LEGACY_DEBUG_STATE_COUNT) {
959 return KERN_INVALID_ARGUMENT;
960 }
961
962 if (thread_is_64bit_data(thread)) {
963 return KERN_INVALID_ARGUMENT;
964 }
965
966 state = (arm_legacy_debug_state_t *) tstate;
967 thread_state = find_debug_state32(thread);
968
969 if (thread_state == NULL) {
970 bzero(state, sizeof(arm_legacy_debug_state_t));
971 } else {
972 bcopy(thread_state, state, sizeof(arm_legacy_debug_state_t));
973 }
974
975 *count = ARM_LEGACY_DEBUG_STATE_COUNT;
976 break;
977 }
978 case ARM_DEBUG_STATE32:{
979 arm_debug_state32_t *state;
980 arm_debug_state32_t *thread_state;
981
982 if (*count < ARM_DEBUG_STATE32_COUNT) {
983 return KERN_INVALID_ARGUMENT;
984 }
985
986 if (thread_is_64bit_data(thread)) {
987 return KERN_INVALID_ARGUMENT;
988 }
989
990 state = (arm_debug_state32_t *) tstate;
991 thread_state = find_debug_state32(thread);
992
993 if (thread_state == NULL) {
994 bzero(state, sizeof(arm_debug_state32_t));
995 } else {
996 bcopy(thread_state, state, sizeof(arm_debug_state32_t));
997 }
998
999 *count = ARM_DEBUG_STATE32_COUNT;
1000 break;
1001 }
1002
1003 case ARM_DEBUG_STATE64:{
1004 arm_debug_state64_t *state;
1005 arm_debug_state64_t *thread_state;
1006
1007 if (*count < ARM_DEBUG_STATE64_COUNT) {
1008 return KERN_INVALID_ARGUMENT;
1009 }
1010
1011 if (!thread_is_64bit_data(thread)) {
1012 return KERN_INVALID_ARGUMENT;
1013 }
1014
1015 state = (arm_debug_state64_t *) tstate;
1016 thread_state = find_debug_state64(thread);
1017
1018 if (thread_state == NULL) {
1019 bzero(state, sizeof(arm_debug_state64_t));
1020 } else {
1021 bcopy(thread_state, state, sizeof(arm_debug_state64_t));
1022 }
1023
1024 *count = ARM_DEBUG_STATE64_COUNT;
1025 break;
1026 }
1027
1028 case ARM_VFP_STATE:{
1029 struct arm_vfp_state *state;
1030 arm_neon_saved_state32_t *thread_state;
1031 unsigned int max;
1032
1033 if (*count < ARM_VFP_STATE_COUNT) {
1034 if (*count < ARM_VFPV2_STATE_COUNT) {
1035 return KERN_INVALID_ARGUMENT;
1036 } else {
1037 *count = ARM_VFPV2_STATE_COUNT;
1038 }
1039 }
1040
1041 if (*count == ARM_VFPV2_STATE_COUNT) {
1042 max = 32;
1043 } else {
1044 max = 64;
1045 }
1046
1047 state = (struct arm_vfp_state *) tstate;
1048 thread_state = neon_state32(thread->machine.uNeon);
1049 /* ARM64 TODO: set fpsr and fpcr from state->fpscr */
1050
1051 bcopy(thread_state, state, (max + 1) * sizeof(uint32_t));
1052 *count = (max + 1);
1053 break;
1054 }
1055 case ARM_NEON_STATE:{
1056 arm_neon_state_t *state;
1057 arm_neon_saved_state32_t *thread_state;
1058
1059 if (*count < ARM_NEON_STATE_COUNT) {
1060 return KERN_INVALID_ARGUMENT;
1061 }
1062
1063 if (thread_is_64bit_data(thread)) {
1064 return KERN_INVALID_ARGUMENT;
1065 }
1066
1067 state = (arm_neon_state_t *)tstate;
1068 thread_state = neon_state32(thread->machine.uNeon);
1069
1070 assert(sizeof(*thread_state) == sizeof(*state));
1071 bcopy(thread_state, state, sizeof(arm_neon_state_t));
1072
1073 *count = ARM_NEON_STATE_COUNT;
1074 break;
1075 }
1076
1077 case ARM_NEON_STATE64:{
1078 arm_neon_state64_t *state;
1079 arm_neon_saved_state64_t *thread_state;
1080
1081 if (*count < ARM_NEON_STATE64_COUNT) {
1082 return KERN_INVALID_ARGUMENT;
1083 }
1084
1085 if (!thread_is_64bit_data(thread)) {
1086 return KERN_INVALID_ARGUMENT;
1087 }
1088
1089 state = (arm_neon_state64_t *)tstate;
1090 thread_state = neon_state64(thread->machine.uNeon);
1091
1092 /* For now, these are identical */
1093 assert(sizeof(*state) == sizeof(*thread_state));
1094 bcopy(thread_state, state, sizeof(arm_neon_state64_t));
1095
1096
1097 *count = ARM_NEON_STATE64_COUNT;
1098 break;
1099 }
1100
1101
1102 case ARM_PAGEIN_STATE: {
1103 arm_pagein_state_t *state;
1104
1105 if (*count < ARM_PAGEIN_STATE_COUNT) {
1106 return KERN_INVALID_ARGUMENT;
1107 }
1108
1109 state = (arm_pagein_state_t *)tstate;
1110 state->__pagein_error = thread->t_pagein_error;
1111
1112 *count = ARM_PAGEIN_STATE_COUNT;
1113 break;
1114 }
1115
1116
1117 default:
1118 return KERN_INVALID_ARGUMENT;
1119 }
1120 return KERN_SUCCESS;
1121 }
1122
1123
1124 /*
1125 * Routine: machine_thread_get_kern_state
1126 *
1127 */
1128 kern_return_t
machine_thread_get_kern_state(thread_t thread,thread_flavor_t flavor,thread_state_t tstate,mach_msg_type_number_t * count)1129 machine_thread_get_kern_state(thread_t thread,
1130 thread_flavor_t flavor,
1131 thread_state_t tstate,
1132 mach_msg_type_number_t * count)
1133 {
1134 /*
1135 * This works only for an interrupted kernel thread
1136 */
1137 if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) {
1138 return KERN_FAILURE;
1139 }
1140
1141 switch (flavor) {
1142 case ARM_THREAD_STATE:
1143 {
1144 kern_return_t rn = handle_get_arm_thread_state(tstate, count, getCpuDatap()->cpu_int_state);
1145 if (rn) {
1146 return rn;
1147 }
1148 break;
1149 }
1150 case ARM_THREAD_STATE32:
1151 {
1152 kern_return_t rn = handle_get_arm32_thread_state(tstate, count, getCpuDatap()->cpu_int_state);
1153 if (rn) {
1154 return rn;
1155 }
1156 break;
1157 }
1158 #if __arm64__
1159 case ARM_THREAD_STATE64:
1160 {
1161 kern_return_t rn = handle_get_arm64_thread_state(tstate, count, getCpuDatap()->cpu_int_state);
1162 if (rn) {
1163 return rn;
1164 }
1165 break;
1166 }
1167 #endif
1168 default:
1169 return KERN_INVALID_ARGUMENT;
1170 }
1171 return KERN_SUCCESS;
1172 }
1173
1174 void
machine_thread_switch_addrmode(thread_t thread)1175 machine_thread_switch_addrmode(thread_t thread)
1176 {
1177 if (task_has_64Bit_data(get_threadtask(thread))) {
1178 thread->machine.upcb->ash.flavor = ARM_SAVED_STATE64;
1179 thread->machine.upcb->ash.count = ARM_SAVED_STATE64_COUNT;
1180 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64;
1181 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT;
1182
1183 /*
1184 * Reinitialize the NEON state.
1185 */
1186 bzero(&thread->machine.uNeon->uns, sizeof(thread->machine.uNeon->uns));
1187 thread->machine.uNeon->ns_64.fpcr = FPCR_DEFAULT;
1188 } else {
1189 thread->machine.upcb->ash.flavor = ARM_SAVED_STATE32;
1190 thread->machine.upcb->ash.count = ARM_SAVED_STATE32_COUNT;
1191 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
1192 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
1193
1194 /*
1195 * Reinitialize the NEON state.
1196 */
1197 bzero(&thread->machine.uNeon->uns, sizeof(thread->machine.uNeon->uns));
1198 thread->machine.uNeon->ns_32.fpcr = FPCR_DEFAULT_32;
1199 }
1200 }
1201
1202 extern long long arm_debug_get(void);
1203
1204 /*
1205 * Routine: machine_thread_set_state
1206 *
1207 */
1208 kern_return_t
machine_thread_set_state(thread_t thread,thread_flavor_t flavor,thread_state_t tstate,mach_msg_type_number_t count)1209 machine_thread_set_state(thread_t thread,
1210 thread_flavor_t flavor,
1211 thread_state_t tstate,
1212 mach_msg_type_number_t count)
1213 {
1214 kern_return_t rn;
1215
1216 switch (flavor) {
1217 case ARM_THREAD_STATE:
1218 rn = handle_set_arm_thread_state(tstate, count, thread->machine.upcb);
1219 if (rn) {
1220 return rn;
1221 }
1222 break;
1223
1224 case ARM_THREAD_STATE32:
1225 if (thread_is_64bit_data(thread)) {
1226 return KERN_INVALID_ARGUMENT;
1227 }
1228
1229 rn = handle_set_arm32_thread_state(tstate, count, thread->machine.upcb);
1230 if (rn) {
1231 return rn;
1232 }
1233 break;
1234
1235 #if __arm64__
1236 case ARM_THREAD_STATE64:
1237 if (!thread_is_64bit_data(thread)) {
1238 return KERN_INVALID_ARGUMENT;
1239 }
1240
1241
1242 rn = handle_set_arm64_thread_state(tstate, count, thread->machine.upcb);
1243 if (rn) {
1244 return rn;
1245 }
1246 break;
1247 #endif
1248 case ARM_EXCEPTION_STATE:{
1249 if (count != ARM_EXCEPTION_STATE_COUNT) {
1250 return KERN_INVALID_ARGUMENT;
1251 }
1252 if (thread_is_64bit_data(thread)) {
1253 return KERN_INVALID_ARGUMENT;
1254 }
1255
1256 break;
1257 }
1258 case ARM_EXCEPTION_STATE64:{
1259 if (count != ARM_EXCEPTION_STATE64_COUNT) {
1260 return KERN_INVALID_ARGUMENT;
1261 }
1262 if (!thread_is_64bit_data(thread)) {
1263 return KERN_INVALID_ARGUMENT;
1264 }
1265
1266 break;
1267 }
1268 case ARM_DEBUG_STATE:
1269 {
1270 arm_legacy_debug_state_t *state;
1271 boolean_t enabled = FALSE;
1272 unsigned int i;
1273
1274 if (count != ARM_LEGACY_DEBUG_STATE_COUNT) {
1275 return KERN_INVALID_ARGUMENT;
1276 }
1277 if (thread_is_64bit_data(thread)) {
1278 return KERN_INVALID_ARGUMENT;
1279 }
1280
1281 state = (arm_legacy_debug_state_t *) tstate;
1282
1283 for (i = 0; i < 16; i++) {
1284 /* do not allow context IDs to be set */
1285 if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
1286 || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)
1287 || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
1288 || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) {
1289 return KERN_PROTECTION_FAILURE;
1290 }
1291 if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE))
1292 || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) {
1293 enabled = TRUE;
1294 }
1295 }
1296
1297 if (!enabled) {
1298 free_debug_state(thread);
1299 } else {
1300 arm_debug_state32_t *thread_state = find_or_allocate_debug_state32(thread);
1301
1302 if (thread_state == NULL) {
1303 return KERN_FAILURE;
1304 }
1305
1306 for (i = 0; i < 16; i++) {
1307 /* set appropriate privilege; mask out unknown bits */
1308 thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
1309 | ARM_DBGBCR_MATCH_MASK
1310 | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK
1311 | ARM_DBG_CR_ENABLE_MASK))
1312 | ARM_DBGBCR_TYPE_IVA
1313 | ARM_DBG_CR_LINKED_UNLINKED
1314 | ARM_DBG_CR_SECURITY_STATE_BOTH
1315 | ARM_DBG_CR_MODE_CONTROL_USER;
1316 thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK;
1317 thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
1318 | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK
1319 | ARM_DBGWCR_ACCESS_CONTROL_MASK
1320 | ARM_DBG_CR_ENABLE_MASK))
1321 | ARM_DBG_CR_LINKED_UNLINKED
1322 | ARM_DBG_CR_SECURITY_STATE_BOTH
1323 | ARM_DBG_CR_MODE_CONTROL_USER;
1324 thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK;
1325 }
1326
1327 thread_state->mdscr_el1 = 0ULL; // Legacy customers issuing ARM_DEBUG_STATE dont drive single stepping.
1328 }
1329
1330 if (thread == current_thread()) {
1331 arm_debug_set32(thread->machine.DebugData);
1332 }
1333
1334 break;
1335 }
1336 case ARM_DEBUG_STATE32:
1337 /* ARM64_TODO subtle bcr/wcr semantic differences e.g. wcr and ARM_DBGBCR_TYPE_IVA */
1338 {
1339 arm_debug_state32_t *state;
1340 boolean_t enabled = FALSE;
1341 unsigned int i;
1342
1343 if (count != ARM_DEBUG_STATE32_COUNT) {
1344 return KERN_INVALID_ARGUMENT;
1345 }
1346 if (thread_is_64bit_data(thread)) {
1347 return KERN_INVALID_ARGUMENT;
1348 }
1349
1350 state = (arm_debug_state32_t *) tstate;
1351
1352 if (state->mdscr_el1 & MDSCR_SS) {
1353 enabled = TRUE;
1354 }
1355
1356 for (i = 0; i < 16; i++) {
1357 /* do not allow context IDs to be set */
1358 if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
1359 || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)
1360 || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
1361 || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) {
1362 return KERN_PROTECTION_FAILURE;
1363 }
1364 if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE))
1365 || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) {
1366 enabled = TRUE;
1367 }
1368 }
1369
1370 if (!enabled) {
1371 free_debug_state(thread);
1372 } else {
1373 arm_debug_state32_t * thread_state = find_or_allocate_debug_state32(thread);
1374
1375 if (thread_state == NULL) {
1376 return KERN_FAILURE;
1377 }
1378
1379 if (state->mdscr_el1 & MDSCR_SS) {
1380 thread_state->mdscr_el1 |= MDSCR_SS;
1381 } else {
1382 thread_state->mdscr_el1 &= ~MDSCR_SS;
1383 }
1384
1385 for (i = 0; i < 16; i++) {
1386 /* set appropriate privilege; mask out unknown bits */
1387 thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
1388 | ARM_DBGBCR_MATCH_MASK
1389 | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK
1390 | ARM_DBG_CR_ENABLE_MASK))
1391 | ARM_DBGBCR_TYPE_IVA
1392 | ARM_DBG_CR_LINKED_UNLINKED
1393 | ARM_DBG_CR_SECURITY_STATE_BOTH
1394 | ARM_DBG_CR_MODE_CONTROL_USER;
1395 thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK;
1396 thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
1397 | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK
1398 | ARM_DBGWCR_ACCESS_CONTROL_MASK
1399 | ARM_DBG_CR_ENABLE_MASK))
1400 | ARM_DBG_CR_LINKED_UNLINKED
1401 | ARM_DBG_CR_SECURITY_STATE_BOTH
1402 | ARM_DBG_CR_MODE_CONTROL_USER;
1403 thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK;
1404 }
1405 }
1406
1407 if (thread == current_thread()) {
1408 arm_debug_set32(thread->machine.DebugData);
1409 }
1410
1411 break;
1412 }
1413
1414 case ARM_DEBUG_STATE64:
1415 {
1416 arm_debug_state64_t *state;
1417 boolean_t enabled = FALSE;
1418 unsigned int i;
1419
1420 if (count != ARM_DEBUG_STATE64_COUNT) {
1421 return KERN_INVALID_ARGUMENT;
1422 }
1423 if (!thread_is_64bit_data(thread)) {
1424 return KERN_INVALID_ARGUMENT;
1425 }
1426
1427 state = (arm_debug_state64_t *) tstate;
1428
1429 if (state->mdscr_el1 & MDSCR_SS) {
1430 enabled = TRUE;
1431 }
1432
1433 for (i = 0; i < 16; i++) {
1434 /* do not allow context IDs to be set */
1435 if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
1436 || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)
1437 || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) {
1438 return KERN_PROTECTION_FAILURE;
1439 }
1440 if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE))
1441 || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) {
1442 enabled = TRUE;
1443 }
1444 }
1445
1446 if (!enabled) {
1447 free_debug_state(thread);
1448 } else {
1449 arm_debug_state64_t *thread_state = find_or_allocate_debug_state64(thread);
1450
1451 if (thread_state == NULL) {
1452 return KERN_FAILURE;
1453 }
1454
1455 if (state->mdscr_el1 & MDSCR_SS) {
1456 thread_state->mdscr_el1 |= MDSCR_SS;
1457 } else {
1458 thread_state->mdscr_el1 &= ~MDSCR_SS;
1459 }
1460
1461 for (i = 0; i < 16; i++) {
1462 /* set appropriate privilege; mask out unknown bits */
1463 thread_state->bcr[i] = (state->bcr[i] & (0 /* Was ARM_DBG_CR_ADDRESS_MASK_MASK deprecated in v8 */
1464 | 0 /* Was ARM_DBGBCR_MATCH_MASK, ignored in AArch64 state */
1465 | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK
1466 | ARM_DBG_CR_ENABLE_MASK))
1467 | ARM_DBGBCR_TYPE_IVA
1468 | ARM_DBG_CR_LINKED_UNLINKED
1469 | ARM_DBG_CR_SECURITY_STATE_BOTH
1470 | ARM_DBG_CR_MODE_CONTROL_USER;
1471 thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK64;
1472 thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
1473 | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK
1474 | ARM_DBGWCR_ACCESS_CONTROL_MASK
1475 | ARM_DBG_CR_ENABLE_MASK))
1476 | ARM_DBG_CR_LINKED_UNLINKED
1477 | ARM_DBG_CR_SECURITY_STATE_BOTH
1478 | ARM_DBG_CR_MODE_CONTROL_USER;
1479 thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK64;
1480 }
1481 }
1482
1483 if (thread == current_thread()) {
1484 arm_debug_set64(thread->machine.DebugData);
1485 }
1486
1487 break;
1488 }
1489
1490 case ARM_VFP_STATE:{
1491 struct arm_vfp_state *state;
1492 arm_neon_saved_state32_t *thread_state;
1493 unsigned int max;
1494
1495 if (count != ARM_VFP_STATE_COUNT && count != ARM_VFPV2_STATE_COUNT) {
1496 return KERN_INVALID_ARGUMENT;
1497 }
1498
1499 if (count == ARM_VFPV2_STATE_COUNT) {
1500 max = 32;
1501 } else {
1502 max = 64;
1503 }
1504
1505 state = (struct arm_vfp_state *) tstate;
1506 thread_state = neon_state32(thread->machine.uNeon);
1507 /* ARM64 TODO: combine fpsr and fpcr into state->fpscr */
1508
1509 bcopy(state, thread_state, (max + 1) * sizeof(uint32_t));
1510
1511 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
1512 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
1513 break;
1514 }
1515
1516 case ARM_NEON_STATE:{
1517 arm_neon_state_t *state;
1518 arm_neon_saved_state32_t *thread_state;
1519
1520 if (count != ARM_NEON_STATE_COUNT) {
1521 return KERN_INVALID_ARGUMENT;
1522 }
1523
1524 if (thread_is_64bit_data(thread)) {
1525 return KERN_INVALID_ARGUMENT;
1526 }
1527
1528 state = (arm_neon_state_t *)tstate;
1529 thread_state = neon_state32(thread->machine.uNeon);
1530
1531 assert(sizeof(*state) == sizeof(*thread_state));
1532 bcopy(state, thread_state, sizeof(arm_neon_state_t));
1533
1534 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
1535 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
1536 break;
1537 }
1538
1539 case ARM_NEON_STATE64:{
1540 arm_neon_state64_t *state;
1541 arm_neon_saved_state64_t *thread_state;
1542
1543 if (count != ARM_NEON_STATE64_COUNT) {
1544 return KERN_INVALID_ARGUMENT;
1545 }
1546
1547 if (!thread_is_64bit_data(thread)) {
1548 return KERN_INVALID_ARGUMENT;
1549 }
1550
1551 state = (arm_neon_state64_t *)tstate;
1552 thread_state = neon_state64(thread->machine.uNeon);
1553
1554 assert(sizeof(*state) == sizeof(*thread_state));
1555 bcopy(state, thread_state, sizeof(arm_neon_state64_t));
1556
1557
1558 thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64;
1559 thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT;
1560 break;
1561 }
1562
1563
1564 default:
1565 return KERN_INVALID_ARGUMENT;
1566 }
1567 return KERN_SUCCESS;
1568 }
1569
1570 mach_vm_address_t
machine_thread_pc(thread_t thread)1571 machine_thread_pc(thread_t thread)
1572 {
1573 struct arm_saved_state *ss = get_user_regs(thread);
1574 return (mach_vm_address_t)get_saved_state_pc(ss);
1575 }
1576
1577 void
machine_thread_reset_pc(thread_t thread,mach_vm_address_t pc)1578 machine_thread_reset_pc(thread_t thread, mach_vm_address_t pc)
1579 {
1580 set_saved_state_pc(get_user_regs(thread), (register_t)pc);
1581 }
1582
1583 /*
1584 * Routine: machine_thread_state_initialize
1585 *
1586 */
1587 void
machine_thread_state_initialize(thread_t thread)1588 machine_thread_state_initialize(thread_t thread)
1589 {
1590 arm_context_t *context = thread->machine.contextData;
1591
1592 /*
1593 * Should always be set up later. For a kernel thread, we don't care
1594 * about this state. For a user thread, we'll set the state up in
1595 * setup_wqthread, bsdthread_create, load_main(), or load_unixthread().
1596 */
1597
1598 if (context != NULL) {
1599 bzero(&context->ss.uss, sizeof(context->ss.uss));
1600 bzero(&context->ns.uns, sizeof(context->ns.uns));
1601
1602 if (context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64) {
1603 context->ns.ns_64.fpcr = FPCR_DEFAULT;
1604 } else {
1605 context->ns.ns_32.fpcr = FPCR_DEFAULT_32;
1606 }
1607 context->ss.ss_64.cpsr = PSR64_USER64_DEFAULT;
1608 }
1609
1610 thread->machine.DebugData = NULL;
1611
1612 #if defined(HAS_APPLE_PAC)
1613 /* Sign the initial user-space thread state */
1614 if (thread->machine.upcb != NULL) {
1615 uint64_t intr = ml_pac_safe_interrupts_disable();
1616 asm volatile (
1617 "mov x0, %[iss]" "\n"
1618 "mov x1, #0" "\n"
1619 "mov w2, %w[usr]" "\n"
1620 "mov x3, #0" "\n"
1621 "mov x4, #0" "\n"
1622 "mov x5, #0" "\n"
1623 "mov x6, lr" "\n"
1624 "msr SPSel, #1" "\n"
1625 "bl _ml_sign_thread_state" "\n"
1626 "msr SPSel, #0" "\n"
1627 "mov lr, x6" "\n"
1628 :
1629 : [iss] "r"(thread->machine.upcb), [usr] "r"(thread->machine.upcb->ss_64.cpsr)
1630 : "x0", "x1", "x2", "x3", "x4", "x5", "x6"
1631 );
1632 ml_pac_safe_interrupts_restore(intr);
1633 }
1634 #endif /* defined(HAS_APPLE_PAC) */
1635 }
1636
1637 /*
1638 * Routine: machine_thread_dup
1639 *
1640 */
1641 kern_return_t
machine_thread_dup(thread_t self,thread_t target,__unused boolean_t is_corpse)1642 machine_thread_dup(thread_t self,
1643 thread_t target,
1644 __unused boolean_t is_corpse)
1645 {
1646 struct arm_saved_state *self_saved_state;
1647 struct arm_saved_state *target_saved_state;
1648
1649 target->machine.cthread_self = self->machine.cthread_self;
1650
1651 self_saved_state = self->machine.upcb;
1652 target_saved_state = target->machine.upcb;
1653 bcopy(self_saved_state, target_saved_state, sizeof(struct arm_saved_state));
1654 #if defined(HAS_APPLE_PAC)
1655 if (!is_corpse && is_saved_state64(self_saved_state)) {
1656 check_and_sign_copied_thread_state(target_saved_state, self_saved_state);
1657 }
1658 #endif /* defined(HAS_APPLE_PAC) */
1659
1660 arm_neon_saved_state_t *self_neon_state = self->machine.uNeon;
1661 arm_neon_saved_state_t *target_neon_state = target->machine.uNeon;
1662 bcopy(self_neon_state, target_neon_state, sizeof(*target_neon_state));
1663
1664 return KERN_SUCCESS;
1665 }
1666
1667 /*
1668 * Routine: get_user_regs
1669 *
1670 */
1671 struct arm_saved_state *
get_user_regs(thread_t thread)1672 get_user_regs(thread_t thread)
1673 {
1674 return thread->machine.upcb;
1675 }
1676
1677 arm_neon_saved_state_t *
get_user_neon_regs(thread_t thread)1678 get_user_neon_regs(thread_t thread)
1679 {
1680 return thread->machine.uNeon;
1681 }
1682
1683 /*
1684 * Routine: find_user_regs
1685 *
1686 */
1687 struct arm_saved_state *
find_user_regs(thread_t thread)1688 find_user_regs(thread_t thread)
1689 {
1690 return thread->machine.upcb;
1691 }
1692
1693 /*
1694 * Routine: find_kern_regs
1695 *
1696 */
1697 struct arm_saved_state *
find_kern_regs(thread_t thread)1698 find_kern_regs(thread_t thread)
1699 {
1700 /*
1701 * This works only for an interrupted kernel thread
1702 */
1703 if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) {
1704 return (struct arm_saved_state *) NULL;
1705 } else {
1706 return getCpuDatap()->cpu_int_state;
1707 }
1708 }
1709
1710 arm_debug_state32_t *
find_debug_state32(thread_t thread)1711 find_debug_state32(thread_t thread)
1712 {
1713 if (thread && thread->machine.DebugData) {
1714 return &(thread->machine.DebugData->uds.ds32);
1715 } else {
1716 return NULL;
1717 }
1718 }
1719
1720 arm_debug_state64_t *
find_debug_state64(thread_t thread)1721 find_debug_state64(thread_t thread)
1722 {
1723 if (thread && thread->machine.DebugData) {
1724 return &(thread->machine.DebugData->uds.ds64);
1725 } else {
1726 return NULL;
1727 }
1728 }
1729
1730 os_refgrp_decl(static, dbg_refgrp, "arm_debug_state", NULL);
1731
1732 /**
1733 * Finds the debug state for the given 64 bit thread, allocating one if it
1734 * does not exist.
1735 *
1736 * @param thread 64 bit thread to find or allocate debug state for
1737 *
1738 * @returns A pointer to the given thread's 64 bit debug state or a null
1739 * pointer if the given thread is null or the allocation of a new
1740 * debug state fails.
1741 */
1742 arm_debug_state64_t *
find_or_allocate_debug_state64(thread_t thread)1743 find_or_allocate_debug_state64(thread_t thread)
1744 {
1745 arm_debug_state64_t *thread_state = find_debug_state64(thread);
1746 if (thread != NULL && thread_state == NULL) {
1747 thread->machine.DebugData = zalloc_flags(ads_zone,
1748 Z_WAITOK | Z_NOFAIL);
1749 bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData));
1750 thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE64;
1751 thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE64_COUNT;
1752 os_ref_init(&thread->machine.DebugData->ref, &dbg_refgrp);
1753 thread_state = find_debug_state64(thread);
1754 }
1755 return thread_state;
1756 }
1757
1758 /**
1759 * Finds the debug state for the given 32 bit thread, allocating one if it
1760 * does not exist.
1761 *
1762 * @param thread 32 bit thread to find or allocate debug state for
1763 *
1764 * @returns A pointer to the given thread's 32 bit debug state or a null
1765 * pointer if the given thread is null or the allocation of a new
1766 * debug state fails.
1767 */
1768 arm_debug_state32_t *
find_or_allocate_debug_state32(thread_t thread)1769 find_or_allocate_debug_state32(thread_t thread)
1770 {
1771 arm_debug_state32_t *thread_state = find_debug_state32(thread);
1772 if (thread != NULL && thread_state == NULL) {
1773 thread->machine.DebugData = zalloc_flags(ads_zone,
1774 Z_WAITOK | Z_NOFAIL);
1775 bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData));
1776 thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE32;
1777 thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE32_COUNT;
1778 os_ref_init(&thread->machine.DebugData->ref, &dbg_refgrp);
1779 thread_state = find_debug_state32(thread);
1780 }
1781 return thread_state;
1782 }
1783
1784 /**
1785 * Frees a thread's debug state if allocated. Otherwise does nothing.
1786 *
1787 * @param thread thread to free the debug state of
1788 */
1789 static inline void
free_debug_state(thread_t thread)1790 free_debug_state(thread_t thread)
1791 {
1792 if (thread != NULL && thread->machine.DebugData != NULL) {
1793 arm_debug_state_t *pTmp = thread->machine.DebugData;
1794 thread->machine.DebugData = NULL;
1795
1796 if (os_ref_release(&pTmp->ref) == 0) {
1797 zfree(ads_zone, pTmp);
1798 }
1799 }
1800 }
1801
1802 /*
1803 * Routine: thread_userstack
1804 *
1805 */
1806 kern_return_t
thread_userstack(__unused thread_t thread,int flavor,thread_state_t tstate,unsigned int count,mach_vm_offset_t * user_stack,int * customstack,boolean_t is_64bit_data)1807 thread_userstack(__unused thread_t thread,
1808 int flavor,
1809 thread_state_t tstate,
1810 unsigned int count,
1811 mach_vm_offset_t * user_stack,
1812 int * customstack,
1813 boolean_t is_64bit_data
1814 )
1815 {
1816 register_t sp;
1817
1818 switch (flavor) {
1819 case ARM_THREAD_STATE:
1820 if (count == ARM_UNIFIED_THREAD_STATE_COUNT) {
1821 #if __arm64__
1822 if (is_64bit_data) {
1823 sp = ((arm_unified_thread_state_t *)tstate)->ts_64.sp;
1824 } else
1825 #endif
1826 {
1827 sp = ((arm_unified_thread_state_t *)tstate)->ts_32.sp;
1828 }
1829
1830 break;
1831 }
1832
1833 /* INTENTIONAL FALL THROUGH (see machine_thread_set_state) */
1834 OS_FALLTHROUGH;
1835 case ARM_THREAD_STATE32:
1836 if (count != ARM_THREAD_STATE32_COUNT) {
1837 return KERN_INVALID_ARGUMENT;
1838 }
1839 if (is_64bit_data) {
1840 return KERN_INVALID_ARGUMENT;
1841 }
1842
1843 sp = ((arm_thread_state32_t *)tstate)->sp;
1844 break;
1845 #if __arm64__
1846 case ARM_THREAD_STATE64:
1847 if (count != ARM_THREAD_STATE64_COUNT) {
1848 return KERN_INVALID_ARGUMENT;
1849 }
1850 if (!is_64bit_data) {
1851 return KERN_INVALID_ARGUMENT;
1852 }
1853
1854 sp = ((arm_thread_state32_t *)tstate)->sp;
1855 break;
1856 #endif
1857 default:
1858 return KERN_INVALID_ARGUMENT;
1859 }
1860
1861 if (sp) {
1862 *user_stack = CAST_USER_ADDR_T(sp);
1863 if (customstack) {
1864 *customstack = 1;
1865 }
1866 } else {
1867 *user_stack = CAST_USER_ADDR_T(USRSTACK64);
1868 if (customstack) {
1869 *customstack = 0;
1870 }
1871 }
1872
1873 return KERN_SUCCESS;
1874 }
1875
1876 /*
1877 * thread_userstackdefault:
1878 *
1879 * Return the default stack location for the
1880 * thread, if otherwise unknown.
1881 */
1882 kern_return_t
thread_userstackdefault(mach_vm_offset_t * default_user_stack,boolean_t is64bit)1883 thread_userstackdefault(mach_vm_offset_t * default_user_stack,
1884 boolean_t is64bit)
1885 {
1886 if (is64bit) {
1887 *default_user_stack = USRSTACK64;
1888 } else {
1889 *default_user_stack = USRSTACK;
1890 }
1891
1892 return KERN_SUCCESS;
1893 }
1894
1895 /*
1896 * Routine: thread_setuserstack
1897 *
1898 */
1899 void
thread_setuserstack(thread_t thread,mach_vm_address_t user_stack)1900 thread_setuserstack(thread_t thread,
1901 mach_vm_address_t user_stack)
1902 {
1903 struct arm_saved_state *sv;
1904
1905 sv = get_user_regs(thread);
1906
1907 set_saved_state_sp(sv, user_stack);
1908
1909 return;
1910 }
1911
1912 /*
1913 * Routine: thread_adjuserstack
1914 *
1915 */
1916 user_addr_t
thread_adjuserstack(thread_t thread,int adjust)1917 thread_adjuserstack(thread_t thread,
1918 int adjust)
1919 {
1920 struct arm_saved_state *sv;
1921 uint64_t sp;
1922
1923 sv = get_user_regs(thread);
1924
1925 sp = get_saved_state_sp(sv);
1926 sp += adjust;
1927 set_saved_state_sp(sv, sp);
1928
1929 return sp;
1930 }
1931
1932
1933 /*
1934 * Routine: thread_setentrypoint
1935 *
1936 */
1937 void
thread_setentrypoint(thread_t thread,mach_vm_offset_t entry)1938 thread_setentrypoint(thread_t thread,
1939 mach_vm_offset_t entry)
1940 {
1941 struct arm_saved_state *sv;
1942
1943 sv = get_user_regs(thread);
1944
1945 set_saved_state_pc(sv, entry);
1946
1947 return;
1948 }
1949
1950 /*
1951 * Routine: thread_entrypoint
1952 *
1953 */
1954 kern_return_t
thread_entrypoint(__unused thread_t thread,int flavor,thread_state_t tstate,unsigned int count,mach_vm_offset_t * entry_point)1955 thread_entrypoint(__unused thread_t thread,
1956 int flavor,
1957 thread_state_t tstate,
1958 unsigned int count,
1959 mach_vm_offset_t * entry_point
1960 )
1961 {
1962 switch (flavor) {
1963 case ARM_THREAD_STATE:
1964 {
1965 struct arm_thread_state *state;
1966
1967 if (count != ARM_THREAD_STATE_COUNT) {
1968 return KERN_INVALID_ARGUMENT;
1969 }
1970
1971 state = (struct arm_thread_state *) tstate;
1972
1973 /*
1974 * If a valid entry point is specified, use it.
1975 */
1976 if (state->pc) {
1977 *entry_point = CAST_USER_ADDR_T(state->pc);
1978 } else {
1979 *entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS);
1980 }
1981 }
1982 break;
1983
1984 case ARM_THREAD_STATE64:
1985 {
1986 struct arm_thread_state64 *state;
1987
1988 if (count != ARM_THREAD_STATE64_COUNT) {
1989 return KERN_INVALID_ARGUMENT;
1990 }
1991
1992 state = (struct arm_thread_state64*) tstate;
1993
1994 /*
1995 * If a valid entry point is specified, use it.
1996 */
1997 if (state->pc) {
1998 *entry_point = CAST_USER_ADDR_T(state->pc);
1999 } else {
2000 *entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS);
2001 }
2002
2003 break;
2004 }
2005 default:
2006 return KERN_INVALID_ARGUMENT;
2007 }
2008
2009 return KERN_SUCCESS;
2010 }
2011
2012
2013 /*
2014 * Routine: thread_set_child
2015 *
2016 */
2017 void
thread_set_child(thread_t child,int pid)2018 thread_set_child(thread_t child,
2019 int pid)
2020 {
2021 struct arm_saved_state *child_state;
2022
2023 child_state = get_user_regs(child);
2024
2025 set_saved_state_reg(child_state, 0, pid);
2026 set_saved_state_reg(child_state, 1, 1ULL);
2027 }
2028
2029
2030 /*
2031 * Routine: thread_set_parent
2032 *
2033 */
2034 void
thread_set_parent(thread_t parent,int pid)2035 thread_set_parent(thread_t parent,
2036 int pid)
2037 {
2038 struct arm_saved_state *parent_state;
2039
2040 parent_state = get_user_regs(parent);
2041
2042 set_saved_state_reg(parent_state, 0, pid);
2043 set_saved_state_reg(parent_state, 1, 0);
2044 }
2045
2046
2047 struct arm_act_context {
2048 struct arm_unified_thread_state ss;
2049 #if __ARM_VFP__
2050 struct arm_neon_saved_state ns;
2051 #endif
2052 };
2053
2054 /*
2055 * Routine: act_thread_csave
2056 *
2057 */
2058 void *
act_thread_csave(void)2059 act_thread_csave(void)
2060 {
2061 struct arm_act_context *ic;
2062 kern_return_t kret;
2063 unsigned int val;
2064 thread_t thread = current_thread();
2065
2066 ic = kalloc_type(struct arm_act_context, Z_WAITOK);
2067 if (ic == (struct arm_act_context *) NULL) {
2068 return (void *) 0;
2069 }
2070
2071 val = ARM_UNIFIED_THREAD_STATE_COUNT;
2072 kret = machine_thread_get_state(thread, ARM_THREAD_STATE, (thread_state_t)&ic->ss, &val);
2073 if (kret != KERN_SUCCESS) {
2074 kfree_type(struct arm_act_context, ic);
2075 return (void *) 0;
2076 }
2077
2078 #if __ARM_VFP__
2079 if (thread_is_64bit_data(thread)) {
2080 val = ARM_NEON_STATE64_COUNT;
2081 kret = machine_thread_get_state(thread,
2082 ARM_NEON_STATE64,
2083 (thread_state_t)&ic->ns,
2084 &val);
2085 } else {
2086 val = ARM_NEON_STATE_COUNT;
2087 kret = machine_thread_get_state(thread,
2088 ARM_NEON_STATE,
2089 (thread_state_t)&ic->ns,
2090 &val);
2091 }
2092 if (kret != KERN_SUCCESS) {
2093 kfree_type(struct arm_act_context, ic);
2094 return (void *) 0;
2095 }
2096 #endif
2097 return ic;
2098 }
2099
2100 /*
2101 * Routine: act_thread_catt
2102 *
2103 */
2104 void
act_thread_catt(void * ctx)2105 act_thread_catt(void * ctx)
2106 {
2107 struct arm_act_context *ic;
2108 kern_return_t kret;
2109 thread_t thread = current_thread();
2110
2111 ic = (struct arm_act_context *) ctx;
2112 if (ic == (struct arm_act_context *) NULL) {
2113 return;
2114 }
2115
2116 kret = machine_thread_set_state(thread, ARM_THREAD_STATE, (thread_state_t)&ic->ss, ARM_UNIFIED_THREAD_STATE_COUNT);
2117 if (kret != KERN_SUCCESS) {
2118 goto out;
2119 }
2120
2121 #if __ARM_VFP__
2122 if (thread_is_64bit_data(thread)) {
2123 kret = machine_thread_set_state(thread,
2124 ARM_NEON_STATE64,
2125 (thread_state_t)&ic->ns,
2126 ARM_NEON_STATE64_COUNT);
2127 } else {
2128 kret = machine_thread_set_state(thread,
2129 ARM_NEON_STATE,
2130 (thread_state_t)&ic->ns,
2131 ARM_NEON_STATE_COUNT);
2132 }
2133 if (kret != KERN_SUCCESS) {
2134 goto out;
2135 }
2136 #endif
2137 out:
2138 kfree_type(struct arm_act_context, ic);
2139 }
2140
2141 /*
2142 * Routine: act_thread_catt
2143 *
2144 */
2145 void
act_thread_cfree(void * ctx)2146 act_thread_cfree(void *ctx)
2147 {
2148 kfree_type(struct arm_act_context, ctx);
2149 }
2150
2151 kern_return_t
thread_set_wq_state32(thread_t thread,thread_state_t tstate)2152 thread_set_wq_state32(thread_t thread,
2153 thread_state_t tstate)
2154 {
2155 arm_thread_state_t *state;
2156 struct arm_saved_state *saved_state;
2157 struct arm_saved_state32 *saved_state_32;
2158 thread_t curth = current_thread();
2159 spl_t s = 0;
2160
2161 assert(!thread_is_64bit_data(thread));
2162
2163 saved_state = thread->machine.upcb;
2164 saved_state_32 = saved_state32(saved_state);
2165
2166 state = (arm_thread_state_t *)tstate;
2167
2168 if (curth != thread) {
2169 s = splsched();
2170 thread_lock(thread);
2171 }
2172
2173 /*
2174 * do not zero saved_state, it can be concurrently accessed
2175 * and zero is not a valid state for some of the registers,
2176 * like sp.
2177 */
2178 thread_state32_to_saved_state(state, saved_state);
2179 saved_state_32->cpsr = PSR64_USER32_DEFAULT;
2180
2181 if (curth != thread) {
2182 thread_unlock(thread);
2183 splx(s);
2184 }
2185
2186 return KERN_SUCCESS;
2187 }
2188
2189 kern_return_t
thread_set_wq_state64(thread_t thread,thread_state_t tstate)2190 thread_set_wq_state64(thread_t thread,
2191 thread_state_t tstate)
2192 {
2193 arm_thread_state64_t *state;
2194 struct arm_saved_state *saved_state;
2195 struct arm_saved_state64 *saved_state_64;
2196 thread_t curth = current_thread();
2197 spl_t s = 0;
2198
2199 assert(thread_is_64bit_data(thread));
2200
2201 saved_state = thread->machine.upcb;
2202 saved_state_64 = saved_state64(saved_state);
2203 state = (arm_thread_state64_t *)tstate;
2204
2205 if (curth != thread) {
2206 s = splsched();
2207 thread_lock(thread);
2208 }
2209
2210 /*
2211 * do not zero saved_state, it can be concurrently accessed
2212 * and zero is not a valid state for some of the registers,
2213 * like sp.
2214 */
2215 thread_state64_to_saved_state(state, saved_state);
2216 set_saved_state_cpsr(saved_state, PSR64_USER64_DEFAULT);
2217
2218 if (curth != thread) {
2219 thread_unlock(thread);
2220 splx(s);
2221 }
2222
2223 return KERN_SUCCESS;
2224 }
2225