1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * FILE_ID: thread_status.h
30 */
31
32
33 #ifndef _ARM_THREAD_STATUS_H_
34 #define _ARM_THREAD_STATUS_H_
35
36 #if defined (__arm__) || defined (__arm64__)
37
38 #include <mach/machine/_structs.h>
39 #include <mach/machine/thread_state.h>
40 #include <mach/message.h>
41 #include <mach/vm_types.h>
42
43 /*
44 * Support for determining the state of a thread
45 */
46
47
48 /*
49 * Flavors
50 */
51
52 #define ARM_THREAD_STATE 1
53 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
54 #define ARM_VFP_STATE 2
55 #define ARM_EXCEPTION_STATE 3
56 #define ARM_DEBUG_STATE 4 /* pre-armv8 */
57 #define THREAD_STATE_NONE 5
58 #define ARM_THREAD_STATE64 6
59 #define ARM_EXCEPTION_STATE64 7
60 // ARM_THREAD_STATE_LAST 8 /* legacy */
61 #define ARM_THREAD_STATE32 9
62
63 #ifdef XNU_KERNEL_PRIVATE
64 #define X86_THREAD_STATE_NONE 13 /* i386/thread_status.h THREAD_STATE_NONE */
65 #endif /* XNU_KERNEL_PRIVATE */
66
67 /* API */
68 #define ARM_DEBUG_STATE32 14
69 #define ARM_DEBUG_STATE64 15
70 #define ARM_NEON_STATE 16
71 #define ARM_NEON_STATE64 17
72 #define ARM_CPMU_STATE64 18
73
74 #ifdef XNU_KERNEL_PRIVATE
75 /* For kernel use */
76 #define ARM_SAVED_STATE32 20
77 #define ARM_SAVED_STATE64 21
78 #define ARM_NEON_SAVED_STATE32 22
79 #define ARM_NEON_SAVED_STATE64 23
80 #endif /* XNU_KERNEL_PRIVATE */
81
82 #define ARM_PAGEIN_STATE 27
83
84 #ifndef ARM_STATE_FLAVOR_IS_OTHER_VALID
85 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
86 #endif
87
88 #define VALID_THREAD_STATE_FLAVOR(x) \
89 ((x == ARM_THREAD_STATE) || \
90 (x == ARM_VFP_STATE) || \
91 (x == ARM_EXCEPTION_STATE) || \
92 (x == ARM_DEBUG_STATE) || \
93 (x == THREAD_STATE_NONE) || \
94 (x == ARM_THREAD_STATE32) || \
95 (x == ARM_THREAD_STATE64) || \
96 (x == ARM_EXCEPTION_STATE64) || \
97 (x == ARM_NEON_STATE) || \
98 (x == ARM_NEON_STATE64) || \
99 (x == ARM_DEBUG_STATE32) || \
100 (x == ARM_DEBUG_STATE64) || \
101 (x == ARM_PAGEIN_STATE) || \
102 (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
103
104 struct arm_state_hdr {
105 uint32_t flavor;
106 uint32_t count;
107 };
108 typedef struct arm_state_hdr arm_state_hdr_t;
109
110 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t;
111 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t;
112 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
113
114 #if !defined(KERNEL)
115 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
116
117 /* Accessor macros for arm_thread_state64_t pointer fields */
118
119 /* Return pc field of arm_thread_state64_t as a data pointer value */
120 #define arm_thread_state64_get_pc(ts) \
121 __darwin_arm_thread_state64_get_pc(ts)
122 /* Return pc field of arm_thread_state64_t as a function pointer. May return
123 * NULL if a valid function pointer cannot be constructed, the caller should
124 * fall back to the arm_thread_state64_get_pc() macro in that case. */
125 #define arm_thread_state64_get_pc_fptr(ts) \
126 __darwin_arm_thread_state64_get_pc_fptr(ts)
127 /* Set pc field of arm_thread_state64_t to a function pointer */
128 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
129 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
130 /* Return lr field of arm_thread_state64_t as a data pointer value */
131 #define arm_thread_state64_get_lr(ts) \
132 __darwin_arm_thread_state64_get_lr(ts)
133 /* Return lr field of arm_thread_state64_t as a function pointer. May return
134 * NULL if a valid function pointer cannot be constructed, the caller should
135 * fall back to the arm_thread_state64_get_lr() macro in that case. */
136 #define arm_thread_state64_get_lr_fptr(ts) \
137 __darwin_arm_thread_state64_get_lr_fptr(ts)
138 /* Set lr field of arm_thread_state64_t to a function pointer */
139 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
140 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
141 /* Return sp field of arm_thread_state64_t as a data pointer value */
142 #define arm_thread_state64_get_sp(ts) \
143 __darwin_arm_thread_state64_get_sp(ts)
144 /* Set sp field of arm_thread_state64_t to a data pointer value */
145 #define arm_thread_state64_set_sp(ts, ptr) \
146 __darwin_arm_thread_state64_set_sp(ts, ptr)
147 /* Return fp field of arm_thread_state64_t as a data pointer value */
148 #define arm_thread_state64_get_fp(ts) \
149 __darwin_arm_thread_state64_get_fp(ts)
150 /* Set fp field of arm_thread_state64_t to a data pointer value */
151 #define arm_thread_state64_set_fp(ts, ptr) \
152 __darwin_arm_thread_state64_set_fp(ts, ptr)
153 /* Strip ptr auth bits from pc, lr, sp and fp field of arm_thread_state64_t */
154 #define arm_thread_state64_ptrauth_strip(ts) \
155 __darwin_arm_thread_state64_ptrauth_strip(ts)
156
157 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
158 #endif /* !defined(KERNEL) */
159
160 struct arm_unified_thread_state {
161 arm_state_hdr_t ash;
162 union {
163 arm_thread_state32_t ts_32;
164 arm_thread_state64_t ts_64;
165 } uts;
166 };
167 #define ts_32 uts.ts_32
168 #define ts_64 uts.ts_64
169 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
170
171 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
172 (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
173 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
174 (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
175 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
176 (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
177 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
178 (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
179
180
181 typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t;
182 typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t;
183 typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t;
184 typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t;
185
186
187 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t;
188 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t;
189 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
190
191 typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t;
192 typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t;
193
194 typedef _STRUCT_ARM_PAGEIN_STATE arm_pagein_state_t;
195
196 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
197 /* See below for ARM64 kernel structure definition for arm_debug_state. */
198 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
199 /*
200 * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
201 * arm_debug_state for binary compatability of userland consumers of this file.
202 */
203 #if defined(__arm__)
204 typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t;
205 #elif defined(__arm64__)
206 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
207 #else /* defined(__arm__) */
208 #error Undefined architecture
209 #endif /* defined(__arm__) */
210 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
211
212 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
213 (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
214
215 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
216 (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
217
218 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
219 (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
220
221 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
222 (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
223
224 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
225 (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
226
227 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
228 (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
229
230 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
231 (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
232
233 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
234 (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
235
236 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
237 (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
238
239 #define MACHINE_THREAD_STATE ARM_THREAD_STATE
240 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
241
242
243 /*
244 * Largest state on this machine:
245 */
246 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
247
248 #ifdef XNU_KERNEL_PRIVATE
249
250 static inline boolean_t
is_thread_state32(const arm_unified_thread_state_t * its)251 is_thread_state32(const arm_unified_thread_state_t *its)
252 {
253 return its->ash.flavor == ARM_THREAD_STATE32;
254 }
255
256 static inline boolean_t
is_thread_state64(const arm_unified_thread_state_t * its)257 is_thread_state64(const arm_unified_thread_state_t *its)
258 {
259 return its->ash.flavor == ARM_THREAD_STATE64;
260 }
261
262 static inline arm_thread_state32_t*
thread_state32(arm_unified_thread_state_t * its)263 thread_state32(arm_unified_thread_state_t *its)
264 {
265 return &its->ts_32;
266 }
267
268 static inline arm_thread_state64_t*
thread_state64(arm_unified_thread_state_t * its)269 thread_state64(arm_unified_thread_state_t *its)
270 {
271 return &its->ts_64;
272 }
273
274 static inline const arm_thread_state32_t*
const_thread_state32(const arm_unified_thread_state_t * its)275 const_thread_state32(const arm_unified_thread_state_t *its)
276 {
277 return &its->ts_32;
278 }
279
280 static inline const arm_thread_state64_t*
const_thread_state64(const arm_unified_thread_state_t * its)281 const_thread_state64(const arm_unified_thread_state_t *its)
282 {
283 return &its->ts_64;
284 }
285
286 #if defined(__arm__)
287 #include <arm/proc_reg.h>
288
289 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
290
291 #if __ARM_VFP__
292 #define VFPSAVE_ALIGN 16
293 #define VFPSAVE_ATTRIB __attribute__((aligned (VFPSAVE_ALIGN)))
294 #define THREAD_ALIGN VFPSAVE_ALIGN
295
296 /*
297 * vector floating point saved state
298 */
299 struct arm_vfpsaved_state {
300 uint32_t r[64];
301 uint32_t fpscr;
302 uint32_t fpexc;
303 };
304 #endif
305
306 struct arm_saved_state {
307 uint32_t r[13]; /* General purpose register r0-r12 */
308 uint32_t sp; /* Stack pointer r13 */
309 uint32_t lr; /* Link register r14 */
310 uint32_t pc; /* Program counter r15 */
311 uint32_t cpsr; /* Current program status register */
312 uint32_t fsr; /* Fault status */
313 uint32_t far; /* Virtual Fault Address */
314 uint32_t exception; /* exception number */
315
316 #if __ARM_VFP__
317 /* VFP state */
318 struct arm_vfpsaved_state VFPdata VFPSAVE_ATTRIB;
319 // for packing reasons chtread_self and DebugData
320 // are inside the the PcbData when __ARM_VFP__ is set
321 arm_debug_state_t *VFPpadding_DebugData;
322 vm_address_t VFPpadding_cthread_self;
323 #endif
324 };
325 typedef struct arm_saved_state arm_saved_state_t;
326
327 /*
328 * Just for coexistence with AArch64 code.
329 */
330 typedef struct arm_saved_state arm_saved_state32_t;
331
332 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)333 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
334 {
335 *dst = *src;
336 }
337
338 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)339 saved_state32(arm_saved_state_t *iss)
340 {
341 return iss;
342 }
343
344 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss __unused)345 is_saved_state32(const arm_saved_state_t *iss __unused)
346 {
347 return TRUE;
348 }
349
350
351 struct arm_saved_state_tagged {
352 uint32_t tag;
353 struct arm_saved_state state;
354 };
355 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
356
357 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
358 (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
359
360
361 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)362 get_saved_state_pc(const arm_saved_state_t *iss)
363 {
364 return iss->pc;
365 }
366
367 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)368 add_saved_state_pc(arm_saved_state_t *iss, int diff)
369 {
370 iss->pc += (unsigned int)diff;
371 }
372
373 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)374 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
375 {
376 iss->pc = (typeof(iss->pc))pc;
377 }
378
379 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)380 get_saved_state_sp(const arm_saved_state_t *iss)
381 {
382 return iss->sp;
383 }
384
385 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)386 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
387 {
388 iss->sp = sp;
389 }
390
391 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)392 get_saved_state_fp(const arm_saved_state_t *iss)
393 {
394 return iss->r[7];
395 }
396
397 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)398 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
399 {
400 iss->r[7] = fp;
401 }
402
403 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)404 get_saved_state_lr(const arm_saved_state_t *iss)
405 {
406 return (register_t)iss->lr;
407 }
408
409 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)410 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
411 {
412 iss->lr = lr;
413 }
414
415 static inline register_t
get_saved_state_cpsr(const arm_saved_state_t * iss)416 get_saved_state_cpsr(const arm_saved_state_t *iss)
417 {
418 return iss->cpsr;
419 }
420
421 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)422 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
423 {
424 iss->cpsr |= set_bits;
425 iss->cpsr &= ~clear_bits;
426 }
427
428 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,register_t cpsr)429 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
430 {
431 iss->cpsr = cpsr;
432 }
433
434 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned regno)435 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
436 {
437 return iss->r[regno];
438 }
439
440 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned regno,register_t val)441 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
442 {
443 iss->r[regno] = val;
444 }
445
446 #elif defined(__arm64__)
447
448 #include <kern/assert.h>
449 #include <arm64/proc_reg.h>
450 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
451
452 /*
453 * GPR context
454 */
455
456 struct arm_saved_state32 {
457 uint32_t r[13]; /* General purpose register r0-r12 */
458 uint32_t sp; /* Stack pointer r13 */
459 uint32_t lr; /* Link register r14 */
460 uint32_t pc; /* Program counter r15 */
461 uint32_t cpsr; /* Current program status register */
462 uint32_t far; /* Virtual fault address */
463 uint32_t esr; /* Exception syndrome register */
464 uint32_t exception; /* Exception number */
465 };
466 typedef struct arm_saved_state32 arm_saved_state32_t;
467
468 struct arm_saved_state32_tagged {
469 uint32_t tag;
470 struct arm_saved_state32 state;
471 };
472 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
473
474 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
475 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
476
477 struct arm_saved_state64 {
478 uint64_t x[29]; /* General purpose registers x0-x28 */
479 uint64_t fp; /* Frame pointer x29 */
480 uint64_t lr; /* Link register x30 */
481 uint64_t sp; /* Stack pointer x31 */
482 uint64_t pc; /* Program counter */
483 uint32_t cpsr; /* Current program status register */
484 uint32_t reserved; /* Reserved padding */
485 uint64_t far; /* Virtual fault address */
486 uint32_t esr; /* Exception syndrome register */
487 uint32_t exception; /* Exception number */
488 #if defined(HAS_APPLE_PAC)
489 uint64_t jophash;
490 #endif /* defined(HAS_APPLE_PAC) */
491 };
492 typedef struct arm_saved_state64 arm_saved_state64_t;
493
494 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
495 (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
496
497 struct arm_saved_state64_tagged {
498 uint32_t tag;
499 struct arm_saved_state64 state;
500 };
501 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
502
503 struct arm_saved_state {
504 arm_state_hdr_t ash;
505 union {
506 struct arm_saved_state32 ss_32;
507 struct arm_saved_state64 ss_64;
508 } uss;
509 } __attribute__((aligned(16)));
510 #define ss_32 uss.ss_32
511 #define ss_64 uss.ss_64
512
513 typedef struct arm_saved_state arm_saved_state_t;
514
515 struct arm_kernel_saved_state {
516 uint64_t x[10]; /* General purpose registers x19-x28 */
517 uint64_t fp; /* Frame pointer x29 */
518 uint64_t lr; /* Link register x30 */
519 uint64_t sp; /* Stack pointer x31 */
520 uint64_t pc; /* Program counter */
521 } __attribute__((aligned(16)));
522
523 typedef struct arm_kernel_saved_state arm_kernel_saved_state_t;
524
525 #if defined(XNU_KERNEL_PRIVATE)
526 #if defined(HAS_APPLE_PAC)
527
528 #include <sys/cdefs.h>
529
530 /*
531 * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
532 * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
533 * and osfmk/arm/machine_routines.h.
534 */
535 __BEGIN_DECLS
536 extern uint64_t ml_pac_safe_interrupts_disable(void);
537 extern void ml_pac_safe_interrupts_restore(uint64_t);
538 __END_DECLS
539
540 /*
541 * Methods used to sign and check thread state to detect corruptions of saved
542 * thread state across exceptions and context switches.
543 */
544 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
545
546 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
547
548 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
549 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
550
551 /**
552 * Verify the signed thread state in _iss, execute the assembly instructions
553 * _instr, and re-sign the modified thread state. Varargs specify additional
554 * inputs.
555 *
556 * _instr may read or modify the thread state in the following registers:
557 *
558 * x0: _iss
559 * x1: authed _iss->ss_64.pc
560 * w2: authed _iss->ss_64.cpsr
561 * x3: authed _iss->ss_64.lr
562 * x4: authed _iss->ss_64.x16
563 * x5: authed _iss->ss_64.x17
564 * x6: scratch register
565 * x7: scratch register
566 */
567 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...) \
568 do { \
569 uint64_t _intr = ml_pac_safe_interrupts_disable(); \
570 asm volatile ( \
571 "mov x8, lr" "\n" \
572 "mov x0, %[iss]" "\n" \
573 "msr SPSel, #1" "\n" \
574 "ldp x4, x5, [x0, %[SS64_X16]]" "\n" \
575 "ldr x6, [x0, %[SS64_PC]]" "\n" \
576 "ldr w7, [x0, %[SS64_CPSR]]" "\n" \
577 "ldr x3, [x0, %[SS64_LR]]" "\n" \
578 "mov x1, x6" "\n" \
579 "mov w2, w7" "\n" \
580 "bl _ml_check_signed_state" "\n" \
581 "mov x1, x6" "\n" \
582 "mov w2, w7" "\n" \
583 _instr "\n" \
584 "bl _ml_sign_thread_state" "\n" \
585 "msr SPSel, #0" "\n" \
586 "mov lr, x8" "\n" \
587 : \
588 : [iss] "r"(_iss), \
589 [SS64_X16] "i"(ss64_offsetof(x[16])), \
590 [SS64_PC] "i"(ss64_offsetof(pc)), \
591 [SS64_CPSR] "i"(ss64_offsetof(cpsr)), \
592 [SS64_LR] "i"(ss64_offsetof(lr)),##__VA_ARGS__ \
593 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8" \
594 ); \
595 ml_pac_safe_interrupts_restore(_intr); \
596 } while (0)
597
598 static inline void
check_and_sign_copied_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)599 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
600 {
601 MANIPULATE_SIGNED_THREAD_STATE(src,
602 "mov x0, %[dst]",
603 [dst] "r"(dst)
604 );
605 }
606 #endif /* defined(HAS_APPLE_PAC) */
607
608 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)609 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
610 {
611 *dst = *src;
612 #if defined(HAS_APPLE_PAC)
613 check_and_sign_copied_thread_state(dst, src);
614 #endif
615 }
616
617 #endif /* defined(XNU_KERNEL_PRIVATE) */
618
619 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss)620 is_saved_state32(const arm_saved_state_t *iss)
621 {
622 return iss->ash.flavor == ARM_SAVED_STATE32;
623 }
624
625 static inline boolean_t
is_saved_state64(const arm_saved_state_t * iss)626 is_saved_state64(const arm_saved_state_t *iss)
627 {
628 return iss->ash.flavor == ARM_SAVED_STATE64;
629 }
630
631 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)632 saved_state32(arm_saved_state_t *iss)
633 {
634 return &iss->ss_32;
635 }
636
637 static inline const arm_saved_state32_t*
const_saved_state32(const arm_saved_state_t * iss)638 const_saved_state32(const arm_saved_state_t *iss)
639 {
640 return &iss->ss_32;
641 }
642
643 static inline arm_saved_state64_t*
saved_state64(arm_saved_state_t * iss)644 saved_state64(arm_saved_state_t *iss)
645 {
646 return &iss->ss_64;
647 }
648
649 static inline const arm_saved_state64_t*
const_saved_state64(const arm_saved_state_t * iss)650 const_saved_state64(const arm_saved_state_t *iss)
651 {
652 return &iss->ss_64;
653 }
654
655 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)656 get_saved_state_pc(const arm_saved_state_t *iss)
657 {
658 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc);
659 }
660
661 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)662 add_saved_state_pc(arm_saved_state_t *iss, int diff)
663 {
664 if (is_saved_state32(iss)) {
665 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
666 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
667 } else {
668 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
669 MANIPULATE_SIGNED_THREAD_STATE(iss,
670 "mov w6, %w[diff] \n"
671 "add x1, x1, w6, sxtw \n"
672 "str x1, [x0, %[SS64_PC]] \n",
673 [diff] "r"(diff)
674 );
675 #else
676 saved_state64(iss)->pc += (unsigned long)diff;
677 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
678 }
679 }
680
681 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)682 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
683 {
684 if (is_saved_state32(iss)) {
685 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
686 } else {
687 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
688 MANIPULATE_SIGNED_THREAD_STATE(iss,
689 "mov x1, %[pc] \n"
690 "str x1, [x0, %[SS64_PC]] \n",
691 [pc] "r"(pc)
692 );
693 #else
694 saved_state64(iss)->pc = (unsigned long)pc;
695 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
696 }
697 }
698
699 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)700 get_saved_state_sp(const arm_saved_state_t *iss)
701 {
702 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp);
703 }
704
705 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)706 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
707 {
708 if (is_saved_state32(iss)) {
709 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
710 } else {
711 saved_state64(iss)->sp = (uint64_t)sp;
712 }
713 }
714
715 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)716 get_saved_state_lr(const arm_saved_state_t *iss)
717 {
718 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr);
719 }
720
721 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)722 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
723 {
724 if (is_saved_state32(iss)) {
725 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
726 } else {
727 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
728 MANIPULATE_SIGNED_THREAD_STATE(iss,
729 "mov x3, %[lr] \n"
730 "str x3, [x0, %[SS64_LR]] \n",
731 [lr] "r"(lr)
732 );
733 #else
734 saved_state64(iss)->lr = (unsigned long)lr;
735 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
736 }
737 }
738
739 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)740 get_saved_state_fp(const arm_saved_state_t *iss)
741 {
742 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp);
743 }
744
745 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)746 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
747 {
748 if (is_saved_state32(iss)) {
749 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
750 } else {
751 saved_state64(iss)->fp = (uint64_t)fp;
752 }
753 }
754
755 static inline int
check_saved_state_reglimit(const arm_saved_state_t * iss,unsigned reg)756 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
757 {
758 return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
759 }
760
761 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned reg)762 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
763 {
764 if (!check_saved_state_reglimit(iss, reg)) {
765 return 0;
766 }
767
768 return (register_t)(is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]));
769 }
770
771 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned reg,register_t value)772 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
773 {
774 if (!check_saved_state_reglimit(iss, reg)) {
775 return;
776 }
777
778 if (is_saved_state32(iss)) {
779 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
780 } else {
781 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
782 /* x16 and x17 are part of the jophash */
783 if (reg == 16) {
784 MANIPULATE_SIGNED_THREAD_STATE(iss,
785 "mov x4, %[value] \n"
786 "str x4, [x0, %[SS64_X16]] \n",
787 [value] "r"(value)
788 );
789 return;
790 } else if (reg == 17) {
791 MANIPULATE_SIGNED_THREAD_STATE(iss,
792 "mov x5, %[value] \n"
793 "str x5, [x0, %[SS64_X17]] \n",
794 [value] "r"(value),
795 [SS64_X17] "i"(ss64_offsetof(x[17]))
796 );
797 return;
798 }
799 #endif
800 saved_state64(iss)->x[reg] = (uint64_t)value;
801 }
802 }
803
804
805 static inline uint32_t
get_saved_state_cpsr(const arm_saved_state_t * iss)806 get_saved_state_cpsr(const arm_saved_state_t *iss)
807 {
808 return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
809 }
810
811 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)812 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
813 {
814 if (is_saved_state32(iss)) {
815 saved_state32(iss)->cpsr |= set_bits;
816 saved_state32(iss)->cpsr &= ~clear_bits;
817 } else {
818 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
819 MANIPULATE_SIGNED_THREAD_STATE(iss,
820 "mov w6, %w[set_bits] \n"
821 "orr w2, w2, w6, lsl #0 \n"
822 "mov w6, %w[clear_bits] \n"
823 "bic w2, w2, w6, lsl #0 \n"
824 "str w2, [x0, %[SS64_CPSR]] \n",
825 [set_bits] "r"(set_bits),
826 [clear_bits] "r"(clear_bits)
827 );
828 #else
829 saved_state64(iss)->cpsr |= set_bits;
830 saved_state64(iss)->cpsr &= ~clear_bits;
831 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
832 }
833 }
834
835 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,uint32_t cpsr)836 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
837 {
838 if (is_saved_state32(iss)) {
839 saved_state32(iss)->cpsr = cpsr;
840 } else {
841 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
842 MANIPULATE_SIGNED_THREAD_STATE(iss,
843 "mov w2, %w[cpsr] \n"
844 "str w2, [x0, %[SS64_CPSR]] \n",
845 [cpsr] "r"(cpsr)
846 );
847 #else
848 saved_state64(iss)->cpsr = cpsr;
849 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
850 }
851 }
852
853 static inline register_t
get_saved_state_far(const arm_saved_state_t * iss)854 get_saved_state_far(const arm_saved_state_t *iss)
855 {
856 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far);
857 }
858
859 static inline void
set_saved_state_far(arm_saved_state_t * iss,register_t far)860 set_saved_state_far(arm_saved_state_t *iss, register_t far)
861 {
862 if (is_saved_state32(iss)) {
863 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
864 } else {
865 saved_state64(iss)->far = (uint64_t)far;
866 }
867 }
868
869 static inline uint32_t
get_saved_state_esr(const arm_saved_state_t * iss)870 get_saved_state_esr(const arm_saved_state_t *iss)
871 {
872 return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
873 }
874
875 static inline void
set_saved_state_esr(arm_saved_state_t * iss,uint32_t esr)876 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
877 {
878 if (is_saved_state32(iss)) {
879 saved_state32(iss)->esr = esr;
880 } else {
881 saved_state64(iss)->esr = esr;
882 }
883 }
884
885 static inline uint32_t
get_saved_state_exc(const arm_saved_state_t * iss)886 get_saved_state_exc(const arm_saved_state_t *iss)
887 {
888 return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
889 }
890
891 static inline void
set_saved_state_exc(arm_saved_state_t * iss,uint32_t exc)892 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
893 {
894 if (is_saved_state32(iss)) {
895 saved_state32(iss)->exception = exc;
896 } else {
897 saved_state64(iss)->exception = exc;
898 }
899 }
900
901 extern void panic_unimplemented(void);
902
903 /**
904 * Extracts the SVC (Supervisor Call) number from the appropriate GPR (General
905 * Purpose Register).
906 *
907 * @param[in] iss the 32-bit or 64-bit ARM saved state (i.e. trap frame).
908 *
909 * @return The SVC number.
910 */
911 static inline int
get_saved_state_svc_number(const arm_saved_state_t * iss)912 get_saved_state_svc_number(const arm_saved_state_t *iss)
913 {
914 return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
915 }
916
917 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
918
919 struct arm_debug_aggregate_state {
920 arm_state_hdr_t dsh;
921 union {
922 arm_debug_state32_t ds32;
923 arm_debug_state64_t ds64;
924 } uds;
925 } __attribute__((aligned(16)));
926
927 typedef struct arm_debug_aggregate_state arm_debug_state_t;
928
929 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
930 (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
931
932 /*
933 * NEON context
934 */
935 typedef __uint128_t uint128_t;
936 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
937 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
938
939 struct arm_neon_saved_state32 {
940 union {
941 uint128_t q[16];
942 uint64_t d[32];
943 uint32_t s[32];
944 } v;
945 uint32_t fpsr;
946 uint32_t fpcr;
947 };
948 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
949
950 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
951 (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
952
953 struct arm_neon_saved_state64 {
954 union {
955 uint128_t q[32];
956 uint64x2_t d[32];
957 uint32x4_t s[32];
958 } v;
959 uint32_t fpsr;
960 uint32_t fpcr;
961 };
962 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
963
964 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
965 (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
966
967 struct arm_neon_saved_state {
968 arm_state_hdr_t nsh;
969 union {
970 struct arm_neon_saved_state32 ns_32;
971 struct arm_neon_saved_state64 ns_64;
972 } uns;
973 };
974 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
975 #define ns_32 uns.ns_32
976 #define ns_64 uns.ns_64
977
978 struct arm_kernel_neon_saved_state {
979 uint64_t d[8];
980 uint32_t fpcr;
981 };
982 typedef struct arm_kernel_neon_saved_state arm_kernel_neon_saved_state_t;
983
984 static inline boolean_t
is_neon_saved_state32(const arm_neon_saved_state_t * state)985 is_neon_saved_state32(const arm_neon_saved_state_t *state)
986 {
987 return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
988 }
989
990 static inline boolean_t
is_neon_saved_state64(const arm_neon_saved_state_t * state)991 is_neon_saved_state64(const arm_neon_saved_state_t *state)
992 {
993 return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
994 }
995
996 static inline arm_neon_saved_state32_t *
neon_state32(arm_neon_saved_state_t * state)997 neon_state32(arm_neon_saved_state_t *state)
998 {
999 return &state->ns_32;
1000 }
1001
1002 static inline arm_neon_saved_state64_t *
neon_state64(arm_neon_saved_state_t * state)1003 neon_state64(arm_neon_saved_state_t *state)
1004 {
1005 return &state->ns_64;
1006 }
1007
1008
1009 /*
1010 * Aggregated context
1011 */
1012
1013 struct arm_context {
1014 struct arm_saved_state ss;
1015 struct arm_neon_saved_state ns;
1016 };
1017 typedef struct arm_context arm_context_t;
1018
1019 struct arm_kernel_context {
1020 struct arm_kernel_saved_state ss;
1021 struct arm_kernel_neon_saved_state ns;
1022 };
1023 typedef struct arm_kernel_context arm_kernel_context_t;
1024
1025 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
1026 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
1027
1028 #else /* defined(__arm__) */
1029 #error Unknown arch
1030 #endif /* defined(__arm__) */
1031
1032 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
1033 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
1034
1035 #endif /* XNU_KERNEL_PRIVATE */
1036
1037 #endif /* defined (__arm__) || defined (__arm64__) */
1038
1039 #endif /* _ARM_THREAD_STATUS_H_ */
1040