1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * FILE_ID: thread_status.h
30 */
31
32
33 #ifndef _ARM_THREAD_STATUS_H_
34 #define _ARM_THREAD_STATUS_H_
35
36 #if defined (__arm__) || defined (__arm64__)
37
38 #include <mach/machine/_structs.h>
39 #include <mach/machine/thread_state.h>
40 #include <mach/message.h>
41 #include <mach/vm_types.h>
42
43 #ifdef XNU_KERNEL_PRIVATE
44 #include <os/refcnt.h>
45 #endif
46
47 /*
48 * Support for determining the state of a thread
49 */
50
51
52 /*
53 * Flavors
54 */
55
56 #define ARM_THREAD_STATE 1
57 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
58 #define ARM_VFP_STATE 2
59 #define ARM_EXCEPTION_STATE 3
60 #define ARM_DEBUG_STATE 4 /* pre-armv8 */
61 #define THREAD_STATE_NONE 5
62 #define ARM_THREAD_STATE64 6
63 #define ARM_EXCEPTION_STATE64 7
64 // ARM_THREAD_STATE_LAST 8 /* legacy */
65 #define ARM_THREAD_STATE32 9
66
67 #ifdef XNU_KERNEL_PRIVATE
68 #define X86_THREAD_STATE_NONE 13 /* i386/thread_status.h THREAD_STATE_NONE */
69 #endif /* XNU_KERNEL_PRIVATE */
70
71 /* API */
72 #define ARM_DEBUG_STATE32 14
73 #define ARM_DEBUG_STATE64 15
74 #define ARM_NEON_STATE 16
75 #define ARM_NEON_STATE64 17
76 #define ARM_CPMU_STATE64 18
77
78 #ifdef XNU_KERNEL_PRIVATE
79 /* For kernel use */
80 #define ARM_SAVED_STATE32 20
81 #define ARM_SAVED_STATE64 21
82 #define ARM_NEON_SAVED_STATE32 22
83 #define ARM_NEON_SAVED_STATE64 23
84 #endif /* XNU_KERNEL_PRIVATE */
85
86 #define ARM_PAGEIN_STATE 27
87
88 #ifndef ARM_STATE_FLAVOR_IS_OTHER_VALID
89 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
90 #endif
91
92 #define VALID_THREAD_STATE_FLAVOR(x) \
93 ((x == ARM_THREAD_STATE) || \
94 (x == ARM_VFP_STATE) || \
95 (x == ARM_EXCEPTION_STATE) || \
96 (x == ARM_DEBUG_STATE) || \
97 (x == THREAD_STATE_NONE) || \
98 (x == ARM_THREAD_STATE32) || \
99 (x == ARM_THREAD_STATE64) || \
100 (x == ARM_EXCEPTION_STATE64) || \
101 (x == ARM_NEON_STATE) || \
102 (x == ARM_NEON_STATE64) || \
103 (x == ARM_DEBUG_STATE32) || \
104 (x == ARM_DEBUG_STATE64) || \
105 (x == ARM_PAGEIN_STATE) || \
106 (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
107
108 struct arm_state_hdr {
109 uint32_t flavor;
110 uint32_t count;
111 };
112 typedef struct arm_state_hdr arm_state_hdr_t;
113
114 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t;
115 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t;
116 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
117
118 #if !defined(KERNEL)
119 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
120
121 /* Accessor macros for arm_thread_state64_t pointer fields */
122
123 /* Return pc field of arm_thread_state64_t as a data pointer value */
124 #define arm_thread_state64_get_pc(ts) \
125 __darwin_arm_thread_state64_get_pc(ts)
126 /* Return pc field of arm_thread_state64_t as a function pointer. May return
127 * NULL if a valid function pointer cannot be constructed, the caller should
128 * fall back to the arm_thread_state64_get_pc() macro in that case. */
129 #define arm_thread_state64_get_pc_fptr(ts) \
130 __darwin_arm_thread_state64_get_pc_fptr(ts)
131 /* Set pc field of arm_thread_state64_t to a function pointer */
132 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
133 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
134 /* Return lr field of arm_thread_state64_t as a data pointer value */
135 #define arm_thread_state64_get_lr(ts) \
136 __darwin_arm_thread_state64_get_lr(ts)
137 /* Return lr field of arm_thread_state64_t as a function pointer. May return
138 * NULL if a valid function pointer cannot be constructed, the caller should
139 * fall back to the arm_thread_state64_get_lr() macro in that case. */
140 #define arm_thread_state64_get_lr_fptr(ts) \
141 __darwin_arm_thread_state64_get_lr_fptr(ts)
142 /* Set lr field of arm_thread_state64_t to a function pointer */
143 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
144 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
145 /* Return sp field of arm_thread_state64_t as a data pointer value */
146 #define arm_thread_state64_get_sp(ts) \
147 __darwin_arm_thread_state64_get_sp(ts)
148 /* Set sp field of arm_thread_state64_t to a data pointer value */
149 #define arm_thread_state64_set_sp(ts, ptr) \
150 __darwin_arm_thread_state64_set_sp(ts, ptr)
151 /* Return fp field of arm_thread_state64_t as a data pointer value */
152 #define arm_thread_state64_get_fp(ts) \
153 __darwin_arm_thread_state64_get_fp(ts)
154 /* Set fp field of arm_thread_state64_t to a data pointer value */
155 #define arm_thread_state64_set_fp(ts, ptr) \
156 __darwin_arm_thread_state64_set_fp(ts, ptr)
157 /* Strip ptr auth bits from pc, lr, sp and fp field of arm_thread_state64_t */
158 #define arm_thread_state64_ptrauth_strip(ts) \
159 __darwin_arm_thread_state64_ptrauth_strip(ts)
160
161 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
162 #endif /* !defined(KERNEL) */
163
164 struct arm_unified_thread_state {
165 arm_state_hdr_t ash;
166 union {
167 arm_thread_state32_t ts_32;
168 arm_thread_state64_t ts_64;
169 } uts;
170 };
171 #define ts_32 uts.ts_32
172 #define ts_64 uts.ts_64
173 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
174
175 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
176 (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
177 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
178 (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
179 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
180 (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
181 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
182 (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
183
184
185 typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t;
186 typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t;
187 typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t;
188 typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t;
189
190
191 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t;
192 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t;
193 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
194
195 typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t;
196 typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t;
197
198 typedef _STRUCT_ARM_PAGEIN_STATE arm_pagein_state_t;
199
200 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
201 /* See below for ARM64 kernel structure definition for arm_debug_state. */
202 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
203 /*
204 * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
205 * arm_debug_state for binary compatability of userland consumers of this file.
206 */
207 #if defined(__arm__)
208 typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t;
209 #elif defined(__arm64__)
210 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
211 #else /* defined(__arm__) */
212 #error Undefined architecture
213 #endif /* defined(__arm__) */
214 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
215
216 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
217 (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
218
219 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
220 (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
221
222 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
223 (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
224
225 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
226 (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
227
228 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
229 (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
230
231 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
232 (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
233
234 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
235 (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
236
237 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
238 (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
239
240 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
241 (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
242
243 #define MACHINE_THREAD_STATE ARM_THREAD_STATE
244 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
245
246
247 /*
248 * Largest state on this machine:
249 */
250 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
251
252 #ifdef XNU_KERNEL_PRIVATE
253
254 static inline boolean_t
is_thread_state32(const arm_unified_thread_state_t * its)255 is_thread_state32(const arm_unified_thread_state_t *its)
256 {
257 return its->ash.flavor == ARM_THREAD_STATE32;
258 }
259
260 static inline boolean_t
is_thread_state64(const arm_unified_thread_state_t * its)261 is_thread_state64(const arm_unified_thread_state_t *its)
262 {
263 return its->ash.flavor == ARM_THREAD_STATE64;
264 }
265
266 static inline arm_thread_state32_t*
thread_state32(arm_unified_thread_state_t * its)267 thread_state32(arm_unified_thread_state_t *its)
268 {
269 return &its->ts_32;
270 }
271
272 static inline arm_thread_state64_t*
thread_state64(arm_unified_thread_state_t * its)273 thread_state64(arm_unified_thread_state_t *its)
274 {
275 return &its->ts_64;
276 }
277
278 static inline const arm_thread_state32_t*
const_thread_state32(const arm_unified_thread_state_t * its)279 const_thread_state32(const arm_unified_thread_state_t *its)
280 {
281 return &its->ts_32;
282 }
283
284 static inline const arm_thread_state64_t*
const_thread_state64(const arm_unified_thread_state_t * its)285 const_thread_state64(const arm_unified_thread_state_t *its)
286 {
287 return &its->ts_64;
288 }
289
290 #if defined(__arm__)
291 #include <arm64/proc_reg.h>
292
293 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
294
295 #if __ARM_VFP__
296 #define VFPSAVE_ALIGN 16
297 #define VFPSAVE_ATTRIB __attribute__((aligned (VFPSAVE_ALIGN)))
298 #define THREAD_ALIGN VFPSAVE_ALIGN
299
300 /*
301 * vector floating point saved state
302 */
303 struct arm_vfpsaved_state {
304 uint32_t r[64];
305 uint32_t fpscr;
306 uint32_t fpexc;
307 };
308 #endif
309
310 struct arm_saved_state {
311 uint32_t r[13]; /* General purpose register r0-r12 */
312 uint32_t sp; /* Stack pointer r13 */
313 uint32_t lr; /* Link register r14 */
314 uint32_t pc; /* Program counter r15 */
315 uint32_t cpsr; /* Current program status register */
316 uint32_t fsr; /* Fault status */
317 uint32_t far; /* Virtual Fault Address */
318 uint32_t exception; /* exception number */
319
320 #if __ARM_VFP__
321 /* VFP state */
322 struct arm_vfpsaved_state VFPdata VFPSAVE_ATTRIB;
323 // for packing reasons chtread_self and DebugData
324 // are inside the the PcbData when __ARM_VFP__ is set
325 arm_debug_state_t *VFPpadding_DebugData;
326 vm_address_t VFPpadding_cthread_self;
327 #endif
328 };
329 typedef struct arm_saved_state arm_saved_state_t;
330
331 /*
332 * Just for coexistence with AArch64 code.
333 */
334 typedef struct arm_saved_state arm_saved_state32_t;
335
336 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)337 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
338 {
339 *dst = *src;
340 }
341
342 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)343 saved_state32(arm_saved_state_t *iss)
344 {
345 return iss;
346 }
347
348 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss __unused)349 is_saved_state32(const arm_saved_state_t *iss __unused)
350 {
351 return TRUE;
352 }
353
354
355 struct arm_saved_state_tagged {
356 uint32_t tag;
357 struct arm_saved_state state;
358 };
359 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
360
361 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
362 (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
363
364
365 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)366 get_saved_state_pc(const arm_saved_state_t *iss)
367 {
368 return iss->pc;
369 }
370
371 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)372 add_saved_state_pc(arm_saved_state_t *iss, int diff)
373 {
374 iss->pc += (unsigned int)diff;
375 }
376
377 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)378 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
379 {
380 iss->pc = (typeof(iss->pc))pc;
381 }
382
383 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)384 get_saved_state_sp(const arm_saved_state_t *iss)
385 {
386 return iss->sp;
387 }
388
389 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)390 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
391 {
392 iss->sp = sp;
393 }
394
395 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)396 get_saved_state_fp(const arm_saved_state_t *iss)
397 {
398 return iss->r[7];
399 }
400
401 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)402 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
403 {
404 iss->r[7] = fp;
405 }
406
407 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)408 get_saved_state_lr(const arm_saved_state_t *iss)
409 {
410 return (register_t)iss->lr;
411 }
412
413 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)414 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
415 {
416 iss->lr = lr;
417 }
418
419 static inline register_t
get_saved_state_cpsr(const arm_saved_state_t * iss)420 get_saved_state_cpsr(const arm_saved_state_t *iss)
421 {
422 return iss->cpsr;
423 }
424
425 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)426 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
427 {
428 iss->cpsr |= set_bits;
429 iss->cpsr &= ~clear_bits;
430 }
431
432 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,register_t cpsr)433 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
434 {
435 iss->cpsr = cpsr;
436 }
437
438 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned regno)439 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
440 {
441 return iss->r[regno];
442 }
443
444 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned regno,register_t val)445 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
446 {
447 iss->r[regno] = val;
448 }
449
450 #elif defined(__arm64__)
451
452 #include <kern/assert.h>
453 #include <arm64/proc_reg.h>
454 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
455
456 /*
457 * GPR context
458 */
459
460 struct arm_saved_state32 {
461 uint32_t r[13]; /* General purpose register r0-r12 */
462 uint32_t sp; /* Stack pointer r13 */
463 uint32_t lr; /* Link register r14 */
464 uint32_t pc; /* Program counter r15 */
465 uint32_t cpsr; /* Current program status register */
466 uint32_t far; /* Virtual fault address */
467 uint32_t esr; /* Exception syndrome register */
468 uint32_t exception; /* Exception number */
469 };
470 typedef struct arm_saved_state32 arm_saved_state32_t;
471
472 struct arm_saved_state32_tagged {
473 uint32_t tag;
474 struct arm_saved_state32 state;
475 };
476 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
477
478 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
479 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
480
481 struct arm_saved_state64 {
482 uint64_t x[29]; /* General purpose registers x0-x28 */
483 uint64_t fp; /* Frame pointer x29 */
484 uint64_t lr; /* Link register x30 */
485 uint64_t sp; /* Stack pointer x31 */
486 uint64_t pc; /* Program counter */
487 uint32_t cpsr; /* Current program status register */
488 uint32_t reserved; /* Reserved padding */
489 uint64_t far; /* Virtual fault address */
490 uint32_t esr; /* Exception syndrome register */
491 uint32_t exception; /* Exception number */
492 #if defined(HAS_APPLE_PAC)
493 uint64_t jophash;
494 #endif /* defined(HAS_APPLE_PAC) */
495 };
496 typedef struct arm_saved_state64 arm_saved_state64_t;
497
498 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
499 (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
500
501 struct arm_saved_state64_tagged {
502 uint32_t tag;
503 struct arm_saved_state64 state;
504 };
505 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
506
507 struct arm_saved_state {
508 arm_state_hdr_t ash;
509 union {
510 struct arm_saved_state32 ss_32;
511 struct arm_saved_state64 ss_64;
512 } uss;
513 } __attribute__((aligned(16)));
514 #define ss_32 uss.ss_32
515 #define ss_64 uss.ss_64
516
517 typedef struct arm_saved_state arm_saved_state_t;
518
519 struct arm_kernel_saved_state {
520 uint64_t x[10]; /* General purpose registers x19-x28 */
521 uint64_t fp; /* Frame pointer x29 */
522 uint64_t lr; /* Link register x30 */
523 uint64_t sp; /* Stack pointer x31 */
524 /* Some things here we DO need to preserve */
525 uint8_t pc_was_in_userspace;
526 uint8_t ssbs;
527 uint8_t dit;
528 uint8_t uao;
529 } __attribute__((aligned(16)));
530
531 typedef struct arm_kernel_saved_state arm_kernel_saved_state_t;
532
533 #if defined(XNU_KERNEL_PRIVATE)
534 #if defined(HAS_APPLE_PAC)
535
536 #include <sys/cdefs.h>
537
538 /*
539 * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
540 * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
541 * and osfmk/arm/machine_routines.h.
542 */
543 __BEGIN_DECLS
544 extern uint64_t ml_pac_safe_interrupts_disable(void);
545 extern void ml_pac_safe_interrupts_restore(uint64_t);
546 __END_DECLS
547
548 /*
549 * Methods used to sign and check thread state to detect corruptions of saved
550 * thread state across exceptions and context switches.
551 */
552 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
553
554 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
555
556 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
557 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
558
559 /**
560 * Verify the signed thread state in _iss, execute the assembly instructions
561 * _instr, and re-sign the modified thread state. Varargs specify additional
562 * inputs.
563 *
564 * _instr may read or modify the thread state in the following registers:
565 *
566 * x0: _iss
567 * x1: authed _iss->ss_64.pc
568 * w2: authed _iss->ss_64.cpsr
569 * x3: authed _iss->ss_64.lr
570 * x4: authed _iss->ss_64.x16
571 * x5: authed _iss->ss_64.x17
572 * x6: scratch register
573 * x7: scratch register
574 */
575 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...) \
576 do { \
577 uint64_t _intr = ml_pac_safe_interrupts_disable(); \
578 asm volatile ( \
579 "mov x8, lr" "\n" \
580 "mov x0, %[iss]" "\n" \
581 "msr SPSel, #1" "\n" \
582 "ldp x4, x5, [x0, %[SS64_X16]]" "\n" \
583 "ldr x6, [x0, %[SS64_PC]]" "\n" \
584 "ldr w7, [x0, %[SS64_CPSR]]" "\n" \
585 "ldr x3, [x0, %[SS64_LR]]" "\n" \
586 "mov x1, x6" "\n" \
587 "mov w2, w7" "\n" \
588 "bl _ml_check_signed_state" "\n" \
589 "mov x1, x6" "\n" \
590 "mov w2, w7" "\n" \
591 _instr "\n" \
592 "bl _ml_sign_thread_state" "\n" \
593 "msr SPSel, #0" "\n" \
594 "mov lr, x8" "\n" \
595 : \
596 : [iss] "r"(_iss), \
597 [SS64_X16] "i"(ss64_offsetof(x[16])), \
598 [SS64_PC] "i"(ss64_offsetof(pc)), \
599 [SS64_CPSR] "i"(ss64_offsetof(cpsr)), \
600 [SS64_LR] "i"(ss64_offsetof(lr)),##__VA_ARGS__ \
601 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8" \
602 ); \
603 ml_pac_safe_interrupts_restore(_intr); \
604 } while (0)
605
606 static inline void
check_and_sign_copied_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)607 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
608 {
609 MANIPULATE_SIGNED_THREAD_STATE(src,
610 "mov x0, %[dst]",
611 [dst] "r"(dst)
612 );
613 }
614 #endif /* defined(HAS_APPLE_PAC) */
615
616 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)617 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
618 {
619 *dst = *src;
620 #if defined(HAS_APPLE_PAC)
621 check_and_sign_copied_thread_state(dst, src);
622 #endif
623 }
624
625 #endif /* defined(XNU_KERNEL_PRIVATE) */
626
627 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss)628 is_saved_state32(const arm_saved_state_t *iss)
629 {
630 return iss->ash.flavor == ARM_SAVED_STATE32;
631 }
632
633 static inline boolean_t
is_saved_state64(const arm_saved_state_t * iss)634 is_saved_state64(const arm_saved_state_t *iss)
635 {
636 return iss->ash.flavor == ARM_SAVED_STATE64;
637 }
638
639 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)640 saved_state32(arm_saved_state_t *iss)
641 {
642 return &iss->ss_32;
643 }
644
645 static inline const arm_saved_state32_t*
const_saved_state32(const arm_saved_state_t * iss)646 const_saved_state32(const arm_saved_state_t *iss)
647 {
648 return &iss->ss_32;
649 }
650
651 static inline arm_saved_state64_t*
saved_state64(arm_saved_state_t * iss)652 saved_state64(arm_saved_state_t *iss)
653 {
654 return &iss->ss_64;
655 }
656
657 static inline const arm_saved_state64_t*
const_saved_state64(const arm_saved_state_t * iss)658 const_saved_state64(const arm_saved_state_t *iss)
659 {
660 return &iss->ss_64;
661 }
662
663 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)664 get_saved_state_pc(const arm_saved_state_t *iss)
665 {
666 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc);
667 }
668
669 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)670 add_saved_state_pc(arm_saved_state_t *iss, int diff)
671 {
672 if (is_saved_state32(iss)) {
673 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
674 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
675 } else {
676 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
677 MANIPULATE_SIGNED_THREAD_STATE(iss,
678 "mov w6, %w[diff] \n"
679 "add x1, x1, w6, sxtw \n"
680 "str x1, [x0, %[SS64_PC]] \n",
681 [diff] "r"(diff)
682 );
683 #else
684 saved_state64(iss)->pc += (unsigned long)diff;
685 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
686 }
687 }
688
689 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)690 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
691 {
692 if (is_saved_state32(iss)) {
693 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
694 } else {
695 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
696 MANIPULATE_SIGNED_THREAD_STATE(iss,
697 "mov x1, %[pc] \n"
698 "str x1, [x0, %[SS64_PC]] \n",
699 [pc] "r"(pc)
700 );
701 #else
702 saved_state64(iss)->pc = (unsigned long)pc;
703 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
704 }
705 }
706
707 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)708 get_saved_state_sp(const arm_saved_state_t *iss)
709 {
710 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp);
711 }
712
713 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)714 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
715 {
716 if (is_saved_state32(iss)) {
717 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
718 } else {
719 saved_state64(iss)->sp = (uint64_t)sp;
720 }
721 }
722
723 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)724 get_saved_state_lr(const arm_saved_state_t *iss)
725 {
726 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr);
727 }
728
729 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)730 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
731 {
732 if (is_saved_state32(iss)) {
733 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
734 } else {
735 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
736 MANIPULATE_SIGNED_THREAD_STATE(iss,
737 "mov x3, %[lr] \n"
738 "str x3, [x0, %[SS64_LR]] \n",
739 [lr] "r"(lr)
740 );
741 #else
742 saved_state64(iss)->lr = (unsigned long)lr;
743 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
744 }
745 }
746
747 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)748 get_saved_state_fp(const arm_saved_state_t *iss)
749 {
750 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp);
751 }
752
753 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)754 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
755 {
756 if (is_saved_state32(iss)) {
757 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
758 } else {
759 saved_state64(iss)->fp = (uint64_t)fp;
760 }
761 }
762
763 static inline int
check_saved_state_reglimit(const arm_saved_state_t * iss,unsigned reg)764 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
765 {
766 return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
767 }
768
769 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned reg)770 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
771 {
772 if (!check_saved_state_reglimit(iss, reg)) {
773 return 0;
774 }
775
776 return (register_t)(is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]));
777 }
778
779 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned reg,register_t value)780 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
781 {
782 if (!check_saved_state_reglimit(iss, reg)) {
783 return;
784 }
785
786 if (is_saved_state32(iss)) {
787 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
788 } else {
789 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
790 /* x16 and x17 are part of the jophash */
791 if (reg == 16) {
792 MANIPULATE_SIGNED_THREAD_STATE(iss,
793 "mov x4, %[value] \n"
794 "str x4, [x0, %[SS64_X16]] \n",
795 [value] "r"(value)
796 );
797 return;
798 } else if (reg == 17) {
799 MANIPULATE_SIGNED_THREAD_STATE(iss,
800 "mov x5, %[value] \n"
801 "str x5, [x0, %[SS64_X17]] \n",
802 [value] "r"(value),
803 [SS64_X17] "i"(ss64_offsetof(x[17]))
804 );
805 return;
806 }
807 #endif
808 saved_state64(iss)->x[reg] = (uint64_t)value;
809 }
810 }
811
812
813 static inline uint32_t
get_saved_state_cpsr(const arm_saved_state_t * iss)814 get_saved_state_cpsr(const arm_saved_state_t *iss)
815 {
816 return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
817 }
818
819 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)820 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
821 {
822 if (is_saved_state32(iss)) {
823 saved_state32(iss)->cpsr |= set_bits;
824 saved_state32(iss)->cpsr &= ~clear_bits;
825 } else {
826 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
827 MANIPULATE_SIGNED_THREAD_STATE(iss,
828 "mov w6, %w[set_bits] \n"
829 "orr w2, w2, w6, lsl #0 \n"
830 "mov w6, %w[clear_bits] \n"
831 "bic w2, w2, w6, lsl #0 \n"
832 "str w2, [x0, %[SS64_CPSR]] \n",
833 [set_bits] "r"(set_bits),
834 [clear_bits] "r"(clear_bits)
835 );
836 #else
837 saved_state64(iss)->cpsr |= set_bits;
838 saved_state64(iss)->cpsr &= ~clear_bits;
839 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
840 }
841 }
842
843 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,uint32_t cpsr)844 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
845 {
846 if (is_saved_state32(iss)) {
847 saved_state32(iss)->cpsr = cpsr;
848 } else {
849 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
850 MANIPULATE_SIGNED_THREAD_STATE(iss,
851 "mov w2, %w[cpsr] \n"
852 "str w2, [x0, %[SS64_CPSR]] \n",
853 [cpsr] "r"(cpsr)
854 );
855 #else
856 saved_state64(iss)->cpsr = cpsr;
857 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
858 }
859 }
860
861 static inline register_t
get_saved_state_far(const arm_saved_state_t * iss)862 get_saved_state_far(const arm_saved_state_t *iss)
863 {
864 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far);
865 }
866
867 static inline void
set_saved_state_far(arm_saved_state_t * iss,register_t far)868 set_saved_state_far(arm_saved_state_t *iss, register_t far)
869 {
870 if (is_saved_state32(iss)) {
871 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
872 } else {
873 saved_state64(iss)->far = (uint64_t)far;
874 }
875 }
876
877 static inline uint32_t
get_saved_state_esr(const arm_saved_state_t * iss)878 get_saved_state_esr(const arm_saved_state_t *iss)
879 {
880 return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
881 }
882
883 static inline void
set_saved_state_esr(arm_saved_state_t * iss,uint32_t esr)884 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
885 {
886 if (is_saved_state32(iss)) {
887 saved_state32(iss)->esr = esr;
888 } else {
889 saved_state64(iss)->esr = esr;
890 }
891 }
892
893 static inline uint32_t
get_saved_state_exc(const arm_saved_state_t * iss)894 get_saved_state_exc(const arm_saved_state_t *iss)
895 {
896 return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
897 }
898
899 static inline void
set_saved_state_exc(arm_saved_state_t * iss,uint32_t exc)900 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
901 {
902 if (is_saved_state32(iss)) {
903 saved_state32(iss)->exception = exc;
904 } else {
905 saved_state64(iss)->exception = exc;
906 }
907 }
908
909 extern void panic_unimplemented(void);
910
911 /**
912 * Extracts the SVC (Supervisor Call) number from the appropriate GPR (General
913 * Purpose Register).
914 *
915 * @param[in] iss the 32-bit or 64-bit ARM saved state (i.e. trap frame).
916 *
917 * @return The SVC number.
918 */
919 static inline int
get_saved_state_svc_number(const arm_saved_state_t * iss)920 get_saved_state_svc_number(const arm_saved_state_t *iss)
921 {
922 return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
923 }
924
925 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
926
927 struct arm_debug_aggregate_state {
928 arm_state_hdr_t dsh;
929 union {
930 arm_debug_state32_t ds32;
931 arm_debug_state64_t ds64;
932 } uds;
933 os_refcnt_t ref;
934 } __attribute__((aligned(16)));
935
936 typedef struct arm_debug_aggregate_state arm_debug_state_t;
937
938 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
939 (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
940
941 /*
942 * NEON context
943 */
944 typedef __uint128_t uint128_t;
945 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
946 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
947
948 struct arm_neon_saved_state32 {
949 union {
950 uint128_t q[16];
951 uint64_t d[32];
952 uint32_t s[32];
953 } v;
954 uint32_t fpsr;
955 uint32_t fpcr;
956 };
957 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
958
959 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
960 (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
961
962 struct arm_neon_saved_state64 {
963 union {
964 uint128_t q[32];
965 uint64x2_t d[32];
966 uint32x4_t s[32];
967 } v;
968 uint32_t fpsr;
969 uint32_t fpcr;
970 };
971 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
972
973 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
974 (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
975
976 struct arm_neon_saved_state {
977 arm_state_hdr_t nsh;
978 union {
979 struct arm_neon_saved_state32 ns_32;
980 struct arm_neon_saved_state64 ns_64;
981 } uns;
982 };
983 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
984 #define ns_32 uns.ns_32
985 #define ns_64 uns.ns_64
986
987 struct arm_kernel_neon_saved_state {
988 uint64_t d[8];
989 uint32_t fpcr;
990 };
991 typedef struct arm_kernel_neon_saved_state arm_kernel_neon_saved_state_t;
992
993 static inline boolean_t
is_neon_saved_state32(const arm_neon_saved_state_t * state)994 is_neon_saved_state32(const arm_neon_saved_state_t *state)
995 {
996 return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
997 }
998
999 static inline boolean_t
is_neon_saved_state64(const arm_neon_saved_state_t * state)1000 is_neon_saved_state64(const arm_neon_saved_state_t *state)
1001 {
1002 return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
1003 }
1004
1005 static inline arm_neon_saved_state32_t *
neon_state32(arm_neon_saved_state_t * state)1006 neon_state32(arm_neon_saved_state_t *state)
1007 {
1008 return &state->ns_32;
1009 }
1010
1011 static inline arm_neon_saved_state64_t *
neon_state64(arm_neon_saved_state_t * state)1012 neon_state64(arm_neon_saved_state_t *state)
1013 {
1014 return &state->ns_64;
1015 }
1016
1017
1018 /*
1019 * Aggregated context
1020 */
1021
1022 struct arm_context {
1023 struct arm_saved_state ss;
1024 struct arm_neon_saved_state ns;
1025 };
1026 typedef struct arm_context arm_context_t;
1027
1028 struct arm_kernel_context {
1029 struct arm_kernel_saved_state ss;
1030 struct arm_kernel_neon_saved_state ns;
1031 };
1032 typedef struct arm_kernel_context arm_kernel_context_t;
1033
1034 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
1035 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
1036
1037 #else /* defined(__arm__) */
1038 #error Unknown arch
1039 #endif /* defined(__arm__) */
1040
1041 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
1042 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
1043
1044 #endif /* XNU_KERNEL_PRIVATE */
1045
1046 #endif /* defined (__arm__) || defined (__arm64__) */
1047
1048 #endif /* _ARM_THREAD_STATUS_H_ */
1049