1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * FILE_ID: thread_status.h
30 */
31
32
33 #ifndef _ARM_THREAD_STATUS_H_
34 #define _ARM_THREAD_STATUS_H_
35
36 #if defined (__arm__) || defined (__arm64__)
37
38 #include <mach/machine/_structs.h>
39 #include <mach/machine/thread_state.h>
40 #include <mach/message.h>
41 #include <mach/vm_types.h>
42
43 #ifdef XNU_KERNEL_PRIVATE
44 #include <os/refcnt.h>
45 #endif
46
47 /*
48 * Support for determining the state of a thread
49 */
50
51
52 /*
53 * Flavors
54 */
55
56 #define ARM_THREAD_STATE 1
57 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
58 #define ARM_VFP_STATE 2
59 #define ARM_EXCEPTION_STATE 3
60 #define ARM_DEBUG_STATE 4 /* pre-armv8 */
61 #define THREAD_STATE_NONE 5
62 #define ARM_THREAD_STATE64 6
63 #define ARM_EXCEPTION_STATE64 7
64 // ARM_THREAD_STATE_LAST 8 /* legacy */
65 #define ARM_THREAD_STATE32 9
66
67 #ifdef XNU_KERNEL_PRIVATE
68 #define X86_THREAD_STATE_NONE 13 /* i386/thread_status.h THREAD_STATE_NONE */
69 #endif /* XNU_KERNEL_PRIVATE */
70
71 /* API */
72 #define ARM_DEBUG_STATE32 14
73 #define ARM_DEBUG_STATE64 15
74 #define ARM_NEON_STATE 16
75 #define ARM_NEON_STATE64 17
76 #define ARM_CPMU_STATE64 18
77
78 #ifdef XNU_KERNEL_PRIVATE
79 /* For kernel use */
80 #define ARM_SAVED_STATE32 20
81 #define ARM_SAVED_STATE64 21
82 #define ARM_NEON_SAVED_STATE32 22
83 #define ARM_NEON_SAVED_STATE64 23
84 #endif /* XNU_KERNEL_PRIVATE */
85
86 #define ARM_PAGEIN_STATE 27
87
88
89 #ifndef ARM_STATE_FLAVOR_IS_OTHER_VALID
90 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
91 #endif
92
93 #define FLAVOR_MODIFIES_CORE_CPU_REGISTERS(x) \
94 ((x == ARM_THREAD_STATE) || \
95 (x == ARM_THREAD_STATE32) || \
96 (x == ARM_THREAD_STATE64))
97
98 #define VALID_THREAD_STATE_FLAVOR(x) \
99 ((x == ARM_THREAD_STATE) || \
100 (x == ARM_VFP_STATE) || \
101 (x == ARM_EXCEPTION_STATE) || \
102 (x == ARM_DEBUG_STATE) || \
103 (x == THREAD_STATE_NONE) || \
104 (x == ARM_THREAD_STATE32) || \
105 (x == ARM_THREAD_STATE64) || \
106 (x == ARM_EXCEPTION_STATE64) || \
107 (x == ARM_NEON_STATE) || \
108 (x == ARM_NEON_STATE64) || \
109 (x == ARM_DEBUG_STATE32) || \
110 (x == ARM_DEBUG_STATE64) || \
111 (x == ARM_PAGEIN_STATE) || \
112 (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
113
114 struct arm_state_hdr {
115 uint32_t flavor;
116 uint32_t count;
117 };
118 typedef struct arm_state_hdr arm_state_hdr_t;
119
120 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t;
121 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t;
122 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
123
124 #if !defined(KERNEL)
125 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
126
127 /* Accessor macros for arm_thread_state64_t pointer fields */
128
129 /* Return pc field of arm_thread_state64_t as a data pointer value */
130 #define arm_thread_state64_get_pc(ts) \
131 __darwin_arm_thread_state64_get_pc(ts)
132 /* Return pc field of arm_thread_state64_t as a function pointer. May return
133 * NULL if a valid function pointer cannot be constructed, the caller should
134 * fall back to the arm_thread_state64_get_pc() macro in that case. */
135 #define arm_thread_state64_get_pc_fptr(ts) \
136 __darwin_arm_thread_state64_get_pc_fptr(ts)
137 /* Set pc field of arm_thread_state64_t to a function pointer */
138 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
139 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
140 /* Return lr field of arm_thread_state64_t as a data pointer value */
141 #define arm_thread_state64_get_lr(ts) \
142 __darwin_arm_thread_state64_get_lr(ts)
143 /* Return lr field of arm_thread_state64_t as a function pointer. May return
144 * NULL if a valid function pointer cannot be constructed, the caller should
145 * fall back to the arm_thread_state64_get_lr() macro in that case. */
146 #define arm_thread_state64_get_lr_fptr(ts) \
147 __darwin_arm_thread_state64_get_lr_fptr(ts)
148 /* Set lr field of arm_thread_state64_t to a function pointer */
149 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
150 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
151 /* Return sp field of arm_thread_state64_t as a data pointer value */
152 #define arm_thread_state64_get_sp(ts) \
153 __darwin_arm_thread_state64_get_sp(ts)
154 /* Set sp field of arm_thread_state64_t to a data pointer value */
155 #define arm_thread_state64_set_sp(ts, ptr) \
156 __darwin_arm_thread_state64_set_sp(ts, ptr)
157 /* Return fp field of arm_thread_state64_t as a data pointer value */
158 #define arm_thread_state64_get_fp(ts) \
159 __darwin_arm_thread_state64_get_fp(ts)
160 /* Set fp field of arm_thread_state64_t to a data pointer value */
161 #define arm_thread_state64_set_fp(ts, ptr) \
162 __darwin_arm_thread_state64_set_fp(ts, ptr)
163 /* Strip ptr auth bits from pc, lr, sp and fp field of arm_thread_state64_t */
164 #define arm_thread_state64_ptrauth_strip(ts) \
165 __darwin_arm_thread_state64_ptrauth_strip(ts)
166
167 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
168 #endif /* !defined(KERNEL) */
169
170 struct arm_unified_thread_state {
171 arm_state_hdr_t ash;
172 union {
173 arm_thread_state32_t ts_32;
174 arm_thread_state64_t ts_64;
175 } uts;
176 };
177 #define ts_32 uts.ts_32
178 #define ts_64 uts.ts_64
179 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
180
181 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
182 (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
183 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
184 (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
185 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
186 (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
187 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
188 (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
189
190
191 typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t;
192 typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t;
193 typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t;
194 typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t;
195
196
197 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t;
198 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t;
199 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
200
201 typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t;
202 typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t;
203
204 typedef _STRUCT_ARM_PAGEIN_STATE arm_pagein_state_t;
205
206 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
207 /* See below for ARM64 kernel structure definition for arm_debug_state. */
208 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
209 /*
210 * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
211 * arm_debug_state for binary compatability of userland consumers of this file.
212 */
213 #if defined(__arm__)
214 typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t;
215 #elif defined(__arm64__)
216 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
217 #else /* defined(__arm__) */
218 #error Undefined architecture
219 #endif /* defined(__arm__) */
220 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
221
222 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
223 (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
224
225 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
226 (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
227
228 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
229 (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
230
231 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
232 (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
233
234 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
235 (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
236
237 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
238 (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
239
240 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
241 (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
242
243 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
244 (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
245
246 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
247 (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
248
249 #define MACHINE_THREAD_STATE ARM_THREAD_STATE
250 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
251
252
253 /*
254 * Largest state on this machine:
255 */
256 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
257
258 #ifdef XNU_KERNEL_PRIVATE
259
260 static inline boolean_t
is_thread_state32(const arm_unified_thread_state_t * its)261 is_thread_state32(const arm_unified_thread_state_t *its)
262 {
263 return its->ash.flavor == ARM_THREAD_STATE32;
264 }
265
266 static inline boolean_t
is_thread_state64(const arm_unified_thread_state_t * its)267 is_thread_state64(const arm_unified_thread_state_t *its)
268 {
269 return its->ash.flavor == ARM_THREAD_STATE64;
270 }
271
272 static inline arm_thread_state32_t*
thread_state32(arm_unified_thread_state_t * its)273 thread_state32(arm_unified_thread_state_t *its)
274 {
275 return &its->ts_32;
276 }
277
278 static inline arm_thread_state64_t*
thread_state64(arm_unified_thread_state_t * its)279 thread_state64(arm_unified_thread_state_t *its)
280 {
281 return &its->ts_64;
282 }
283
284 static inline const arm_thread_state32_t*
const_thread_state32(const arm_unified_thread_state_t * its)285 const_thread_state32(const arm_unified_thread_state_t *its)
286 {
287 return &its->ts_32;
288 }
289
290 static inline const arm_thread_state64_t*
const_thread_state64(const arm_unified_thread_state_t * its)291 const_thread_state64(const arm_unified_thread_state_t *its)
292 {
293 return &its->ts_64;
294 }
295
296 #if defined(__arm__)
297 #include <arm64/proc_reg.h>
298
299 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
300
301 #if __ARM_VFP__
302 #define VFPSAVE_ALIGN 16
303 #define VFPSAVE_ATTRIB __attribute__((aligned (VFPSAVE_ALIGN)))
304 #define THREAD_ALIGN VFPSAVE_ALIGN
305
306 /*
307 * vector floating point saved state
308 */
309 struct arm_vfpsaved_state {
310 uint32_t r[64];
311 uint32_t fpscr;
312 uint32_t fpexc;
313 };
314 #endif
315
316 struct arm_saved_state {
317 uint32_t r[13]; /* General purpose register r0-r12 */
318 uint32_t sp; /* Stack pointer r13 */
319 uint32_t lr; /* Link register r14 */
320 uint32_t pc; /* Program counter r15 */
321 uint32_t cpsr; /* Current program status register */
322 uint32_t fsr; /* Fault status */
323 uint32_t far; /* Virtual Fault Address */
324 uint32_t exception; /* exception number */
325
326 #if __ARM_VFP__
327 /* VFP state */
328 struct arm_vfpsaved_state VFPdata VFPSAVE_ATTRIB;
329 // for packing reasons chtread_self and DebugData
330 // are inside the the PcbData when __ARM_VFP__ is set
331 arm_debug_state_t *VFPpadding_DebugData;
332 vm_address_t VFPpadding_cthread_self;
333 #endif
334 };
335 typedef struct arm_saved_state arm_saved_state_t;
336
337 /*
338 * Just for coexistence with AArch64 code.
339 */
340 typedef struct arm_saved_state arm_saved_state32_t;
341
342 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)343 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
344 {
345 *dst = *src;
346 }
347
348 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)349 saved_state32(arm_saved_state_t *iss)
350 {
351 return iss;
352 }
353
354 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss __unused)355 is_saved_state32(const arm_saved_state_t *iss __unused)
356 {
357 return TRUE;
358 }
359
360
361 struct arm_saved_state_tagged {
362 uint32_t tag;
363 struct arm_saved_state state;
364 };
365 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
366
367 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
368 (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
369
370
371 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)372 get_saved_state_pc(const arm_saved_state_t *iss)
373 {
374 return iss->pc;
375 }
376
377 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)378 add_saved_state_pc(arm_saved_state_t *iss, int diff)
379 {
380 iss->pc += (unsigned int)diff;
381 }
382
383 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)384 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
385 {
386 iss->pc = (typeof(iss->pc))pc;
387 }
388
389 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)390 get_saved_state_sp(const arm_saved_state_t *iss)
391 {
392 return iss->sp;
393 }
394
395 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)396 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
397 {
398 iss->sp = sp;
399 }
400
401 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)402 get_saved_state_fp(const arm_saved_state_t *iss)
403 {
404 return iss->r[7];
405 }
406
407 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)408 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
409 {
410 iss->r[7] = fp;
411 }
412
413 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)414 get_saved_state_lr(const arm_saved_state_t *iss)
415 {
416 return (register_t)iss->lr;
417 }
418
419 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)420 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
421 {
422 iss->lr = lr;
423 }
424
425 static inline register_t
get_saved_state_cpsr(const arm_saved_state_t * iss)426 get_saved_state_cpsr(const arm_saved_state_t *iss)
427 {
428 return iss->cpsr;
429 }
430
431 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)432 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
433 {
434 iss->cpsr |= set_bits;
435 iss->cpsr &= ~clear_bits;
436 }
437
438 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,register_t cpsr)439 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
440 {
441 iss->cpsr = cpsr;
442 }
443
444 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned regno)445 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
446 {
447 return iss->r[regno];
448 }
449
450 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned regno,register_t val)451 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
452 {
453 iss->r[regno] = val;
454 }
455
456 #elif defined(__arm64__)
457
458 #include <kern/assert.h>
459 #include <arm64/proc_reg.h>
460 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
461
462 /*
463 * GPR context
464 */
465
466 struct arm_saved_state32 {
467 uint32_t r[13]; /* General purpose register r0-r12 */
468 uint32_t sp; /* Stack pointer r13 */
469 uint32_t lr; /* Link register r14 */
470 uint32_t pc; /* Program counter r15 */
471 uint32_t cpsr; /* Current program status register */
472 uint32_t far; /* Virtual fault address */
473 uint32_t esr; /* Exception syndrome register */
474 uint32_t exception; /* Exception number */
475 };
476 typedef struct arm_saved_state32 arm_saved_state32_t;
477
478 struct arm_saved_state32_tagged {
479 uint32_t tag;
480 struct arm_saved_state32 state;
481 };
482 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
483
484 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
485 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
486
487 struct arm_saved_state64 {
488 uint64_t x[29]; /* General purpose registers x0-x28 */
489 uint64_t fp; /* Frame pointer x29 */
490 uint64_t lr; /* Link register x30 */
491 uint64_t sp; /* Stack pointer x31 */
492 uint64_t pc; /* Program counter */
493 uint32_t cpsr; /* Current program status register */
494 uint32_t reserved; /* Reserved padding */
495 uint64_t far; /* Virtual fault address */
496 uint32_t esr; /* Exception syndrome register */
497 uint32_t exception; /* Exception number */
498 #if defined(HAS_APPLE_PAC)
499 uint64_t jophash;
500 #endif /* defined(HAS_APPLE_PAC) */
501 };
502 typedef struct arm_saved_state64 arm_saved_state64_t;
503
504 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
505 (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
506
507 struct arm_saved_state64_tagged {
508 uint32_t tag;
509 struct arm_saved_state64 state;
510 };
511 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
512
513 struct arm_saved_state {
514 arm_state_hdr_t ash;
515 union {
516 struct arm_saved_state32 ss_32;
517 struct arm_saved_state64 ss_64;
518 } uss;
519 } __attribute__((aligned(16)));
520 #define ss_32 uss.ss_32
521 #define ss_64 uss.ss_64
522
523 typedef struct arm_saved_state arm_saved_state_t;
524
525 struct arm_kernel_saved_state {
526 uint64_t x[10]; /* General purpose registers x19-x28 */
527 uint64_t fp; /* Frame pointer x29 */
528 uint64_t lr; /* Link register x30 */
529 uint64_t sp; /* Stack pointer x31 */
530 /* Some things here we DO need to preserve */
531 uint8_t pc_was_in_userspace;
532 uint8_t ssbs;
533 uint8_t dit;
534 uint8_t uao;
535 } __attribute__((aligned(16)));
536
537 typedef struct arm_kernel_saved_state arm_kernel_saved_state_t;
538
539 #if defined(XNU_KERNEL_PRIVATE)
540 #if defined(HAS_APPLE_PAC)
541
542 #include <sys/cdefs.h>
543
544 /*
545 * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
546 * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
547 * and osfmk/arm/machine_routines.h.
548 */
549 __BEGIN_DECLS
550 extern uint64_t ml_pac_safe_interrupts_disable(void);
551 extern void ml_pac_safe_interrupts_restore(uint64_t);
552 __END_DECLS
553
554 /*
555 * Methods used to sign and check thread state to detect corruptions of saved
556 * thread state across exceptions and context switches.
557 */
558 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
559
560 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
561
562 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
563 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
564
565 /**
566 * Verify the signed thread state in _iss, execute the assembly instructions
567 * _instr, and re-sign the modified thread state. Varargs specify additional
568 * inputs.
569 *
570 * _instr may read or modify the thread state in the following registers:
571 *
572 * x0: _iss
573 * x1: authed _iss->ss_64.pc
574 * w2: authed _iss->ss_64.cpsr
575 * x3: authed _iss->ss_64.lr
576 * x4: authed _iss->ss_64.x16
577 * x5: authed _iss->ss_64.x17
578 * x6: scratch register
579 * x7: scratch register
580 */
581 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...) \
582 do { \
583 uint64_t _intr = ml_pac_safe_interrupts_disable(); \
584 asm volatile ( \
585 "mov x9, lr" "\n" \
586 "mov x0, %[iss]" "\n" \
587 "msr SPSel, #1" "\n" \
588 "ldp x4, x5, [x0, %[SS64_X16]]" "\n" \
589 "ldr x7, [x0, %[SS64_PC]]" "\n" \
590 "ldr w8, [x0, %[SS64_CPSR]]" "\n" \
591 "ldr x3, [x0, %[SS64_LR]]" "\n" \
592 "mov x1, x7" "\n" \
593 "mov w2, w8" "\n" \
594 "bl _ml_check_signed_state" "\n" \
595 "mov x1, x7" "\n" \
596 "mov w2, w8" "\n" \
597 _instr "\n" \
598 "bl _ml_sign_thread_state" "\n" \
599 "msr SPSel, #0" "\n" \
600 "mov lr, x9" "\n" \
601 : \
602 : [iss] "r"(_iss), \
603 [SS64_X16] "i"(ss64_offsetof(x[16])), \
604 [SS64_PC] "i"(ss64_offsetof(pc)), \
605 [SS64_CPSR] "i"(ss64_offsetof(cpsr)), \
606 [SS64_LR] "i"(ss64_offsetof(lr)),##__VA_ARGS__ \
607 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", \
608 "x9" \
609 ); \
610 ml_pac_safe_interrupts_restore(_intr); \
611 } while (0)
612
613 static inline void
check_and_sign_copied_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)614 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
615 {
616 MANIPULATE_SIGNED_THREAD_STATE(src,
617 "mov x0, %[dst]",
618 [dst] "r"(dst)
619 );
620 }
621 #endif /* defined(HAS_APPLE_PAC) */
622
623 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)624 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
625 {
626 *dst = *src;
627 #if defined(HAS_APPLE_PAC)
628 check_and_sign_copied_thread_state(dst, src);
629 #endif
630 }
631
632 #endif /* defined(XNU_KERNEL_PRIVATE) */
633
634 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss)635 is_saved_state32(const arm_saved_state_t *iss)
636 {
637 return iss->ash.flavor == ARM_SAVED_STATE32;
638 }
639
640 static inline boolean_t
is_saved_state64(const arm_saved_state_t * iss)641 is_saved_state64(const arm_saved_state_t *iss)
642 {
643 return iss->ash.flavor == ARM_SAVED_STATE64;
644 }
645
646 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)647 saved_state32(arm_saved_state_t *iss)
648 {
649 return &iss->ss_32;
650 }
651
652 static inline const arm_saved_state32_t*
const_saved_state32(const arm_saved_state_t * iss)653 const_saved_state32(const arm_saved_state_t *iss)
654 {
655 return &iss->ss_32;
656 }
657
658 static inline arm_saved_state64_t*
saved_state64(arm_saved_state_t * iss)659 saved_state64(arm_saved_state_t *iss)
660 {
661 return &iss->ss_64;
662 }
663
664 static inline const arm_saved_state64_t*
const_saved_state64(const arm_saved_state_t * iss)665 const_saved_state64(const arm_saved_state_t *iss)
666 {
667 return &iss->ss_64;
668 }
669
670 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)671 get_saved_state_pc(const arm_saved_state_t *iss)
672 {
673 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc);
674 }
675
676 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)677 add_saved_state_pc(arm_saved_state_t *iss, int diff)
678 {
679 if (is_saved_state32(iss)) {
680 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
681 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
682 } else {
683 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
684 MANIPULATE_SIGNED_THREAD_STATE(iss,
685 "mov w6, %w[diff] \n"
686 "add x1, x1, w6, sxtw \n"
687 "str x1, [x0, %[SS64_PC]] \n",
688 [diff] "r"(diff)
689 );
690 #else
691 saved_state64(iss)->pc += (unsigned long)diff;
692 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
693 }
694 }
695
696 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)697 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
698 {
699 if (is_saved_state32(iss)) {
700 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
701 } else {
702 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
703 MANIPULATE_SIGNED_THREAD_STATE(iss,
704 "mov x1, %[pc] \n"
705 "str x1, [x0, %[SS64_PC]] \n",
706 [pc] "r"(pc)
707 );
708 #else
709 saved_state64(iss)->pc = (unsigned long)pc;
710 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
711 }
712 }
713
714 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)715 get_saved_state_sp(const arm_saved_state_t *iss)
716 {
717 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp);
718 }
719
720 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)721 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
722 {
723 if (is_saved_state32(iss)) {
724 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
725 } else {
726 saved_state64(iss)->sp = (uint64_t)sp;
727 }
728 }
729
730 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)731 get_saved_state_lr(const arm_saved_state_t *iss)
732 {
733 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr);
734 }
735
736 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)737 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
738 {
739 if (is_saved_state32(iss)) {
740 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
741 } else {
742 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
743 MANIPULATE_SIGNED_THREAD_STATE(iss,
744 "mov x3, %[lr] \n"
745 "str x3, [x0, %[SS64_LR]] \n",
746 [lr] "r"(lr)
747 );
748 #else
749 saved_state64(iss)->lr = (unsigned long)lr;
750 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
751 }
752 }
753
754 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)755 get_saved_state_fp(const arm_saved_state_t *iss)
756 {
757 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp);
758 }
759
760 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)761 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
762 {
763 if (is_saved_state32(iss)) {
764 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
765 } else {
766 saved_state64(iss)->fp = (uint64_t)fp;
767 }
768 }
769
770 static inline int
check_saved_state_reglimit(const arm_saved_state_t * iss,unsigned reg)771 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
772 {
773 return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
774 }
775
776 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned reg)777 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
778 {
779 if (!check_saved_state_reglimit(iss, reg)) {
780 return 0;
781 }
782
783 return (register_t)(is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]));
784 }
785
786 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned reg,register_t value)787 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
788 {
789 if (!check_saved_state_reglimit(iss, reg)) {
790 return;
791 }
792
793 if (is_saved_state32(iss)) {
794 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
795 } else {
796 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
797 /* x16 and x17 are part of the jophash */
798 if (reg == 16) {
799 MANIPULATE_SIGNED_THREAD_STATE(iss,
800 "mov x4, %[value] \n"
801 "str x4, [x0, %[SS64_X16]] \n",
802 [value] "r"(value)
803 );
804 return;
805 } else if (reg == 17) {
806 MANIPULATE_SIGNED_THREAD_STATE(iss,
807 "mov x5, %[value] \n"
808 "str x5, [x0, %[SS64_X17]] \n",
809 [value] "r"(value),
810 [SS64_X17] "i"(ss64_offsetof(x[17]))
811 );
812 return;
813 }
814 #endif
815 saved_state64(iss)->x[reg] = (uint64_t)value;
816 }
817 }
818
819
820 static inline uint32_t
get_saved_state_cpsr(const arm_saved_state_t * iss)821 get_saved_state_cpsr(const arm_saved_state_t *iss)
822 {
823 return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
824 }
825
826 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)827 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
828 {
829 if (is_saved_state32(iss)) {
830 saved_state32(iss)->cpsr |= set_bits;
831 saved_state32(iss)->cpsr &= ~clear_bits;
832 } else {
833 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
834 MANIPULATE_SIGNED_THREAD_STATE(iss,
835 "mov w6, %w[set_bits] \n"
836 "orr w2, w2, w6, lsl #0 \n"
837 "mov w6, %w[clear_bits] \n"
838 "bic w2, w2, w6, lsl #0 \n"
839 "str w2, [x0, %[SS64_CPSR]] \n",
840 [set_bits] "r"(set_bits),
841 [clear_bits] "r"(clear_bits)
842 );
843 #else
844 saved_state64(iss)->cpsr |= set_bits;
845 saved_state64(iss)->cpsr &= ~clear_bits;
846 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
847 }
848 }
849
850 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,uint32_t cpsr)851 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
852 {
853 if (is_saved_state32(iss)) {
854 saved_state32(iss)->cpsr = cpsr;
855 } else {
856 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
857 MANIPULATE_SIGNED_THREAD_STATE(iss,
858 "mov w2, %w[cpsr] \n"
859 "str w2, [x0, %[SS64_CPSR]] \n",
860 [cpsr] "r"(cpsr)
861 );
862 #else
863 saved_state64(iss)->cpsr = cpsr;
864 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
865 }
866 }
867
868 static inline register_t
get_saved_state_far(const arm_saved_state_t * iss)869 get_saved_state_far(const arm_saved_state_t *iss)
870 {
871 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far);
872 }
873
874 static inline void
set_saved_state_far(arm_saved_state_t * iss,register_t far)875 set_saved_state_far(arm_saved_state_t *iss, register_t far)
876 {
877 if (is_saved_state32(iss)) {
878 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
879 } else {
880 saved_state64(iss)->far = (uint64_t)far;
881 }
882 }
883
884 static inline uint32_t
get_saved_state_esr(const arm_saved_state_t * iss)885 get_saved_state_esr(const arm_saved_state_t *iss)
886 {
887 return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
888 }
889
890 static inline void
set_saved_state_esr(arm_saved_state_t * iss,uint32_t esr)891 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
892 {
893 if (is_saved_state32(iss)) {
894 saved_state32(iss)->esr = esr;
895 } else {
896 saved_state64(iss)->esr = esr;
897 }
898 }
899
900 static inline uint32_t
get_saved_state_exc(const arm_saved_state_t * iss)901 get_saved_state_exc(const arm_saved_state_t *iss)
902 {
903 return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
904 }
905
906 static inline void
set_saved_state_exc(arm_saved_state_t * iss,uint32_t exc)907 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
908 {
909 if (is_saved_state32(iss)) {
910 saved_state32(iss)->exception = exc;
911 } else {
912 saved_state64(iss)->exception = exc;
913 }
914 }
915
916 extern void panic_unimplemented(void);
917
918 /**
919 * Extracts the SVC (Supervisor Call) number from the appropriate GPR (General
920 * Purpose Register).
921 *
922 * @param[in] iss the 32-bit or 64-bit ARM saved state (i.e. trap frame).
923 *
924 * @return The SVC number.
925 */
926 static inline int
get_saved_state_svc_number(const arm_saved_state_t * iss)927 get_saved_state_svc_number(const arm_saved_state_t *iss)
928 {
929 return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
930 }
931
932 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
933
934 struct arm_debug_aggregate_state {
935 arm_state_hdr_t dsh;
936 union {
937 arm_debug_state32_t ds32;
938 arm_debug_state64_t ds64;
939 } uds;
940 os_refcnt_t ref;
941 } __attribute__((aligned(16)));
942
943 typedef struct arm_debug_aggregate_state arm_debug_state_t;
944
945 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
946 (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
947
948 /*
949 * NEON context
950 */
951 typedef __uint128_t uint128_t;
952 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
953 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
954
955 struct arm_neon_saved_state32 {
956 union {
957 uint128_t q[16];
958 uint64_t d[32];
959 uint32_t s[32];
960 } v;
961 uint32_t fpsr;
962 uint32_t fpcr;
963 };
964 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
965
966 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
967 (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
968
969 struct arm_neon_saved_state64 {
970 union {
971 uint128_t q[32];
972 uint64x2_t d[32];
973 uint32x4_t s[32];
974 } v;
975 uint32_t fpsr;
976 uint32_t fpcr;
977 };
978 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
979
980 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
981 (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
982
983 struct arm_neon_saved_state {
984 arm_state_hdr_t nsh;
985 union {
986 struct arm_neon_saved_state32 ns_32;
987 struct arm_neon_saved_state64 ns_64;
988 } uns;
989 };
990 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
991 #define ns_32 uns.ns_32
992 #define ns_64 uns.ns_64
993
994 struct arm_kernel_neon_saved_state {
995 uint64_t d[8];
996 uint32_t fpcr;
997 };
998 typedef struct arm_kernel_neon_saved_state arm_kernel_neon_saved_state_t;
999
1000 static inline boolean_t
is_neon_saved_state32(const arm_neon_saved_state_t * state)1001 is_neon_saved_state32(const arm_neon_saved_state_t *state)
1002 {
1003 return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
1004 }
1005
1006 static inline boolean_t
is_neon_saved_state64(const arm_neon_saved_state_t * state)1007 is_neon_saved_state64(const arm_neon_saved_state_t *state)
1008 {
1009 return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
1010 }
1011
1012 static inline arm_neon_saved_state32_t *
neon_state32(arm_neon_saved_state_t * state)1013 neon_state32(arm_neon_saved_state_t *state)
1014 {
1015 return &state->ns_32;
1016 }
1017
1018 static inline arm_neon_saved_state64_t *
neon_state64(arm_neon_saved_state_t * state)1019 neon_state64(arm_neon_saved_state_t *state)
1020 {
1021 return &state->ns_64;
1022 }
1023
1024
1025
1026 /*
1027 * Aggregated context
1028 */
1029
1030 struct arm_context {
1031 struct arm_saved_state ss;
1032 struct arm_neon_saved_state ns;
1033 };
1034 typedef struct arm_context arm_context_t;
1035
1036 struct arm_kernel_context {
1037 struct arm_kernel_saved_state ss;
1038 struct arm_kernel_neon_saved_state ns;
1039 };
1040 typedef struct arm_kernel_context arm_kernel_context_t;
1041
1042 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
1043 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
1044
1045 #else /* defined(__arm__) */
1046 #error Unknown arch
1047 #endif /* defined(__arm__) */
1048
1049 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
1050 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
1051
1052 #endif /* XNU_KERNEL_PRIVATE */
1053
1054 #endif /* defined (__arm__) || defined (__arm64__) */
1055
1056 #endif /* _ARM_THREAD_STATUS_H_ */
1057