1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * FILE_ID: thread_status.h
30 */
31
32
33 #ifndef _ARM_THREAD_STATUS_H_
34 #define _ARM_THREAD_STATUS_H_
35
36 #if defined (__arm__) || defined (__arm64__)
37
38 #include <mach/machine/_structs.h>
39 #include <mach/machine/thread_state.h>
40 #include <mach/message.h>
41 #include <mach/vm_types.h>
42
43 #ifdef XNU_KERNEL_PRIVATE
44 #include <os/refcnt.h>
45 #endif
46
47 /*
48 * Support for determining the state of a thread
49 */
50
51
52 /*
53 * Flavors
54 */
55
56 #define ARM_THREAD_STATE 1
57 #define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
58 #define ARM_VFP_STATE 2
59 #define ARM_EXCEPTION_STATE 3
60 #define ARM_DEBUG_STATE 4 /* pre-armv8 */
61 #define THREAD_STATE_NONE 5
62 #define ARM_THREAD_STATE64 6
63 #define ARM_EXCEPTION_STATE64 7
64 // ARM_THREAD_STATE_LAST 8 /* legacy */
65 #define ARM_THREAD_STATE32 9
66
67 #ifdef XNU_KERNEL_PRIVATE
68 #define X86_THREAD_STATE_NONE 13 /* i386/thread_status.h THREAD_STATE_NONE */
69 #endif /* XNU_KERNEL_PRIVATE */
70
71 /* API */
72 #define ARM_DEBUG_STATE32 14
73 #define ARM_DEBUG_STATE64 15
74 #define ARM_NEON_STATE 16
75 #define ARM_NEON_STATE64 17
76 #define ARM_CPMU_STATE64 18
77
78 #ifdef XNU_KERNEL_PRIVATE
79 /* For kernel use */
80 #define ARM_SAVED_STATE32 20
81 #define ARM_SAVED_STATE64 21
82 #define ARM_NEON_SAVED_STATE32 22
83 #define ARM_NEON_SAVED_STATE64 23
84 #endif /* XNU_KERNEL_PRIVATE */
85
86 #define ARM_PAGEIN_STATE 27
87
88 #ifndef ARM_STATE_FLAVOR_IS_OTHER_VALID
89 #define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
90 #endif
91
92 #define FLAVOR_MODIFIES_CORE_CPU_REGISTERS(x) \
93 ((x == ARM_THREAD_STATE) || \
94 (x == ARM_THREAD_STATE32) || \
95 (x == ARM_THREAD_STATE64))
96
97 #define VALID_THREAD_STATE_FLAVOR(x) \
98 ((x == ARM_THREAD_STATE) || \
99 (x == ARM_VFP_STATE) || \
100 (x == ARM_EXCEPTION_STATE) || \
101 (x == ARM_DEBUG_STATE) || \
102 (x == THREAD_STATE_NONE) || \
103 (x == ARM_THREAD_STATE32) || \
104 (x == ARM_THREAD_STATE64) || \
105 (x == ARM_EXCEPTION_STATE64) || \
106 (x == ARM_NEON_STATE) || \
107 (x == ARM_NEON_STATE64) || \
108 (x == ARM_DEBUG_STATE32) || \
109 (x == ARM_DEBUG_STATE64) || \
110 (x == ARM_PAGEIN_STATE) || \
111 (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
112
113 struct arm_state_hdr {
114 uint32_t flavor;
115 uint32_t count;
116 };
117 typedef struct arm_state_hdr arm_state_hdr_t;
118
119 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t;
120 typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t;
121 typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
122
123 #if !defined(KERNEL)
124 #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
125
126 /* Accessor macros for arm_thread_state64_t pointer fields */
127
128 /* Return pc field of arm_thread_state64_t as a data pointer value */
129 #define arm_thread_state64_get_pc(ts) \
130 __darwin_arm_thread_state64_get_pc(ts)
131 /* Return pc field of arm_thread_state64_t as a function pointer. May return
132 * NULL if a valid function pointer cannot be constructed, the caller should
133 * fall back to the arm_thread_state64_get_pc() macro in that case. */
134 #define arm_thread_state64_get_pc_fptr(ts) \
135 __darwin_arm_thread_state64_get_pc_fptr(ts)
136 /* Set pc field of arm_thread_state64_t to a function pointer */
137 #define arm_thread_state64_set_pc_fptr(ts, fptr) \
138 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
139 /* Return lr field of arm_thread_state64_t as a data pointer value */
140 #define arm_thread_state64_get_lr(ts) \
141 __darwin_arm_thread_state64_get_lr(ts)
142 /* Return lr field of arm_thread_state64_t as a function pointer. May return
143 * NULL if a valid function pointer cannot be constructed, the caller should
144 * fall back to the arm_thread_state64_get_lr() macro in that case. */
145 #define arm_thread_state64_get_lr_fptr(ts) \
146 __darwin_arm_thread_state64_get_lr_fptr(ts)
147 /* Set lr field of arm_thread_state64_t to a function pointer */
148 #define arm_thread_state64_set_lr_fptr(ts, fptr) \
149 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
150 /* Return sp field of arm_thread_state64_t as a data pointer value */
151 #define arm_thread_state64_get_sp(ts) \
152 __darwin_arm_thread_state64_get_sp(ts)
153 /* Set sp field of arm_thread_state64_t to a data pointer value */
154 #define arm_thread_state64_set_sp(ts, ptr) \
155 __darwin_arm_thread_state64_set_sp(ts, ptr)
156 /* Return fp field of arm_thread_state64_t as a data pointer value */
157 #define arm_thread_state64_get_fp(ts) \
158 __darwin_arm_thread_state64_get_fp(ts)
159 /* Set fp field of arm_thread_state64_t to a data pointer value */
160 #define arm_thread_state64_set_fp(ts, ptr) \
161 __darwin_arm_thread_state64_set_fp(ts, ptr)
162 /* Strip ptr auth bits from pc, lr, sp and fp field of arm_thread_state64_t */
163 #define arm_thread_state64_ptrauth_strip(ts) \
164 __darwin_arm_thread_state64_ptrauth_strip(ts)
165
166 #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
167 #endif /* !defined(KERNEL) */
168
169 struct arm_unified_thread_state {
170 arm_state_hdr_t ash;
171 union {
172 arm_thread_state32_t ts_32;
173 arm_thread_state64_t ts_64;
174 } uts;
175 };
176 #define ts_32 uts.ts_32
177 #define ts_64 uts.ts_64
178 typedef struct arm_unified_thread_state arm_unified_thread_state_t;
179
180 #define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
181 (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
182 #define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
183 (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
184 #define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
185 (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
186 #define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
187 (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
188
189
190 typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t;
191 typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t;
192 typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t;
193 typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t;
194
195
196 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t;
197 typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t;
198 typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
199
200 typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t;
201 typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t;
202
203 typedef _STRUCT_ARM_PAGEIN_STATE arm_pagein_state_t;
204
205 #if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
206 /* See below for ARM64 kernel structure definition for arm_debug_state. */
207 #else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
208 /*
209 * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
210 * arm_debug_state for binary compatability of userland consumers of this file.
211 */
212 #if defined(__arm__)
213 typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t;
214 #elif defined(__arm64__)
215 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
216 #else /* defined(__arm__) */
217 #error Undefined architecture
218 #endif /* defined(__arm__) */
219 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
220
221 #define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
222 (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
223
224 #define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
225 (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
226
227 #define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
228 (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
229
230 #define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
231 (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
232
233 #define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
234 (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
235
236 #define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
237 (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
238
239 #define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
240 (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
241
242 #define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
243 (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
244
245 #define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
246 (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
247
248 #define MACHINE_THREAD_STATE ARM_THREAD_STATE
249 #define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
250
251
252 /*
253 * Largest state on this machine:
254 */
255 #define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
256
257 #ifdef XNU_KERNEL_PRIVATE
258
259 static inline boolean_t
is_thread_state32(const arm_unified_thread_state_t * its)260 is_thread_state32(const arm_unified_thread_state_t *its)
261 {
262 return its->ash.flavor == ARM_THREAD_STATE32;
263 }
264
265 static inline boolean_t
is_thread_state64(const arm_unified_thread_state_t * its)266 is_thread_state64(const arm_unified_thread_state_t *its)
267 {
268 return its->ash.flavor == ARM_THREAD_STATE64;
269 }
270
271 static inline arm_thread_state32_t*
thread_state32(arm_unified_thread_state_t * its)272 thread_state32(arm_unified_thread_state_t *its)
273 {
274 return &its->ts_32;
275 }
276
277 static inline arm_thread_state64_t*
thread_state64(arm_unified_thread_state_t * its)278 thread_state64(arm_unified_thread_state_t *its)
279 {
280 return &its->ts_64;
281 }
282
283 static inline const arm_thread_state32_t*
const_thread_state32(const arm_unified_thread_state_t * its)284 const_thread_state32(const arm_unified_thread_state_t *its)
285 {
286 return &its->ts_32;
287 }
288
289 static inline const arm_thread_state64_t*
const_thread_state64(const arm_unified_thread_state_t * its)290 const_thread_state64(const arm_unified_thread_state_t *its)
291 {
292 return &its->ts_64;
293 }
294
295 #if defined(__arm__)
296 #include <arm64/proc_reg.h>
297
298 #define ARM_SAVED_STATE (THREAD_STATE_NONE + 1)
299
300 #if __ARM_VFP__
301 #define VFPSAVE_ALIGN 16
302 #define VFPSAVE_ATTRIB __attribute__((aligned (VFPSAVE_ALIGN)))
303 #define THREAD_ALIGN VFPSAVE_ALIGN
304
305 /*
306 * vector floating point saved state
307 */
308 struct arm_vfpsaved_state {
309 uint32_t r[64];
310 uint32_t fpscr;
311 uint32_t fpexc;
312 };
313 #endif
314
315 struct arm_saved_state {
316 uint32_t r[13]; /* General purpose register r0-r12 */
317 uint32_t sp; /* Stack pointer r13 */
318 uint32_t lr; /* Link register r14 */
319 uint32_t pc; /* Program counter r15 */
320 uint32_t cpsr; /* Current program status register */
321 uint32_t fsr; /* Fault status */
322 uint32_t far; /* Virtual Fault Address */
323 uint32_t exception; /* exception number */
324
325 #if __ARM_VFP__
326 /* VFP state */
327 struct arm_vfpsaved_state VFPdata VFPSAVE_ATTRIB;
328 // for packing reasons chtread_self and DebugData
329 // are inside the the PcbData when __ARM_VFP__ is set
330 arm_debug_state_t *VFPpadding_DebugData;
331 vm_address_t VFPpadding_cthread_self;
332 #endif
333 };
334 typedef struct arm_saved_state arm_saved_state_t;
335
336 /*
337 * Just for coexistence with AArch64 code.
338 */
339 typedef struct arm_saved_state arm_saved_state32_t;
340
341 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)342 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
343 {
344 *dst = *src;
345 }
346
347 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)348 saved_state32(arm_saved_state_t *iss)
349 {
350 return iss;
351 }
352
353 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss __unused)354 is_saved_state32(const arm_saved_state_t *iss __unused)
355 {
356 return TRUE;
357 }
358
359
360 struct arm_saved_state_tagged {
361 uint32_t tag;
362 struct arm_saved_state state;
363 };
364 typedef struct arm_saved_state_tagged arm_saved_state_tagged_t;
365
366 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
367 (sizeof (arm_saved_state_t)/sizeof(unsigned int)))
368
369
370 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)371 get_saved_state_pc(const arm_saved_state_t *iss)
372 {
373 return iss->pc;
374 }
375
376 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)377 add_saved_state_pc(arm_saved_state_t *iss, int diff)
378 {
379 iss->pc += (unsigned int)diff;
380 }
381
382 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)383 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
384 {
385 iss->pc = (typeof(iss->pc))pc;
386 }
387
388 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)389 get_saved_state_sp(const arm_saved_state_t *iss)
390 {
391 return iss->sp;
392 }
393
394 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)395 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
396 {
397 iss->sp = sp;
398 }
399
400 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)401 get_saved_state_fp(const arm_saved_state_t *iss)
402 {
403 return iss->r[7];
404 }
405
406 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)407 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
408 {
409 iss->r[7] = fp;
410 }
411
412 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)413 get_saved_state_lr(const arm_saved_state_t *iss)
414 {
415 return (register_t)iss->lr;
416 }
417
418 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)419 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
420 {
421 iss->lr = lr;
422 }
423
424 static inline register_t
get_saved_state_cpsr(const arm_saved_state_t * iss)425 get_saved_state_cpsr(const arm_saved_state_t *iss)
426 {
427 return iss->cpsr;
428 }
429
430 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)431 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
432 {
433 iss->cpsr |= set_bits;
434 iss->cpsr &= ~clear_bits;
435 }
436
437 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,register_t cpsr)438 set_saved_state_cpsr(arm_saved_state_t *iss, register_t cpsr)
439 {
440 iss->cpsr = cpsr;
441 }
442
443 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned regno)444 get_saved_state_reg(const arm_saved_state_t *iss, unsigned regno)
445 {
446 return iss->r[regno];
447 }
448
449 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned regno,register_t val)450 set_saved_state_reg(arm_saved_state_t *iss, unsigned regno, register_t val)
451 {
452 iss->r[regno] = val;
453 }
454
455 #elif defined(__arm64__)
456
457 #include <kern/assert.h>
458 #include <arm64/proc_reg.h>
459 #define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
460
461 /*
462 * GPR context
463 */
464
465 struct arm_saved_state32 {
466 uint32_t r[13]; /* General purpose register r0-r12 */
467 uint32_t sp; /* Stack pointer r13 */
468 uint32_t lr; /* Link register r14 */
469 uint32_t pc; /* Program counter r15 */
470 uint32_t cpsr; /* Current program status register */
471 uint32_t far; /* Virtual fault address */
472 uint32_t esr; /* Exception syndrome register */
473 uint32_t exception; /* Exception number */
474 };
475 typedef struct arm_saved_state32 arm_saved_state32_t;
476
477 struct arm_saved_state32_tagged {
478 uint32_t tag;
479 struct arm_saved_state32 state;
480 };
481 typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
482
483 #define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
484 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
485
486 struct arm_saved_state64 {
487 uint64_t x[29]; /* General purpose registers x0-x28 */
488 uint64_t fp; /* Frame pointer x29 */
489 uint64_t lr; /* Link register x30 */
490 uint64_t sp; /* Stack pointer x31 */
491 uint64_t pc; /* Program counter */
492 uint32_t cpsr; /* Current program status register */
493 uint32_t reserved; /* Reserved padding */
494 uint64_t far; /* Virtual fault address */
495 uint32_t esr; /* Exception syndrome register */
496 uint32_t exception; /* Exception number */
497 #if defined(HAS_APPLE_PAC)
498 uint64_t jophash;
499 #endif /* defined(HAS_APPLE_PAC) */
500 };
501 typedef struct arm_saved_state64 arm_saved_state64_t;
502
503 #define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
504 (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
505
506 struct arm_saved_state64_tagged {
507 uint32_t tag;
508 struct arm_saved_state64 state;
509 };
510 typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
511
512 struct arm_saved_state {
513 arm_state_hdr_t ash;
514 union {
515 struct arm_saved_state32 ss_32;
516 struct arm_saved_state64 ss_64;
517 } uss;
518 } __attribute__((aligned(16)));
519 #define ss_32 uss.ss_32
520 #define ss_64 uss.ss_64
521
522 typedef struct arm_saved_state arm_saved_state_t;
523
524 struct arm_kernel_saved_state {
525 uint64_t x[10]; /* General purpose registers x19-x28 */
526 uint64_t fp; /* Frame pointer x29 */
527 uint64_t lr; /* Link register x30 */
528 uint64_t sp; /* Stack pointer x31 */
529 /* Some things here we DO need to preserve */
530 uint8_t pc_was_in_userspace;
531 uint8_t ssbs;
532 uint8_t dit;
533 uint8_t uao;
534 } __attribute__((aligned(16)));
535
536 typedef struct arm_kernel_saved_state arm_kernel_saved_state_t;
537
538 #if defined(XNU_KERNEL_PRIVATE)
539 #if defined(HAS_APPLE_PAC)
540
541 #include <sys/cdefs.h>
542
543 /*
544 * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
545 * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
546 * and osfmk/arm/machine_routines.h.
547 */
548 __BEGIN_DECLS
549 extern uint64_t ml_pac_safe_interrupts_disable(void);
550 extern void ml_pac_safe_interrupts_restore(uint64_t);
551 __END_DECLS
552
553 /*
554 * Methods used to sign and check thread state to detect corruptions of saved
555 * thread state across exceptions and context switches.
556 */
557 extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
558
559 extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
560
561 /* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
562 #define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
563
564 /**
565 * Verify the signed thread state in _iss, execute the assembly instructions
566 * _instr, and re-sign the modified thread state. Varargs specify additional
567 * inputs.
568 *
569 * _instr may read or modify the thread state in the following registers:
570 *
571 * x0: _iss
572 * x1: authed _iss->ss_64.pc
573 * w2: authed _iss->ss_64.cpsr
574 * x3: authed _iss->ss_64.lr
575 * x4: authed _iss->ss_64.x16
576 * x5: authed _iss->ss_64.x17
577 * x6: scratch register
578 * x7: scratch register
579 */
580 #define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...) \
581 do { \
582 uint64_t _intr = ml_pac_safe_interrupts_disable(); \
583 asm volatile ( \
584 "mov x9, lr" "\n" \
585 "mov x0, %[iss]" "\n" \
586 "msr SPSel, #1" "\n" \
587 "ldp x4, x5, [x0, %[SS64_X16]]" "\n" \
588 "ldr x7, [x0, %[SS64_PC]]" "\n" \
589 "ldr w8, [x0, %[SS64_CPSR]]" "\n" \
590 "ldr x3, [x0, %[SS64_LR]]" "\n" \
591 "mov x1, x7" "\n" \
592 "mov w2, w8" "\n" \
593 "bl _ml_check_signed_state" "\n" \
594 "mov x1, x7" "\n" \
595 "mov w2, w8" "\n" \
596 _instr "\n" \
597 "bl _ml_sign_thread_state" "\n" \
598 "msr SPSel, #0" "\n" \
599 "mov lr, x9" "\n" \
600 : \
601 : [iss] "r"(_iss), \
602 [SS64_X16] "i"(ss64_offsetof(x[16])), \
603 [SS64_PC] "i"(ss64_offsetof(pc)), \
604 [SS64_CPSR] "i"(ss64_offsetof(cpsr)), \
605 [SS64_LR] "i"(ss64_offsetof(lr)),##__VA_ARGS__ \
606 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", \
607 "x9" \
608 ); \
609 ml_pac_safe_interrupts_restore(_intr); \
610 } while (0)
611
612 static inline void
check_and_sign_copied_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)613 check_and_sign_copied_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
614 {
615 MANIPULATE_SIGNED_THREAD_STATE(src,
616 "mov x0, %[dst]",
617 [dst] "r"(dst)
618 );
619 }
620 #endif /* defined(HAS_APPLE_PAC) */
621
622 static inline void
copy_signed_thread_state(arm_saved_state_t * dst,const arm_saved_state_t * src)623 copy_signed_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
624 {
625 *dst = *src;
626 #if defined(HAS_APPLE_PAC)
627 check_and_sign_copied_thread_state(dst, src);
628 #endif
629 }
630
631 #endif /* defined(XNU_KERNEL_PRIVATE) */
632
633 static inline boolean_t
is_saved_state32(const arm_saved_state_t * iss)634 is_saved_state32(const arm_saved_state_t *iss)
635 {
636 return iss->ash.flavor == ARM_SAVED_STATE32;
637 }
638
639 static inline boolean_t
is_saved_state64(const arm_saved_state_t * iss)640 is_saved_state64(const arm_saved_state_t *iss)
641 {
642 return iss->ash.flavor == ARM_SAVED_STATE64;
643 }
644
645 static inline arm_saved_state32_t*
saved_state32(arm_saved_state_t * iss)646 saved_state32(arm_saved_state_t *iss)
647 {
648 return &iss->ss_32;
649 }
650
651 static inline const arm_saved_state32_t*
const_saved_state32(const arm_saved_state_t * iss)652 const_saved_state32(const arm_saved_state_t *iss)
653 {
654 return &iss->ss_32;
655 }
656
657 static inline arm_saved_state64_t*
saved_state64(arm_saved_state_t * iss)658 saved_state64(arm_saved_state_t *iss)
659 {
660 return &iss->ss_64;
661 }
662
663 static inline const arm_saved_state64_t*
const_saved_state64(const arm_saved_state_t * iss)664 const_saved_state64(const arm_saved_state_t *iss)
665 {
666 return &iss->ss_64;
667 }
668
669 static inline register_t
get_saved_state_pc(const arm_saved_state_t * iss)670 get_saved_state_pc(const arm_saved_state_t *iss)
671 {
672 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc);
673 }
674
675 static inline void
add_saved_state_pc(arm_saved_state_t * iss,int diff)676 add_saved_state_pc(arm_saved_state_t *iss, int diff)
677 {
678 if (is_saved_state32(iss)) {
679 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
680 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
681 } else {
682 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
683 MANIPULATE_SIGNED_THREAD_STATE(iss,
684 "mov w6, %w[diff] \n"
685 "add x1, x1, w6, sxtw \n"
686 "str x1, [x0, %[SS64_PC]] \n",
687 [diff] "r"(diff)
688 );
689 #else
690 saved_state64(iss)->pc += (unsigned long)diff;
691 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
692 }
693 }
694
695 static inline void
set_saved_state_pc(arm_saved_state_t * iss,register_t pc)696 set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
697 {
698 if (is_saved_state32(iss)) {
699 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
700 } else {
701 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
702 MANIPULATE_SIGNED_THREAD_STATE(iss,
703 "mov x1, %[pc] \n"
704 "str x1, [x0, %[SS64_PC]] \n",
705 [pc] "r"(pc)
706 );
707 #else
708 saved_state64(iss)->pc = (unsigned long)pc;
709 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
710 }
711 }
712
713 static inline register_t
get_saved_state_sp(const arm_saved_state_t * iss)714 get_saved_state_sp(const arm_saved_state_t *iss)
715 {
716 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp);
717 }
718
719 static inline void
set_saved_state_sp(arm_saved_state_t * iss,register_t sp)720 set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
721 {
722 if (is_saved_state32(iss)) {
723 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
724 } else {
725 saved_state64(iss)->sp = (uint64_t)sp;
726 }
727 }
728
729 static inline register_t
get_saved_state_lr(const arm_saved_state_t * iss)730 get_saved_state_lr(const arm_saved_state_t *iss)
731 {
732 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr);
733 }
734
735 static inline void
set_saved_state_lr(arm_saved_state_t * iss,register_t lr)736 set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
737 {
738 if (is_saved_state32(iss)) {
739 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
740 } else {
741 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
742 MANIPULATE_SIGNED_THREAD_STATE(iss,
743 "mov x3, %[lr] \n"
744 "str x3, [x0, %[SS64_LR]] \n",
745 [lr] "r"(lr)
746 );
747 #else
748 saved_state64(iss)->lr = (unsigned long)lr;
749 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
750 }
751 }
752
753 static inline register_t
get_saved_state_fp(const arm_saved_state_t * iss)754 get_saved_state_fp(const arm_saved_state_t *iss)
755 {
756 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp);
757 }
758
759 static inline void
set_saved_state_fp(arm_saved_state_t * iss,register_t fp)760 set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
761 {
762 if (is_saved_state32(iss)) {
763 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
764 } else {
765 saved_state64(iss)->fp = (uint64_t)fp;
766 }
767 }
768
769 static inline int
check_saved_state_reglimit(const arm_saved_state_t * iss,unsigned reg)770 check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
771 {
772 return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
773 }
774
775 static inline register_t
get_saved_state_reg(const arm_saved_state_t * iss,unsigned reg)776 get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
777 {
778 if (!check_saved_state_reglimit(iss, reg)) {
779 return 0;
780 }
781
782 return (register_t)(is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]));
783 }
784
785 static inline void
set_saved_state_reg(arm_saved_state_t * iss,unsigned reg,register_t value)786 set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
787 {
788 if (!check_saved_state_reglimit(iss, reg)) {
789 return;
790 }
791
792 if (is_saved_state32(iss)) {
793 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
794 } else {
795 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
796 /* x16 and x17 are part of the jophash */
797 if (reg == 16) {
798 MANIPULATE_SIGNED_THREAD_STATE(iss,
799 "mov x4, %[value] \n"
800 "str x4, [x0, %[SS64_X16]] \n",
801 [value] "r"(value)
802 );
803 return;
804 } else if (reg == 17) {
805 MANIPULATE_SIGNED_THREAD_STATE(iss,
806 "mov x5, %[value] \n"
807 "str x5, [x0, %[SS64_X17]] \n",
808 [value] "r"(value),
809 [SS64_X17] "i"(ss64_offsetof(x[17]))
810 );
811 return;
812 }
813 #endif
814 saved_state64(iss)->x[reg] = (uint64_t)value;
815 }
816 }
817
818
819 static inline uint32_t
get_saved_state_cpsr(const arm_saved_state_t * iss)820 get_saved_state_cpsr(const arm_saved_state_t *iss)
821 {
822 return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
823 }
824
825 static inline void
mask_saved_state_cpsr(arm_saved_state_t * iss,uint32_t set_bits,uint32_t clear_bits)826 mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
827 {
828 if (is_saved_state32(iss)) {
829 saved_state32(iss)->cpsr |= set_bits;
830 saved_state32(iss)->cpsr &= ~clear_bits;
831 } else {
832 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
833 MANIPULATE_SIGNED_THREAD_STATE(iss,
834 "mov w6, %w[set_bits] \n"
835 "orr w2, w2, w6, lsl #0 \n"
836 "mov w6, %w[clear_bits] \n"
837 "bic w2, w2, w6, lsl #0 \n"
838 "str w2, [x0, %[SS64_CPSR]] \n",
839 [set_bits] "r"(set_bits),
840 [clear_bits] "r"(clear_bits)
841 );
842 #else
843 saved_state64(iss)->cpsr |= set_bits;
844 saved_state64(iss)->cpsr &= ~clear_bits;
845 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
846 }
847 }
848
849 static inline void
set_saved_state_cpsr(arm_saved_state_t * iss,uint32_t cpsr)850 set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
851 {
852 if (is_saved_state32(iss)) {
853 saved_state32(iss)->cpsr = cpsr;
854 } else {
855 #if defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC)
856 MANIPULATE_SIGNED_THREAD_STATE(iss,
857 "mov w2, %w[cpsr] \n"
858 "str w2, [x0, %[SS64_CPSR]] \n",
859 [cpsr] "r"(cpsr)
860 );
861 #else
862 saved_state64(iss)->cpsr = cpsr;
863 #endif /* defined(XNU_KERNEL_PRIVATE) && defined(HAS_APPLE_PAC) */
864 }
865 }
866
867 static inline register_t
get_saved_state_far(const arm_saved_state_t * iss)868 get_saved_state_far(const arm_saved_state_t *iss)
869 {
870 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far);
871 }
872
873 static inline void
set_saved_state_far(arm_saved_state_t * iss,register_t far)874 set_saved_state_far(arm_saved_state_t *iss, register_t far)
875 {
876 if (is_saved_state32(iss)) {
877 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
878 } else {
879 saved_state64(iss)->far = (uint64_t)far;
880 }
881 }
882
883 static inline uint32_t
get_saved_state_esr(const arm_saved_state_t * iss)884 get_saved_state_esr(const arm_saved_state_t *iss)
885 {
886 return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
887 }
888
889 static inline void
set_saved_state_esr(arm_saved_state_t * iss,uint32_t esr)890 set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
891 {
892 if (is_saved_state32(iss)) {
893 saved_state32(iss)->esr = esr;
894 } else {
895 saved_state64(iss)->esr = esr;
896 }
897 }
898
899 static inline uint32_t
get_saved_state_exc(const arm_saved_state_t * iss)900 get_saved_state_exc(const arm_saved_state_t *iss)
901 {
902 return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
903 }
904
905 static inline void
set_saved_state_exc(arm_saved_state_t * iss,uint32_t exc)906 set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
907 {
908 if (is_saved_state32(iss)) {
909 saved_state32(iss)->exception = exc;
910 } else {
911 saved_state64(iss)->exception = exc;
912 }
913 }
914
915 extern void panic_unimplemented(void);
916
917 /**
918 * Extracts the SVC (Supervisor Call) number from the appropriate GPR (General
919 * Purpose Register).
920 *
921 * @param[in] iss the 32-bit or 64-bit ARM saved state (i.e. trap frame).
922 *
923 * @return The SVC number.
924 */
925 static inline int
get_saved_state_svc_number(const arm_saved_state_t * iss)926 get_saved_state_svc_number(const arm_saved_state_t *iss)
927 {
928 return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
929 }
930
931 typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
932
933 struct arm_debug_aggregate_state {
934 arm_state_hdr_t dsh;
935 union {
936 arm_debug_state32_t ds32;
937 arm_debug_state64_t ds64;
938 } uds;
939 os_refcnt_t ref;
940 } __attribute__((aligned(16)));
941
942 typedef struct arm_debug_aggregate_state arm_debug_state_t;
943
944 #define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
945 (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
946
947 /*
948 * NEON context
949 */
950 typedef __uint128_t uint128_t;
951 typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
952 typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
953
954 struct arm_neon_saved_state32 {
955 union {
956 uint128_t q[16];
957 uint64_t d[32];
958 uint32_t s[32];
959 } v;
960 uint32_t fpsr;
961 uint32_t fpcr;
962 };
963 typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
964
965 #define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
966 (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
967
968 struct arm_neon_saved_state64 {
969 union {
970 uint128_t q[32];
971 uint64x2_t d[32];
972 uint32x4_t s[32];
973 } v;
974 uint32_t fpsr;
975 uint32_t fpcr;
976 };
977 typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
978
979 #define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
980 (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
981
982 struct arm_neon_saved_state {
983 arm_state_hdr_t nsh;
984 union {
985 struct arm_neon_saved_state32 ns_32;
986 struct arm_neon_saved_state64 ns_64;
987 } uns;
988 };
989 typedef struct arm_neon_saved_state arm_neon_saved_state_t;
990 #define ns_32 uns.ns_32
991 #define ns_64 uns.ns_64
992
993 struct arm_kernel_neon_saved_state {
994 uint64_t d[8];
995 uint32_t fpcr;
996 };
997 typedef struct arm_kernel_neon_saved_state arm_kernel_neon_saved_state_t;
998
999 static inline boolean_t
is_neon_saved_state32(const arm_neon_saved_state_t * state)1000 is_neon_saved_state32(const arm_neon_saved_state_t *state)
1001 {
1002 return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
1003 }
1004
1005 static inline boolean_t
is_neon_saved_state64(const arm_neon_saved_state_t * state)1006 is_neon_saved_state64(const arm_neon_saved_state_t *state)
1007 {
1008 return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
1009 }
1010
1011 static inline arm_neon_saved_state32_t *
neon_state32(arm_neon_saved_state_t * state)1012 neon_state32(arm_neon_saved_state_t *state)
1013 {
1014 return &state->ns_32;
1015 }
1016
1017 static inline arm_neon_saved_state64_t *
neon_state64(arm_neon_saved_state_t * state)1018 neon_state64(arm_neon_saved_state_t *state)
1019 {
1020 return &state->ns_64;
1021 }
1022
1023
1024 /*
1025 * Aggregated context
1026 */
1027
1028 struct arm_context {
1029 struct arm_saved_state ss;
1030 struct arm_neon_saved_state ns;
1031 };
1032 typedef struct arm_context arm_context_t;
1033
1034 struct arm_kernel_context {
1035 struct arm_kernel_saved_state ss;
1036 struct arm_kernel_neon_saved_state ns;
1037 };
1038 typedef struct arm_kernel_context arm_kernel_context_t;
1039
1040 extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
1041 extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
1042
1043 #else /* defined(__arm__) */
1044 #error Unknown arch
1045 #endif /* defined(__arm__) */
1046
1047 extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
1048 extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
1049
1050 #endif /* XNU_KERNEL_PRIVATE */
1051
1052 #endif /* defined (__arm__) || defined (__arm64__) */
1053
1054 #endif /* _ARM_THREAD_STATUS_H_ */
1055