1 /*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef OS_TSD_H
30 #define OS_TSD_H
31
32 /* The low nine slots of the TSD are reserved for libsyscall usage. */
33 #define __TSD_RESERVED_BASE 0
34 #define __TSD_RESERVED_MAX 9
35
36 #define __TSD_THREAD_SELF 0
37 #define __TSD_ERRNO 1
38 #define __TSD_MIG_REPLY 2
39 #define __TSD_MACH_THREAD_SELF 3
40 #define __TSD_THREAD_QOS_CLASS 4
41 #define __TSD_RETURN_TO_KERNEL 5
42 /* slot 6 is reserved for Windows/WINE compatibility reasons */
43 #define __TSD_PTR_MUNGE 7
44 #define __TSD_MACH_SPECIAL_REPLY 8
45 #define __TSD_SEMAPHORE_CACHE 9
46
47 #define __TSD_MACH_MSG_AUX 123
48
49
50 #define __TPIDR_CPU_NUM_MASK 0x0000000000000fff
51 #define __TPIDR_CPU_NUM_SHIFT 0
52
53 #ifndef __ASSEMBLER__
54
55 #include <stdint.h>
56 #include <TargetConditionals.h>
57
58 #ifdef __arm__
59 #include <arm/arch.h>
60 #endif
61
62 extern void _thread_set_tsd_base(void *tsd_base);
63
64 /*
65 * The implementation details of this function are not ABI and are subject to change,
66 * do not copy them in another project
67 */
68 __attribute__((always_inline))
69 static __inline__ unsigned int
_os_cpu_number(void)70 _os_cpu_number(void)
71 {
72 #if defined(__arm__)
73 uintptr_t p;
74 __asm__ __volatile__ ("mrc p15, 0, %[p], c13, c0, 3" : [p] "=&r" (p));
75 return (unsigned int)(p & 0x3ul);
76 #elif defined(__arm64__)
77 uint64_t p;
78 __asm__ __volatile__ ("mrs %0, TPIDR_EL0" : "=r" (p));
79 return (p & __TPIDR_CPU_NUM_MASK) >> __TPIDR_CPU_NUM_SHIFT;
80 #elif defined(__x86_64__) || defined(__i386__)
81 struct { uintptr_t p1, p2; } p;
82 __asm__ __volatile__ ("sidt %[p]" : [p] "=&m" (p));
83 return (unsigned int)(p.p1 & 0xfff);
84 #else
85 #error _os_cpu_number not implemented on this architecture
86 #endif
87 }
88
89 #if defined(__i386__) || defined(__x86_64__)
90
91 #if defined(__has_attribute)
92 #if __has_attribute(address_space)
93 #define OS_GS_RELATIVE __attribute__((address_space(256)))
94 #endif
95 #endif
96
97 #ifdef OS_GS_RELATIVE
98 #define _os_tsd_get_base() ((void * OS_GS_RELATIVE *)0)
99 #else
100 __attribute__((always_inline))
101 static __inline__ void*
_os_tsd_get_direct(unsigned long slot)102 _os_tsd_get_direct(unsigned long slot)
103 {
104 void *ret;
105 __asm__("mov %%gs:%1, %0" : "=r" (ret) : "m" (*(void **)(slot * sizeof(void *))));
106 return ret;
107 }
108
109 __attribute__((always_inline))
110 static __inline__ int
_os_tsd_set_direct(unsigned long slot,void * val)111 _os_tsd_set_direct(unsigned long slot, void *val)
112 {
113 #if defined(__i386__) && defined(__PIC__)
114 __asm__("movl %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "rn" (val));
115 #elif defined(__i386__) && !defined(__PIC__)
116 __asm__("movl %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "ri" (val));
117 #else
118 __asm__("movq %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "rn" (val));
119 #endif
120 return 0;
121 }
122 #endif
123
124 #elif defined(__arm__) || defined(__arm64__)
125
126 __attribute__((always_inline, const))
127 static __inline__ void**
_os_tsd_get_base(void)128 _os_tsd_get_base(void)
129 {
130 #if defined(__arm__)
131 uintptr_t tsd;
132 __asm__("mrc p15, 0, %0, c13, c0, 3\n"
133 "bic %0, %0, #0x3\n" : "=r" (tsd));
134 /* lower 2-bits contain CPU number */
135 #elif defined(__arm64__)
136 /*
137 * <rdar://73762648> Do not use __builtin_arm_rsr64("TPIDRRO_EL0")
138 * so that the "const" attribute takes effect and repeated use
139 * is coalesced properly.
140 */
141 uint64_t tsd;
142 __asm__ ("mrs %0, TPIDRRO_EL0" : "=r" (tsd));
143 #endif
144
145 return (void**)(uintptr_t)tsd;
146 }
147 #define _os_tsd_get_base() _os_tsd_get_base()
148
149 #else
150 #error _os_tsd_get_base not implemented on this architecture
151 #endif
152
153 #ifdef _os_tsd_get_base
154 __attribute__((always_inline))
155 static __inline__ void*
_os_tsd_get_direct(unsigned long slot)156 _os_tsd_get_direct(unsigned long slot)
157 {
158 return _os_tsd_get_base()[slot];
159 }
160
161 __attribute__((always_inline))
162 static __inline__ int
_os_tsd_set_direct(unsigned long slot,void * val)163 _os_tsd_set_direct(unsigned long slot, void *val)
164 {
165 _os_tsd_get_base()[slot] = val;
166 return 0;
167 }
168 #endif
169
170 __attribute__((always_inline, const))
171 static __inline__ uintptr_t
_os_ptr_munge_token(void)172 _os_ptr_munge_token(void)
173 {
174 return (uintptr_t)_os_tsd_get_direct(__TSD_PTR_MUNGE);
175 }
176
177 __attribute__((always_inline, const))
178 static __inline__ uintptr_t
_os_ptr_munge(uintptr_t ptr)179 _os_ptr_munge(uintptr_t ptr)
180 {
181 return ptr ^ _os_ptr_munge_token();
182 }
183 #define _OS_PTR_MUNGE(_ptr) _os_ptr_munge((uintptr_t)(_ptr))
184 #define _OS_PTR_UNMUNGE(_ptr) _os_ptr_munge((uintptr_t)(_ptr))
185
186 #else // __ASSEMBLER__
187
188 #define _OS_TSD_OFFSET(_key) \
189 ((__POINTER_WIDTH__/__CHAR_BIT__)*_key)
190
191 #if defined(__i386__) || defined(__x86_64__)
192
193 #define _OS_PTR_MUNGE(_reg) \
194 xor %gs:_OS_TSD_OFFSET(__TSD_PTR_MUNGE), _reg
195
196 #define _OS_PTR_UNMUNGE(_reg) \
197 _OS_PTR_MUNGE(_reg)
198
199 #elif defined(__arm__) || defined(__arm64__)
200
201 #if defined(__arm__)
202
203 #define _OS_PTR_MUNGE_TOKEN(_reg, _token) \
204 mrc p15, 0, _reg, c13, c0, 3; \
205 bic _reg, _reg, #3; \
206 ldr _token, [ _reg, #_OS_TSD_OFFSET(__TSD_PTR_MUNGE) ]
207
208 #elif defined(__arm64__)
209
210 #define _OS_PTR_MUNGE_TOKEN(_reg, _token) \
211 mrs _reg, TPIDRRO_EL0 %% \
212 ldr _token, [ _reg, #_OS_TSD_OFFSET(__TSD_PTR_MUNGE) ]
213
214 #endif // defined(__arm64__)
215
216 #define _OS_PTR_MUNGE(_regdest, _regsrc, _token) \
217 eor _regdest, _regsrc, _token
218
219 #define _OS_PTR_UNMUNGE(_regdest, _regsrc, _token) \
220 _OS_PTR_MUNGE(_regdest, _regsrc, _token)
221
222 #endif // defined(__arm__) || defined(__arm64__)
223
224 #endif // __ASSEMBLER__
225
226 #endif // OS_TSD_H
227