1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _KERN_PERCPU_H_
30 #define _KERN_PERCPU_H_
31
32 #include <mach/vm_types.h>
33
34 __BEGIN_DECLS
35
36 #if XNU_KERNEL_PRIVATE
37 #include <libkern/section_keywords.h>
38
39 #pragma GCC visibility push(hidden)
40
41 /*!
42 * @macro PERCPU_DECL
43 *
44 * @abstract
45 * Declares a per-CPU variable in a header.
46 *
47 * @param type_t the per-CPU variable type
48 * @param name the per-CPU variable name
49 */
50 #define PERCPU_DECL(type_t, name) \
51 extern type_t __PERCPU_NAME(name)
52
53 /*!
54 * @macro PERCPU_DATA
55 *
56 * @abstract
57 * Defines a per-CPU variable in a translation unit.
58 *
59 * @discussion
60 * @c PERCPU_DECL can be used in headers to export the variable to clients.
61 *
62 * By default, per-cpu data is 0-initialized. Per-CPU data is allocated during
63 * the STARTUP_SUB_KMEM phase and can be initialized with a STARTUP
64 * callback in any later phase.
65 *
66 * Usage is:
67 * <code>
68 * [ static ] type PERCPU_DATA(name);
69 * </code>
70 *
71 * @param name the per-CPU variable name
72 */
73 #define PERCPU_DATA(name) \
74 __percpu __PERCPU_NAME(name) = {0}
75
76 /*
77 * Same as before, but as a temporary hack with a 0 initializer
78 * instead of {0}, because clang has a bug where it does not accept
79 * the latter for _Atomic types. (And we want to keep the initializer
80 * to prevent people to think they can initialize it to anything else
81 * but 0.)
82 */
83 #define PERCPU_DATA_HACK_78750602(name) \
84 __percpu __PERCPU_NAME(name) = 0
85
86 /*!
87 * @macro PERCPU_GET
88 *
89 * @abstract
90 * Gets a pointer to the per-CPU instance of the variable for the processor the
91 * code is currently running on.
92 *
93 * @discussion
94 * It is expected that preemption or interrupts are disabled when this is used,
95 * as a context-switch might move the current thread to another CPU.
96 *
97 * It is also valid in code that wasn't already disabling preemption and cares
98 * about code-gen size a lot to use this outside of a preemption-disabled
99 * section provided that the data is modified using atomics.
100 *
101 * Note that if several per-CPU pointers are acquired in short succession,
102 * @c PERCPU_GET_WITH_BASE can be used to avoid the repeated calls to
103 * @c current_percpu_base() which the compiler wont't elide.
104 *
105 * @param name the per-CPU variable name
106 */
107 #define PERCPU_GET(name) \
108 __PERCPU_CAST(name, current_percpu_base() + __PERCPU_ADDR(name))
109
110 /*!
111 * @function current_percpu_base()
112 *
113 * @abstract
114 * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE().
115 *
116 * @see PERCPU_GET() for conditions of use.
117 */
118 extern vm_offset_t current_percpu_base(void);
119
120 /*!
121 * @function other_percpu_base()
122 *
123 * @abstract
124 * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE(),
125 * for the speficied cpu number.
126 *
127 * @param cpu_number the cpu number for which we want a base.
128 */
129 extern vm_offset_t other_percpu_base(int cpu_number);
130
131 /*!
132 * @macro PERCPU_GET_MASTER
133 *
134 * @abstract
135 * Gets a pointer to the master per-CPU instance of the variable.
136 *
137 * @param base the per-CPU base to use
138 * @param name the per-CPU variable name
139 */
140 #define PERCPU_GET_MASTER(name) \
141 (&__PERCPU_NAME(name))
142
143 /*!
144 * @macro PERCPU_GET_WITH_BASE
145 *
146 * @abstract
147 * Gets a pointer to the per-CPU instance of the variable for the specified
148 * base.
149 *
150 * @param base the per-CPU base to use
151 * @param name the per-CPU variable name
152 */
153 #define PERCPU_GET_WITH_BASE(base, name) \
154 __PERCPU_CAST(name, base + __PERCPU_ADDR(name))
155
156 /*!
157 * @macro PERCPU_GET_RELATIVE
158 *
159 * @abstract
160 * Gets a pointer to the per-CPU instance of a variable relative to another
161 * known one.
162 *
163 * @description
164 * When a per-CPU slot address is known, but the caller doesn't know the base
165 * from which it was derived, then this allows to compute another per-CPU slot
166 * address for a different variable but for the same CPU, without any loads.
167 *
168 * @param name the per-CPU variable name
169 * @param other the other per-CPU variable name
170 * @param ptr a pointer to the other variable slot
171 */
172 #define PERCPU_GET_RELATIVE(name, other, ptr) ({ \
173 __PERCPU_TYPE(other) __other_ptr = (ptr); /* type check */ \
174 vm_offset_t __offs = __PERCPU_ADDR(name) - __PERCPU_ADDR(other); \
175 __PERCPU_CAST(name, (vm_address_t)__other_ptr + __offs); \
176 })
177
178 /*!
179 * @macro percpu_foreach_base()
180 *
181 * @abstract
182 * Enumerates all Per-CPU variable bases.
183 *
184 * @param it the name of the iterator
185 */
186 #define percpu_foreach_base(it) \
187 for (vm_offset_t it = 0, \
188 __next_ ## it = percpu_base.start, \
189 __end_ ## it = percpu_base.end; \
190 \
191 it <= __end_ ## it; \
192 \
193 it = __next_ ## it, \
194 __next_ ## it += percpu_section_size())
195
196 /*!
197 * @macro percpu_foreach()
198 *
199 * @abstract
200 * Enumerates all Per-CPU variable instances.
201 *
202 * @param it the name of the iterator
203 * @param name the per-CPU variable name
204 */
205 #define percpu_foreach(it, name) \
206 for (__PERCPU_TYPE(name) it, \
207 __base_ ## it = NULL, \
208 __next_ ## it = (typeof(it))percpu_base.start, \
209 __end_ ## it = (typeof(it))percpu_base.end; \
210 \
211 (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
212 __base_ ## it <= __end_ ## it); \
213 \
214 __base_ ## it = __next_ ## it, \
215 __next_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
216
217 /*!
218 * @macro percpu_foreach_secondary_base()
219 *
220 * @abstract
221 * Enumerates all Per-CPU variable bases, skipping the master slot.
222 *
223 * @param it the name of the iterator
224 */
225 #define percpu_foreach_secondary_base(it) \
226 for (vm_offset_t it = percpu_base.start, __end_ ## it = percpu_base.end; \
227 it <= __end_ ## it; it += percpu_section_size())
228
229 /*!
230 * @macro percpu_foreach_secondary()
231 *
232 * @abstract
233 * Enumerates all Per-CPU variable instances, skipping the master slot.
234 *
235 * @param it the name of the iterator
236 * @param name the per-CPU variable name
237 */
238 #define percpu_foreach_secondary(it, name) \
239 for (__PERCPU_TYPE(name) it, \
240 __base_ ## it = (typeof(it))__unsafe_forge_bidi_indexable(void *, percpu_base.start, percpu_base.end - percpu_base.start), \
241 __end_ ## it = (typeof(it))((caddr_t)__base_ ## it + percpu_base.end - percpu_base.start); \
242 \
243 (it = (typeof(it))((caddr_t)__base_ ## it + __PERCPU_ADDR(name)), \
244 __base_ ## it <= __end_ ## it); \
245 \
246 __base_ ## it = (typeof(it))((caddr_t)__base_ ## it + percpu_section_size()))
247
248 #pragma mark - implementation details
249
250 /*
251 * Below this point are implementation details that should not be used directly,
252 * except by the macros above, or architecture specific code.
253 */
254
255 #define __percpu __attribute__((section("__DATA, __percpu")))
256 #define __PERCPU_NAME(name) percpu_slot_ ## name
257 #define __PERCPU_ADDR(name) ((vm_offset_t)&__PERCPU_NAME(name))
258 #define __PERCPU_TYPE(name) typeof(&__PERCPU_NAME(name))
259 #define __PERCPU_CAST(name, expr) ((__PERCPU_TYPE(name))__unsafe_forge_bidi_indexable(void *, (expr), sizeof(__PERCPU_NAME(name))))
260
261 /*
262 * Note for implementors:
263 *
264 * A `base` represents a pointer in the percpu allocation offset by
265 * `percpu_section_start()` so that PERCPU_GET() is a single addition.
266 *
267 * percpu_base.end is inclusive, so that percpu_foreach() and
268 * percpu_foreach_base() can do a `<=` comparison.
269 *
270 * Because the first base is `0` (because the master CPU is using the static
271 * percpu section), it allows for the compiler to know that for the first
272 * iteration the comparison is always true.
273 */
274 extern struct percpu_base {
275 vm_address_t start;
276 vm_address_t end;
277 vm_offset_t size;
278 } percpu_base;
279
280 static __pure2 inline vm_offset_t
percpu_section_start(void)281 percpu_section_start(void)
282 {
283 extern char __percpu_section_start[0] __SECTION_START_SYM("__DATA", "__percpu");
284 return (vm_offset_t)__percpu_section_start;
285 }
286
287 static __pure2 inline vm_offset_t
percpu_section_end(void)288 percpu_section_end(void)
289 {
290 extern char __percpu_section_end[0] __SECTION_END_SYM("__DATA", "__percpu");
291 return (vm_offset_t)__percpu_section_end;
292 }
293
294 static __pure2 inline vm_size_t
percpu_section_size(void)295 percpu_section_size(void)
296 {
297 return percpu_section_end() - percpu_section_start();
298 }
299
300 #pragma GCC visibility pop
301 #endif /* XNU_KERNEL_PRIVATE */
302
303 __END_DECLS
304
305 #endif /* _KERN_PERCPU_H_ */
306