1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _KERN_PERCPU_H_
30 #define _KERN_PERCPU_H_
31
32 #include <mach/vm_types.h>
33 #include <mach/machine/vm_param.h> /* For PAGE_MASK */
34
35 __BEGIN_DECLS
36
37 #if XNU_KERNEL_PRIVATE
38 #include <libkern/section_keywords.h>
39
40 #pragma GCC visibility push(hidden)
41
42 /*!
43 * @macro PERCPU_DECL
44 *
45 * @abstract
46 * Declares a per-CPU variable in a header.
47 *
48 * @param type_t the per-CPU variable type
49 * @param name the per-CPU variable name
50 */
51 #define PERCPU_DECL(type_t, name) \
52 extern type_t __PERCPU_NAME(name)
53
54 /*!
55 * @macro PERCPU_DATA
56 *
57 * @abstract
58 * Defines a per-CPU variable in a translation unit.
59 *
60 * @discussion
61 * @c PERCPU_DECL can be used in headers to export the variable to clients.
62 *
63 * By default, per-cpu data is 0-initialized. Per-CPU data is allocated during
64 * the STARTUP_SUB_KMEM phase and can be initialized with a STARTUP
65 * callback in any later phase.
66 *
67 * Usage is:
68 * <code>
69 * [ static ] type PERCPU_DATA(name);
70 * </code>
71 *
72 * @param name the per-CPU variable name
73 */
74 #define PERCPU_DATA(name) \
75 __percpu __PERCPU_NAME(name) = {0}
76
77 /*
78 * Same as before, but as a temporary hack with a 0 initializer
79 * instead of {0}, because clang has a bug where it does not accept
80 * the latter for _Atomic types. (And we want to keep the initializer
81 * to prevent people to think they can initialize it to anything else
82 * but 0.)
83 */
84 #define PERCPU_DATA_HACK_78750602(name) \
85 __percpu __PERCPU_NAME(name) = 0
86
87 /*!
88 * @macro PERCPU_GET
89 *
90 * @abstract
91 * Gets a pointer to the per-CPU instance of the variable for the processor the
92 * code is currently running on.
93 *
94 * @discussion
95 * It is expected that preemption or interrupts are disabled when this is used,
96 * as a context-switch might move the current thread to another CPU.
97 *
98 * It is also valid in code that wasn't already disabling preemption and cares
99 * about code-gen size a lot to use this outside of a preemption-disabled
100 * section provided that the data is modified using atomics.
101 *
102 * Note that if several per-CPU pointers are acquired in short succession,
103 * @c PERCPU_GET_WITH_BASE can be used to avoid the repeated calls to
104 * @c current_percpu_base() which the compiler wont't elide.
105 *
106 * @param name the per-CPU variable name
107 */
108 #define PERCPU_GET(name) \
109 __PERCPU_CAST(name, current_percpu_base() + __PERCPU_ADDR(name))
110
111 /*!
112 * @function current_percpu_base()
113 *
114 * @abstract
115 * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE().
116 *
117 * @see PERCPU_GET() for conditions of use.
118 */
119 extern vm_offset_t current_percpu_base(void);
120
121 /*!
122 * @function other_percpu_base()
123 *
124 * @abstract
125 * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE(),
126 * for the speficied cpu number.
127 *
128 * @param cpu_number the cpu number for which we want a base.
129 */
130 extern vm_offset_t other_percpu_base(int cpu_number);
131
132 /*!
133 * @macro PERCPU_GET_MASTER
134 *
135 * @abstract
136 * Gets a pointer to the master per-CPU instance of the variable.
137 *
138 * @param base the per-CPU base to use
139 * @param name the per-CPU variable name
140 */
141 #define PERCPU_GET_MASTER(name) \
142 (&__PERCPU_NAME(name))
143
144 /*!
145 * @macro PERCPU_GET_WITH_BASE
146 *
147 * @abstract
148 * Gets a pointer to the per-CPU instance of the variable for the specified
149 * base.
150 *
151 * @param base the per-CPU base to use
152 * @param name the per-CPU variable name
153 */
154 #define PERCPU_GET_WITH_BASE(base, name) \
155 __PERCPU_CAST(name, base + __PERCPU_ADDR(name))
156
157 /*!
158 * @macro PERCPU_GET_RELATIVE
159 *
160 * @abstract
161 * Gets a pointer to the per-CPU instance of a variable relative to another
162 * known one.
163 *
164 * @description
165 * When a per-CPU slot address is known, but the caller doesn't know the base
166 * from which it was derived, then this allows to compute another per-CPU slot
167 * address for a different variable but for the same CPU, without any loads.
168 *
169 * @param name the per-CPU variable name
170 * @param other the other per-CPU variable name
171 * @param ptr a pointer to the other variable slot
172 */
173 #define PERCPU_GET_RELATIVE(name, other, ptr) ({ \
174 __PERCPU_TYPE(other) __other_ptr = (ptr); /* type check */ \
175 vm_offset_t __offs = __PERCPU_ADDR(name) - __PERCPU_ADDR(other); \
176 __PERCPU_CAST(name, (vm_address_t)__other_ptr + __offs); \
177 })
178
179 /*!
180 * @macro percpu_foreach_base()
181 *
182 * @abstract
183 * Enumerates all Per-CPU variable bases.
184 *
185 * @param it the name of the iterator
186 */
187 #define percpu_foreach_base(it) \
188 for (vm_offset_t it = 0, \
189 __next_ ## it = percpu_base.start, \
190 __end_ ## it = percpu_base.end; \
191 \
192 it <= __end_ ## it; \
193 \
194 it = __next_ ## it, \
195 __next_ ## it += percpu_section_size())
196
197 /*!
198 * @macro percpu_foreach()
199 *
200 * @abstract
201 * Enumerates all Per-CPU variable instances.
202 *
203 * @param it the name of the iterator
204 * @param name the per-CPU variable name
205 */
206 #define percpu_foreach(it, name) \
207 for (__PERCPU_TYPE(name) it, \
208 __base_ ## it = NULL, \
209 __next_ ## it = (typeof(it))percpu_base.start, \
210 __end_ ## it = (typeof(it))percpu_base.end; \
211 \
212 (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
213 __base_ ## it <= __end_ ## it); \
214 \
215 __base_ ## it = __next_ ## it, \
216 __next_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
217
218 /*!
219 * @macro percpu_foreach_secondary_base()
220 *
221 * @abstract
222 * Enumerates all Per-CPU variable bases, skipping the master slot.
223 *
224 * @param it the name of the iterator
225 */
226 #define percpu_foreach_secondary_base(it) \
227 for (vm_offset_t it = percpu_base.start, __end_ ## it = percpu_base.end; \
228 it <= __end_ ## it; it += percpu_section_size())
229
230 /*!
231 * @macro percpu_foreach_secondary()
232 *
233 * @abstract
234 * Enumerates all Per-CPU variable instances, skipping the master slot.
235 *
236 * @param it the name of the iterator
237 * @param name the per-CPU variable name
238 */
239 #define percpu_foreach_secondary(it, name) \
240 for (__PERCPU_TYPE(name) it, \
241 __base_ ## it = (typeof(it))__unsafe_forge_bidi_indexable(void *, \
242 percpu_base.start, percpu_base.end - percpu_base.start), \
243 __end_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_base.end - percpu_base.start); \
244 \
245 (it = (typeof(it))((vm_address_t)__base_ ## it + __PERCPU_ADDR(name)), \
246 __base_ ## it <= __end_ ## it); \
247 \
248 __base_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
249
250 #pragma mark - implementation details
251
252 /*
253 * Below this point are implementation details that should not be used directly,
254 * except by the macros above, or architecture specific code.
255 */
256
257 #define __percpu __attribute__((section("__DATA, __percpu")))
258 #define __PERCPU_NAME(name) percpu_slot_ ## name
259 #define __PERCPU_ADDR(name) ((vm_offset_t)&__PERCPU_NAME(name))
260 #define __PERCPU_TYPE(name) typeof(&__PERCPU_NAME(name))
261 #define __PERCPU_CAST(name, expr) ((__PERCPU_TYPE(name))__unsafe_forge_bidi_indexable(void *, (expr), sizeof(__PERCPU_NAME(name))))
262
263 /*
264 * Note for implementors:
265 *
266 * A `base` represents a pointer in the percpu allocation offset by
267 * `percpu_section_start()` so that PERCPU_GET() is a single addition.
268 *
269 * percpu_base.end is inclusive, so that percpu_foreach() and
270 * percpu_foreach_base() can do a `<=` comparison.
271 *
272 * Because the first base is `0` (because the master CPU is using the static
273 * percpu section), it allows for the compiler to know that for the first
274 * iteration the comparison is always true.
275 */
276 extern struct percpu_base {
277 vm_address_t start;
278 vm_address_t end;
279 vm_offset_t size;
280 } percpu_base;
281
282 static __pure2 inline vm_offset_t
percpu_section_start(void)283 percpu_section_start(void)
284 {
285 extern char __percpu_section_start[0] __SECTION_START_SYM("__DATA", "__percpu");
286 return (vm_offset_t)__percpu_section_start;
287 }
288
289 static __pure2 inline vm_offset_t
percpu_section_end(void)290 percpu_section_end(void)
291 {
292 extern char __percpu_section_end[0] __SECTION_END_SYM("__DATA", "__percpu");
293 return (vm_offset_t)__percpu_section_end;
294 }
295
296 static __pure2 inline vm_size_t
percpu_section_size(void)297 percpu_section_size(void)
298 {
299 /**
300 * TODO: remove page rounding once we have a linker construct that gives us the correct page-padded size
301 * See rdar://problem/97665399.
302 */
303 return ((percpu_section_end() - percpu_section_start()) + PAGE_MASK) & ~((vm_size_t)PAGE_MASK);
304 }
305
306 #pragma GCC visibility pop
307 #endif /* XNU_KERNEL_PRIVATE */
308
309 __END_DECLS
310
311 #endif /* _KERN_PERCPU_H_ */
312