1 /*
2 * Copyright (c) 2000-2017 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _PTHREAD_PRIORITY_PRIVATE_H_
30 #define _PTHREAD_PRIORITY_PRIVATE_H_
31
32 #if KERNEL
33 #define PTHREAD_EXPOSE_LAYOUT 1
34 #else
35 #include <TargetConditionals.h>
36 #if TARGET_OS_SIMULATOR
37 #define PTHREAD_EXPOSE_LAYOUT 0
38 #else
39 #define PTHREAD_EXPOSE_LAYOUT 1
40 #endif /* TARGET_OS_SIMULATOR */
41 #endif
42
43 /*!
44 * @typedef pthread_priority_t
45 *
46 * @abstract
47 * pthread_priority_t is an on opaque integer that is guaranteed to be ordered
48 * such that combations of QoS classes and relative priorities are ordered
49 * numerically, according to their combined priority.
50 *
51 * <b>xnu, pthread & libdispatch flags</b>
52 *
53 * @const _PTHREAD_PRIORITY_OVERCOMMIT_FLAG
54 * The thread this priority is applied to is overcommit (affects the workqueue
55 * creation policy for this priority).
56 *
57 * @const _PTHREAD_PRIORITY_COOPERATIVE_FLAG
58 * Used to convey that a thread is part of the cooperative pool. This is used
59 * both outgoing form kernel and incoming into kernel
60 *
61 * @const _PTHREAD_PRIORITY_THREAD_TYPE_MASK
62 * The set of bits that encode information about the thread type - whether it is
63 * overcommit, non-overcommit or cooperative
64 *
65 * @const _PTHREAD_PRIORITY_FALLBACK_FLAG
66 * Indicates that this priority is is used only when incoming events have no
67 * priority at all. It is merely used as a fallback (hence the name) instead of
68 * a floor.
69 *
70 * This is usually used with QOS_CLASS_DEFAULT and a 0 relative priority.
71 *
72 * @const _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
73 * The event manager flag indicates that this thread/request is for a event
74 * manager thread. There can only ever be one event manager thread at a time
75 * and it is brought up at the highest of all event manager priorities pthread
76 * knows about.
77 *
78 * @const _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG
79 * This flag indicates that the bits extracted using
80 * _PTHREAD_PRIORITY_QOS_CLASS_MASK represent { QoS override, req QoS } instead
81 * of just req QoS. This is only currently only used as input to the kernel as
82 * part of pthread_set_properties_self(). The override field here represents the
83 * dispatch workqueue override.
84 *
85 * @const _PTHREAD_PRIORITY_SCHED_PRI_FLAG
86 * @const _PTHREAD_PRIORITY_SCHED_PRI_MASK
87 * This flag indicates that the bits extracted using
88 * _PTHREAD_PRIORITY_SCHED_PRI_MASK represent a scheduler priority instead of
89 * a {qos, relative priority} pair.
90 *
91 * This flag is used by the pthread kext to indicate to libdispatch that the
92 * event manager queue priority is a scheduling priority and not a QoS. When
93 * the manager thread's priority is updated due to creation of pthread root
94 * queues, libdispatch passed a pthread_priority_t in to kernel with this flag
95 * to specify the new sched pri of manager. This flag is never used as an input
96 * by anything else and is why it can perform a double duty with
97 * _PTHREAD_PRIORITY_ROOTQUEUE_FLAG.
98 *
99 * @const _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG
100 * This flag is used for the priority of event delivery threads to indicate
101 * to libdispatch that this thread is bound to a kqueue.
102 *
103 * <b>dispatch only flags</b>
104 *
105 * @const _PTHREAD_PRIORITY_INHERIT_FLAG
106 * This flag is meaningful to libdispatch only and has no meaning for the
107 * kernel and/or pthread.
108 *
109 * @const _PTHREAD_PRIORITY_ROOTQUEUE_FLAG
110 * This flag is meaningful to libdispatch only and has no meaning for the
111 * kernel and/or pthread.
112 *
113 * @const _PTHREAD_PRIORITY_ENFORCE_FLAG
114 * This flag is used to indicate that this priority should be prefered for work
115 * submited asynchronously over the intrinsic priority of the queue/thread the
116 * work is submitted to.
117 *
118 *
119 * pthread_priority_t encoding - outgoing from kernel:
120 *
121 * Regular:
122 * flags req QoS class Rel pri
123 * |---------------------------------|--------------------|--------------------|
124 * 22 - 31 8-21 0-7
125 *
126 * With _PTHREAD_PRIORITY_SCHED_PRI_FLAG:
127 *
128 * flags unused sched priority
129 * |---------------------------------|----------|------------------------------|
130 * 22 - 31 16-21 0-15
131 *
132 * pthread_priority_t encoding - incoming to kernel via various syscalls:
133 *
134 * Regular:
135 *
136 * flags req QoS class Rel pri
137 * |---------------------------------|--------------------|--------------------|
138 * 22 - 31 8-21 0-7
139 *
140 * With _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG:
141 *
142 * flags QoS ovr QoS class Rel pri
143 * |---------------------------------|---------|----------|--------------------|
144 * 22 - 31 14-21 8-13 0-7
145 *
146 * With _PTHREAD_PRIORITY_SCHED_PRI_FLAG:
147 *
148 * flags unused sched priority
149 * |---------------------------------|----------|------------------------------|
150 * 22 - 31 16-21 0-15
151 */
152 typedef unsigned long pthread_priority_t;
153
154 #define _PTHREAD_PRIORITY_OVERCOMMIT_FLAG 0x80000000u
155 #define _PTHREAD_PRIORITY_INHERIT_FLAG 0x40000000u /* dispatch only */
156 #define _PTHREAD_PRIORITY_ROOTQUEUE_FLAG 0x20000000u /* dispatch only */
157 #define _PTHREAD_PRIORITY_SCHED_PRI_FLAG 0x20000000u
158 #define _PTHREAD_PRIORITY_ENFORCE_FLAG 0x10000000u /* dispatch only */
159 #define _PTHREAD_PRIORITY_FALLBACK_FLAG 0x04000000u
160 #define _PTHREAD_PRIORITY_COOPERATIVE_FLAG 0x08000000u
161 #define _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG 0x02000000u
162 #define _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG 0x01000000u
163 #define _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG _PTHREAD_PRIORITY_FALLBACK_FLAG // compat
164 #define _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG 0x00800000u
165
166 #define _PTHREAD_PRIORITY_THREAD_TYPE_MASK (_PTHREAD_PRIORITY_COOPERATIVE_FLAG | _PTHREAD_PRIORITY_OVERCOMMIT_FLAG)
167
168 #if PTHREAD_EXPOSE_LAYOUT || defined(__PTHREAD_EXPOSE_INTERNALS__)
169 // Masks for encoding of pthread priority
170 #define _PTHREAD_PRIORITY_FLAGS_MASK 0xff000000u
171
172 #define _PTHREAD_PRIORITY_SCHED_PRI_MASK 0x0000ffffu
173
174 #define _PTHREAD_PRIORITY_QOS_CLASS_MASK 0x003fff00u
175 #define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT (8ull)
176 #define _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK 0x00003f00u
177 #define _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK 0x003fc000u
178 #define _PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT (14ull)
179
180 #define _PTHREAD_PRIORITY_PRIORITY_MASK 0x000000ffu
181 #define _PTHREAD_PRIORITY_PRIORITY_SHIFT (0)
182 #endif /* PTHREAD_EXPOSE_LAYOUT */
183
184 #if PRIVATE
185 #if XNU_KERNEL_PRIVATE && !defined(__PTHREAD_EXPOSE_INTERNALS__)
186 #define __PTHREAD_EXPOSE_INTERNALS__ 1
187 #endif // XNU_KERNEL_PRIVATE
188 #ifdef __PTHREAD_EXPOSE_INTERNALS__
189 /*
190 * This exposes the encoding used for pthread_priority_t
191 * and is meant to be used by pthread and XNU only
192 */
193 #include <mach/thread_policy.h> // THREAD_QOS_*
194 #include <stdbool.h>
195
196 // pthread_priority_t's type is unfortunately 64bits on LP64
197 // so we use this type for people who need to store it in structs
198 typedef unsigned int pthread_priority_compact_t;
199
200 __attribute__((always_inline, const))
201 static inline bool
_pthread_priority_has_qos(pthread_priority_t pp)202 _pthread_priority_has_qos(pthread_priority_t pp)
203 {
204 return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG |
205 _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 &&
206 (pp & _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK) != 0;
207 }
208
209 __attribute__((always_inline, const))
210 static inline bool
_pthread_priority_has_sched_pri(pthread_priority_t pp)211 _pthread_priority_has_sched_pri(pthread_priority_t pp)
212 {
213 return pp & _PTHREAD_PRIORITY_SCHED_PRI_FLAG;
214 }
215
216 __attribute__((always_inline, const))
217 static inline bool
_pthread_priority_has_override_qos(pthread_priority_t pp)218 _pthread_priority_has_override_qos(pthread_priority_t pp)
219 {
220 return (pp & (_PTHREAD_PRIORITY_SCHED_PRI_FLAG |
221 _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) == 0 &&
222 (pp & _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG) != 0 &&
223 (pp & _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK) != 0;
224 }
225
226 __attribute__((always_inline, const))
227 static inline pthread_priority_compact_t
_pthread_priority_make_from_thread_qos(thread_qos_t qos,int relpri,unsigned long flags)228 _pthread_priority_make_from_thread_qos(thread_qos_t qos, int relpri,
229 unsigned long flags)
230 {
231 pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK);
232 if (qos && qos < THREAD_QOS_LAST) {
233 pp |= (1 << (_PTHREAD_PRIORITY_QOS_CLASS_SHIFT + qos - 1));
234 pp |= ((uint8_t)relpri - 1) & _PTHREAD_PRIORITY_PRIORITY_MASK;
235 }
236 return pp;
237 }
238
239 __attribute__((always_inline, const))
240 static inline pthread_priority_compact_t
_pthread_priority_make_from_sched_pri(int sched_pri,unsigned long flags)241 _pthread_priority_make_from_sched_pri(int sched_pri, unsigned long flags)
242 {
243 pthread_priority_compact_t pp = (flags & _PTHREAD_PRIORITY_FLAGS_MASK);
244 pp |= _PTHREAD_PRIORITY_SCHED_PRI_FLAG;
245 pp |= (pthread_priority_compact_t) sched_pri;
246
247 return pp;
248 }
249
250 __attribute__((always_inline, const))
251 static inline pthread_priority_compact_t
_pthread_priority_make_from_thread_qos_and_override(thread_qos_t req_qos,int relpri,thread_qos_t override_qos,unsigned long flags)252 _pthread_priority_make_from_thread_qos_and_override(thread_qos_t req_qos,
253 int relpri, thread_qos_t override_qos, unsigned long flags)
254 {
255 pthread_priority_compact_t pp;
256 pp = _pthread_priority_make_from_thread_qos(req_qos, relpri, flags);
257
258 if (override_qos && override_qos < THREAD_QOS_LAST) {
259 pp |= (1 << (_PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT + (override_qos - 1)));
260 pp |= _PTHREAD_PRIORITY_OVERRIDE_QOS_FLAG;
261 }
262
263 return pp;
264 }
265
266 __attribute__((always_inline, const))
267 static inline pthread_priority_compact_t
_pthread_event_manager_priority(void)268 _pthread_event_manager_priority(void)
269 {
270 return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
271 }
272
273 __attribute__((always_inline, const))
274 static inline pthread_priority_compact_t
_pthread_unspecified_priority(void)275 _pthread_unspecified_priority(void)
276 {
277 return _pthread_priority_make_from_thread_qos(THREAD_QOS_UNSPECIFIED, 0, 0);
278 }
279
280 __attribute__((always_inline, const))
281 static inline pthread_priority_compact_t
_pthread_default_priority(unsigned long flags)282 _pthread_default_priority(unsigned long flags)
283 {
284 return _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, 0, flags);
285 }
286
287 __attribute__((always_inline, const))
288 static inline thread_qos_t
_pthread_priority_thread_qos_fast(pthread_priority_t pp)289 _pthread_priority_thread_qos_fast(pthread_priority_t pp)
290 {
291 pp &= _PTHREAD_PRIORITY_VALID_QOS_CLASS_MASK;
292 pp >>= _PTHREAD_PRIORITY_QOS_CLASS_SHIFT;
293 return (thread_qos_t)__builtin_ffs((int)pp);
294 }
295
296 __attribute__((always_inline, const))
297 static inline thread_qos_t
_pthread_priority_thread_override_qos_fast(pthread_priority_t pp)298 _pthread_priority_thread_override_qos_fast(pthread_priority_t pp)
299 {
300 pp &= _PTHREAD_PRIORITY_VALID_OVERRIDE_QOS_MASK;
301 pp >>= _PTHREAD_PRIORITY_QOS_OVERRIDE_SHIFT;
302 return (thread_qos_t)__builtin_ffs((int)pp);
303 }
304
305 __attribute__((always_inline, const))
306 static inline int
_pthread_priority_sched_pri_fast(pthread_priority_t pp)307 _pthread_priority_sched_pri_fast(pthread_priority_t pp)
308 {
309 return pp & _PTHREAD_PRIORITY_SCHED_PRI_MASK;
310 }
311
312 __attribute__((always_inline, const))
313 static inline thread_qos_t
_pthread_priority_thread_qos(pthread_priority_t pp)314 _pthread_priority_thread_qos(pthread_priority_t pp)
315 {
316 if (_pthread_priority_has_qos(pp)) {
317 return _pthread_priority_thread_qos_fast(pp);
318 }
319 return THREAD_QOS_UNSPECIFIED;
320 }
321
322 __attribute__((always_inline, const))
323 static inline thread_qos_t
_pthread_priority_thread_override_qos(pthread_priority_t pp)324 _pthread_priority_thread_override_qos(pthread_priority_t pp)
325 {
326 if (_pthread_priority_has_override_qos(pp)) {
327 return _pthread_priority_thread_override_qos_fast(pp);
328 }
329 return THREAD_QOS_UNSPECIFIED;
330 }
331
332 __attribute__((always_inline, const))
333 static inline int
_pthread_priority_sched_pri(pthread_priority_t pp)334 _pthread_priority_sched_pri(pthread_priority_t pp)
335 {
336 if (_pthread_priority_has_sched_pri(pp)) {
337 return _pthread_priority_sched_pri_fast(pp);
338 }
339
340 return 0;
341 }
342
343 __attribute__((always_inline, const))
344 static inline int
_pthread_priority_relpri(pthread_priority_t pp)345 _pthread_priority_relpri(pthread_priority_t pp)
346 {
347 if (_pthread_priority_has_qos(pp)) {
348 pp &= _PTHREAD_PRIORITY_PRIORITY_MASK;
349 pp >>= _PTHREAD_PRIORITY_PRIORITY_SHIFT;
350 return (int8_t)pp + 1;
351 }
352 return 0;
353 }
354
355 __attribute__((always_inline, const))
356 static inline bool
_pthread_priority_is_overcommit(pthread_priority_t pp)357 _pthread_priority_is_overcommit(pthread_priority_t pp)
358 {
359 return pp & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
360 }
361
362 __attribute__((always_inline, const))
363 static inline bool
_pthread_priority_is_cooperative(pthread_priority_t pp)364 _pthread_priority_is_cooperative(pthread_priority_t pp)
365 {
366 return pp & _PTHREAD_PRIORITY_COOPERATIVE_FLAG;
367 }
368
369 __attribute__((always_inline, const))
370 static inline bool
_pthread_priority_is_nonovercommit(pthread_priority_t pp)371 _pthread_priority_is_nonovercommit(pthread_priority_t pp)
372 {
373 return !_pthread_priority_is_cooperative(pp) && !_pthread_priority_is_overcommit(pp);
374 }
375
376 #if XNU_KERNEL_PRIVATE
377 // Interfaces only used by the kernel and not implemented in userspace.
378
379 /*
380 * Keep managerness, overcomitness and fallback, discard other flags.
381 * Normalize and validate QoS/relpri
382 */
383 __attribute__((const))
384 pthread_priority_compact_t
385 _pthread_priority_normalize(pthread_priority_t pp);
386
387 /*
388 * Keep managerness, discard other flags.
389 * Normalize and validate QoS/relpri
390 */
391 __attribute__((const))
392 pthread_priority_compact_t
393 _pthread_priority_normalize_for_ipc(pthread_priority_t pp);
394
395 /*
396 * Keep the flags from base_pp and return the priority with the maximum priority
397 * of base_pp and _pthread_priority_make_from_thread_qos(qos, 0, 0)
398 */
399 __attribute__((const))
400 pthread_priority_compact_t
401 _pthread_priority_combine(pthread_priority_t base_pp, thread_qos_t qos);
402
403
404 #endif // XNU_KERNEL_PRIVATE
405 #endif // __PTHREAD_EXPOSE_INTERNALS__
406 #endif // PRIVATE
407 #endif // _PTHREAD_PRIORITY_PRIVATE_H_
408