1 /* 2 * Copyright (c) 2014 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _WORKQUEUE_INTERNAL_H_ 30 #define _WORKQUEUE_INTERNAL_H_ 31 32 // Sometimes something gets passed a bucket number and we need a way to express 33 // that it's actually the event manager. Use the (0)th bucket for that. 34 #define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE) 35 #define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST) 36 #define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY) 37 #define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST) 38 #define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX 39 40 #define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared 41 #define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr 42 43 /* These definitions are only available to the kext, to avoid bleeding 44 * constants and types across the boundary to the userspace library. 45 */ 46 #ifdef KERNEL 47 #pragma mark wq structs 48 49 /* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get 50 * exported to kernel extensions. 51 */ 52 #define SCHED_CALL_BLOCK 0x1 53 #define SCHED_CALL_UNBLOCK 0x2 54 55 /* old workq priority scheme */ 56 57 #define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */ 58 #define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */ 59 #define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */ 60 #define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */ 61 62 /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR) 63 * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding 64 * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should 65 * not change. So the factor is now always 5. 66 */ 67 #define WORKQUEUE_CONSTRAINED_FACTOR 5 68 69 #if BSD_KERNEL_PRIVATE 70 #include <kern/mpsc_queue.h> 71 #include <kern/priority_queue.h> 72 #include <kern/thread_call.h> 73 #include <kern/turnstile.h> 74 #include <mach/kern_return.h> 75 #include <sys/queue.h> 76 #include <sys/kernel_types.h> 77 78 /* struct uthread::uu_workq_flags */ 79 #define UT_WORKQ_NEW 0x01 /* First return to userspace */ 80 #define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */ 81 #define UT_WORKQ_DYING 0x04 /* Thread is being killed */ 82 #define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */ 83 #define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */ 84 #define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */ 85 #define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */ 86 #define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */ 87 #define UT_WORKQ_COOPERATIVE 0x100 /* Thread is part of cooperative pool */ 88 89 typedef union workq_threadreq_param_s { 90 struct { 91 uint16_t trp_flags; 92 uint8_t trp_pri; 93 uint8_t trp_pol; 94 uint32_t trp_cpupercent: 8, 95 trp_refillms: 24; 96 }; 97 uint64_t trp_value; 98 } workq_threadreq_param_t; 99 100 #define TRP_PRIORITY 0x1 101 #define TRP_POLICY 0x2 102 #define TRP_CPUPERCENT 0x4 103 #define TRP_RELEASED 0x8000 104 105 /*! 106 * @enum workq_tr_state_t 107 * 108 * @brief 109 * This enum represents the state of a workq thread request. 110 * 111 * @discussion 112 * The states are used and set by both kevent and the workq subsystem under very 113 * precise locking domains. 114 * 115 * When for kevent requests, this structure is embedded on the kqueue itself, 116 * for non kevent related thread requests, it is allocated. 117 * 118 * Only the BINDING state isn't set under the kqlock, but then only QUEUED could 119 * be read by kqueue in its stead. 120 * 121 * @const WORKQ_TR_STATE_IDLE 122 * This thread request is idle. 123 * The state is only transient for non kevent thread requests. 124 * Set under the kqlock (kevent) or after allocation (workq). 125 * 126 * tr_entry/tr_thread are unused. 127 * 128 * @const WORKQ_TR_STATE_NEW 129 * This thread request is being initialized. This state is transient. 130 * Set workq lock for all kinds, set under the kqlock to for kevent requests. 131 * 132 * tr_entry is initialized, tr_thread is unused. 133 * 134 * @const WORKQ_TR_STATE_QUEUED 135 * This thread request has been pended, waiting for a thread to be bound. 136 * Set workq lock for all kinds, set under the kqlock to for kevent requests. 137 * 138 * tr_entry is used as linkage in a workq priority queue, tr_thread is unused. 139 * 140 * @const WORKQ_TR_STATE_CANCELED 141 * When the process exits, Queued thread requests are marked canceled. 142 * This happens under the workqueue lock. 143 * 144 * @const WORKQ_TR_STATE_BINDING (kevent only) 145 * A thread was found to bind to the thread request. 146 * The bind is preposted this way under the workq lock and will be 147 * acknowledged by the kevent subsystem. 148 * 149 * tr_entry is unused, tr_thread is the thread we're binding to. 150 * 151 * @const WORKQ_TR_STATE_BOUND (kevent only) 152 * A thread bind has been acknowledged by the kevent subsystem. 153 * This is always set under the kqlock, sometimes also under the workq lock. 154 * 155 * tr_entry is unused, tr_thread is the thread we're bound to. 156 */ 157 __enum_decl(workq_tr_state_t, uint8_t, { 158 WORKQ_TR_STATE_IDLE = 0, /* request isn't in flight */ 159 WORKQ_TR_STATE_NEW = 1, /* request is being initiated */ 160 WORKQ_TR_STATE_QUEUED = 2, /* request is being queued */ 161 WORKQ_TR_STATE_CANCELED = 3, /* request is canceled */ 162 WORKQ_TR_STATE_BINDING = 4, /* request is preposted for bind */ 163 WORKQ_TR_STATE_BOUND = 5, /* request is bound to a thread */ 164 }); 165 166 __options_decl(workq_tr_flags_t, uint8_t, { 167 WORKQ_TR_FLAG_KEVENT = 0x01, 168 WORKQ_TR_FLAG_WORKLOOP = 0x02, 169 WORKQ_TR_FLAG_OVERCOMMIT = 0x04, 170 WORKQ_TR_FLAG_WL_PARAMS = 0x08, 171 WORKQ_TR_FLAG_WL_OUTSIDE_QOS = 0x10, 172 WORKQ_TR_FLAG_COOPERATIVE = 0x20, 173 }); 174 175 typedef struct workq_threadreq_s { 176 union { 177 struct priority_queue_entry_sched tr_entry; 178 STAILQ_ENTRY(workq_threadreq_s) tr_link; 179 thread_t tr_thread; 180 }; 181 uint16_t tr_count; 182 workq_tr_flags_t tr_flags; 183 workq_tr_state_t tr_state; 184 thread_qos_t tr_qos; /* qos for the thread request */ 185 186 /* kqueue states, modified under the kqlock */ 187 kq_index_t tr_kq_override_index; /* highest wakeup override index */ 188 kq_index_t tr_kq_qos_index; /* QoS for the servicer */ 189 } workq_threadreq_s, *workq_threadreq_t; 190 191 STAILQ_HEAD(workq_threadreq_tailq, workq_threadreq_s); 192 193 #if defined(__LP64__) 194 typedef unsigned __int128 wq_thactive_t; 195 #else 196 typedef uint64_t wq_thactive_t; 197 #endif 198 199 __options_decl(workq_state_flags_t, uint32_t, { 200 WQ_EXITING = 0x0001, 201 WQ_PROC_SUSPENDED = 0x0002, 202 WQ_DEATH_CALL_SCHEDULED = 0x0004, 203 204 WQ_DELAYED_CALL_SCHEDULED = 0x0010, 205 WQ_DELAYED_CALL_PENDED = 0x0020, 206 WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040, 207 WQ_IMMEDIATE_CALL_PENDED = 0x0080, 208 }); 209 210 TAILQ_HEAD(workq_uthread_head, uthread); 211 212 struct workqueue { 213 thread_call_t wq_delayed_call; 214 thread_call_t wq_immediate_call; 215 thread_call_t wq_death_call; 216 217 union { 218 struct turnstile *wq_turnstile; 219 struct mpsc_queue_chain wq_destroy_link; 220 }; 221 222 lck_ticket_t wq_lock; 223 224 uint64_t wq_thread_call_last_run; 225 struct os_refcnt wq_refcnt; 226 workq_state_flags_t _Atomic wq_flags; 227 uint32_t wq_fulfilled; 228 uint32_t wq_creations; 229 uint32_t wq_timer_interval; 230 uint32_t wq_event_manager_priority; 231 uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */ 232 uint16_t wq_thdying_count; 233 uint16_t wq_threads_scheduled; 234 uint16_t wq_constrained_threads_scheduled; 235 uint16_t wq_nthreads; 236 uint16_t wq_thidlecount; 237 uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager 238 239 workq_threadreq_t wq_event_manager_threadreq; 240 241 _Atomic wq_thactive_t wq_thactive; 242 _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS]; 243 244 struct proc *wq_proc; 245 struct uthread *wq_creator; 246 turnstile_inheritor_t wq_inheritor; 247 thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor 248 struct workq_uthread_head wq_thrunlist; 249 struct workq_uthread_head wq_thnewlist; 250 struct workq_uthread_head wq_thidlelist; 251 252 struct priority_queue_sched_max wq_overcommit_queue; 253 struct priority_queue_sched_max wq_constrained_queue; 254 struct priority_queue_sched_max wq_special_queue; 255 256 // BG/MT, UT, DEF, IN, UI, AUI. No manager bucket for cooperative pool 257 uint8_t wq_cooperative_queue_scheduled_count[WORKQ_NUM_QOS_BUCKETS]; 258 uint16_t wq_cooperative_queue_best_req_qos: 3, /* UN means no request, returns BG for BG/MT bucket */ 259 wq_cooperative_queue_has_limited_max_size:1, /* if set, max size of cooperative pool per QoS is 1 */ 260 unused:12; 261 struct workq_threadreq_tailq wq_cooperative_queue[WORKQ_NUM_QOS_BUCKETS]; 262 }; 263 264 #define WORKQUEUE_MAXTHREADS 512 265 #define WQ_STALLED_WINDOW_USECS 200 266 #define WQ_REDUCE_POOL_WINDOW_USECS 5000000 267 #define WQ_MAX_TIMER_INTERVAL_USECS 50000 268 269 #pragma mark definitions 270 271 struct workq_threadreq_s; 272 uint32_t _get_pwq_state_kdp(proc_t p); 273 274 void workq_exit(struct proc *p); 275 void workq_mark_exiting(struct proc *p); 276 277 bool workq_is_exiting(struct proc *p); 278 279 void workq_thread_set_max_qos(struct proc *p, struct workq_threadreq_s *kqr); 280 281 void workq_thread_terminate(struct proc *p, struct uthread *uth); 282 283 __options_decl(workq_kern_threadreq_flags_t, uint32_t, { 284 WORKQ_THREADREQ_NONE = 0x00, 285 WORKQ_THREADREQ_SET_AST_ON_FAILURE = 0x01, 286 WORKQ_THREADREQ_ATTEMPT_REBIND = 0x02, 287 WORKQ_THREADREQ_CAN_CREATE_THREADS = 0x04, 288 WORKQ_THREADREQ_MAKE_OVERCOMMIT = 0x08, 289 #if CONFIG_PREADOPT_TG 290 WORKQ_THREADREQ_REEVALUATE_PREADOPT_TG = 0x10, 291 #endif 292 }); 293 294 // called with the kq req lock held 295 bool workq_kern_threadreq_initiate(struct proc *p, struct workq_threadreq_s *kqr, 296 struct turnstile *ts, thread_qos_t qos, workq_kern_threadreq_flags_t flags); 297 298 // called with the kq req lock held 299 void workq_kern_threadreq_modify(struct proc *p, struct workq_threadreq_s *kqr, 300 thread_qos_t qos, workq_kern_threadreq_flags_t flags); 301 302 // called with the kq req lock held 303 void workq_kern_threadreq_update_inheritor(struct proc *p, struct workq_threadreq_s *kqr, 304 thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags); 305 306 void workq_kern_threadreq_lock(struct proc *p); 307 void workq_kern_threadreq_unlock(struct proc *p); 308 309 void workq_kern_threadreq_redrive(struct proc *p, workq_kern_threadreq_flags_t flags); 310 311 void workq_kern_quantum_expiry_reevaluate(struct proc *p, thread_t thread); 312 bool bsdthread_part_of_cooperative_workqueue(struct uthread *uth); 313 314 // This enum matches _pthread_set_flags in libpthread's qos_private.h 315 enum workq_set_self_flags { 316 WORKQ_SET_SELF_QOS_FLAG = 0x01, 317 WORKQ_SET_SELF_VOUCHER_FLAG = 0x02, 318 WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x04, 319 WORKQ_SET_SELF_TIMESHARE_FLAG = 0x08, 320 WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10, 321 }; 322 323 void workq_proc_suspended(struct proc *p); 324 void workq_proc_resumed(struct proc *p); 325 struct workqueue *proc_get_wqptr(struct proc *p); 326 327 #endif // BSD_KERNEL_PRIVATE 328 329 void workq_init(void); 330 331 #endif // KERNEL 332 333 #endif // _WORKQUEUE_INTERNAL_H_ 334