1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <mach/mach_types.h>
29 #include <kern/assert.h>
30 #include <kern/clock.h>
31 #include <kern/coalition.h>
32 #include <kern/debug.h>
33 #include <kern/startup.h>
34 #include <kern/host.h>
35 #include <kern/kern_types.h>
36 #include <kern/machine.h>
37 #include <kern/simple_lock.h>
38 #include <kern/misc_protos.h>
39 #include <kern/sched.h>
40 #include <kern/sched_prim.h>
41 #include <kern/sfi.h>
42 #include <kern/timer_call.h>
43 #include <kern/waitq.h>
44 #include <kern/ledger.h>
45 #include <kern/policy_internal.h>
46
47 #include <machine/atomic.h>
48
49 #include <pexpert/pexpert.h>
50
51 #include <libkern/kernel_mach_header.h>
52
53 #include <sys/kdebug.h>
54
55 #if CONFIG_SCHED_SFI
56
57 #define SFI_DEBUG 0
58
59 #if SFI_DEBUG
60 #define dprintf(...) kprintf(__VA_ARGS__)
61 #else
62 #define dprintf(...) do { } while(0)
63 #endif
64
65 /*
66 * SFI (Selective Forced Idle) operates by enabling a global
67 * timer on the SFI window interval. When it fires, all processors
68 * running a thread that should be SFI-ed are sent an AST.
69 * As threads become runnable while in their "off phase", they
70 * are placed on a deferred ready queue. When a per-class
71 * "on timer" fires, the ready threads for that class are
72 * re-enqueued for running. As an optimization to avoid spurious
73 * wakeups, the timer may be lazily programmed.
74 */
75
76 /*
77 * The "sfi_lock" simple lock guards access to static configuration
78 * parameters (as specified by userspace), dynamic state changes
79 * (as updated by the timer event routine), and timer data structures.
80 * Since it can be taken with interrupts disabled in some cases, all
81 * uses should be taken with interrupts disabled at splsched(). The
82 * "sfi_lock" also guards the "sfi_wait_class" field of thread_t, and
83 * must only be accessed with it held.
84 *
85 * When an "on timer" fires, we must deterministically be able to drain
86 * the wait queue, since if any threads are added to the queue afterwards,
87 * they may never get woken out of SFI wait. So sfi_lock must be
88 * taken before the wait queue's own spinlock.
89 *
90 * The wait queue will take the thread's scheduling lock. We may also take
91 * the thread_lock directly to update the "sfi_class" field and determine
92 * if the thread should block in the wait queue, but the lock will be
93 * released before doing so.
94 *
95 * The pset lock may also be taken, but not while any other locks are held.
96 *
97 * The task and thread mutex may also be held while reevaluating sfi state.
98 *
99 * splsched ---> sfi_lock ---> waitq ---> thread_lock
100 * \ \ \__ thread_lock (*)
101 * \ \__ pset_lock
102 * \
103 * \__ thread_lock
104 */
105
106 decl_simple_lock_data(static, sfi_lock);
107 static timer_call_data_t sfi_timer_call_entry;
108 volatile boolean_t sfi_is_enabled;
109
110 boolean_t sfi_window_is_set;
111 uint64_t sfi_window_usecs;
112 uint64_t sfi_window_interval;
113 uint64_t sfi_next_off_deadline;
114
115 typedef struct {
116 sfi_class_id_t class_id;
117 thread_continue_t class_continuation;
118 const char * class_name;
119 const char * class_ledger_name;
120 } sfi_class_registration_t;
121
122 /*
123 * To add a new SFI class:
124 *
125 * 1) Raise MAX_SFI_CLASS_ID in mach/sfi_class.h
126 * 2) Add a #define for it to mach/sfi_class.h. It need not be inserted in order of restrictiveness.
127 * 3) Add a call to SFI_CLASS_REGISTER below
128 * 4) Augment sfi_thread_classify to categorize threads as early as possible for as restrictive as possible.
129 * 5) Modify thermald to use the SFI class
130 */
131
132 static inline void _sfi_wait_cleanup(void);
133
134 static void sfi_class_register(sfi_class_registration_t *);
135
136 #define SFI_CLASS_REGISTER(clsid, ledger_name) \
137 \
138 static void __attribute__((noinline, noreturn)) \
139 SFI_ ## clsid ## _THREAD_IS_WAITING(void *arg __unused, wait_result_t wret __unused) \
140 { \
141 _sfi_wait_cleanup(); \
142 thread_exception_return(); \
143 } \
144 \
145 static_assert(SFI_CLASS_ ## clsid < MAX_SFI_CLASS_ID, "Invalid ID"); \
146 \
147 static __startup_data sfi_class_registration_t \
148 SFI_ ## clsid ## _registration = { \
149 .class_id = SFI_CLASS_ ## clsid, \
150 .class_continuation = SFI_ ## clsid ## _THREAD_IS_WAITING, \
151 .class_name = "SFI_CLASS_" # clsid, \
152 .class_ledger_name = "SFI_CLASS_" # ledger_name, \
153 }; \
154 STARTUP_ARG(TUNABLES, STARTUP_RANK_MIDDLE, \
155 sfi_class_register, &SFI_ ## clsid ## _registration)
156
157 /* SFI_CLASS_UNSPECIFIED not included here */
158 SFI_CLASS_REGISTER(MAINTENANCE, MAINTENANCE);
159 SFI_CLASS_REGISTER(DARWIN_BG, DARWIN_BG);
160 SFI_CLASS_REGISTER(APP_NAP, APP_NAP);
161 SFI_CLASS_REGISTER(MANAGED_FOCAL, MANAGED);
162 SFI_CLASS_REGISTER(MANAGED_NONFOCAL, MANAGED);
163 SFI_CLASS_REGISTER(UTILITY, UTILITY);
164 SFI_CLASS_REGISTER(DEFAULT_FOCAL, DEFAULT);
165 SFI_CLASS_REGISTER(DEFAULT_NONFOCAL, DEFAULT);
166 SFI_CLASS_REGISTER(LEGACY_FOCAL, LEGACY);
167 SFI_CLASS_REGISTER(LEGACY_NONFOCAL, LEGACY);
168 SFI_CLASS_REGISTER(USER_INITIATED_FOCAL, USER_INITIATED);
169 SFI_CLASS_REGISTER(USER_INITIATED_NONFOCAL, USER_INITIATED);
170 SFI_CLASS_REGISTER(USER_INTERACTIVE_FOCAL, USER_INTERACTIVE);
171 SFI_CLASS_REGISTER(USER_INTERACTIVE_NONFOCAL, USER_INTERACTIVE);
172 SFI_CLASS_REGISTER(KERNEL, OPTED_OUT);
173 SFI_CLASS_REGISTER(OPTED_OUT, OPTED_OUT);
174
175 struct sfi_class_state {
176 uint64_t off_time_usecs;
177 uint64_t off_time_interval;
178
179 timer_call_data_t on_timer;
180 uint64_t on_timer_deadline;
181 boolean_t on_timer_programmed;
182
183 boolean_t class_sfi_is_enabled;
184 volatile boolean_t class_in_on_phase;
185
186 struct waitq waitq; /* threads in ready state */
187 thread_continue_t continuation;
188
189 const char * class_name;
190 const char * class_ledger_name;
191 };
192
193 /* Static configuration performed in sfi_early_init() */
194 struct sfi_class_state sfi_classes[MAX_SFI_CLASS_ID];
195
196 int sfi_enabled_class_count; // protected by sfi_lock and used atomically
197
198 static void sfi_timer_global_off(
199 timer_call_param_t param0,
200 timer_call_param_t param1);
201
202 static void sfi_timer_per_class_on(
203 timer_call_param_t param0,
204 timer_call_param_t param1);
205
206 /* Called early in boot, when kernel is single-threaded */
207 __startup_func
208 static void
sfi_class_register(sfi_class_registration_t * reg)209 sfi_class_register(sfi_class_registration_t *reg)
210 {
211 sfi_class_id_t class_id = reg->class_id;
212
213 if (class_id >= MAX_SFI_CLASS_ID) {
214 panic("Invalid SFI class 0x%x", class_id);
215 }
216 if (sfi_classes[class_id].continuation != NULL) {
217 panic("Duplicate SFI registration for class 0x%x", class_id);
218 }
219 sfi_classes[class_id].class_sfi_is_enabled = FALSE;
220 sfi_classes[class_id].class_in_on_phase = TRUE;
221 sfi_classes[class_id].continuation = reg->class_continuation;
222 sfi_classes[class_id].class_name = reg->class_name;
223 sfi_classes[class_id].class_ledger_name = reg->class_ledger_name;
224 }
225
226 void
sfi_init(void)227 sfi_init(void)
228 {
229 sfi_class_id_t i;
230
231 simple_lock_init(&sfi_lock, 0);
232 timer_call_setup(&sfi_timer_call_entry, sfi_timer_global_off, NULL);
233 sfi_window_is_set = FALSE;
234 os_atomic_init(&sfi_enabled_class_count, 0);
235 sfi_is_enabled = FALSE;
236
237 for (i = 0; i < MAX_SFI_CLASS_ID; i++) {
238 /* If the class was set up in sfi_early_init(), initialize remaining fields */
239 if (sfi_classes[i].continuation) {
240 timer_call_setup(&sfi_classes[i].on_timer, sfi_timer_per_class_on, (void *)(uintptr_t)i);
241 sfi_classes[i].on_timer_programmed = FALSE;
242
243 waitq_init(&sfi_classes[i].waitq, WQT_QUEUE, SYNC_POLICY_FIFO);
244 } else {
245 /* The only allowed gap is for SFI_CLASS_UNSPECIFIED */
246 if (i != SFI_CLASS_UNSPECIFIED) {
247 panic("Gap in registered SFI classes");
248 }
249 }
250 }
251 }
252
253 /* Can be called before sfi_init() by task initialization, but after sfi_early_init() */
254 sfi_class_id_t
sfi_get_ledger_alias_for_class(sfi_class_id_t class_id)255 sfi_get_ledger_alias_for_class(sfi_class_id_t class_id)
256 {
257 sfi_class_id_t i;
258 const char *ledger_name = NULL;
259
260 ledger_name = sfi_classes[class_id].class_ledger_name;
261
262 /* Find the first class in the registration table with this ledger name */
263 if (ledger_name) {
264 for (i = SFI_CLASS_UNSPECIFIED + 1; i < class_id; i++) {
265 if (0 == strcmp(sfi_classes[i].class_ledger_name, ledger_name)) {
266 dprintf("sfi_get_ledger_alias_for_class(0x%x) -> 0x%x\n", class_id, i);
267 return i;
268 }
269 }
270
271 /* This class is the primary one for the ledger, so there is no alias */
272 dprintf("sfi_get_ledger_alias_for_class(0x%x) -> 0x%x\n", class_id, SFI_CLASS_UNSPECIFIED);
273 return SFI_CLASS_UNSPECIFIED;
274 }
275
276 /* We are permissive on SFI class lookup failures. In sfi_init(), we assert more */
277 return SFI_CLASS_UNSPECIFIED;
278 }
279
280 int
sfi_ledger_entry_add(ledger_template_t template,sfi_class_id_t class_id)281 sfi_ledger_entry_add(ledger_template_t template, sfi_class_id_t class_id)
282 {
283 const char *ledger_name = NULL;
284
285 ledger_name = sfi_classes[class_id].class_ledger_name;
286
287 dprintf("sfi_ledger_entry_add(%p, 0x%x) -> %s\n", template, class_id, ledger_name);
288 return ledger_entry_add(template, ledger_name, "sfi", "MATUs");
289 }
290
291 static void
sfi_timer_global_off(timer_call_param_t param0 __unused,timer_call_param_t param1 __unused)292 sfi_timer_global_off(
293 timer_call_param_t param0 __unused,
294 timer_call_param_t param1 __unused)
295 {
296 uint64_t now = mach_absolute_time();
297 sfi_class_id_t i;
298 processor_set_t pset, nset;
299 processor_t processor;
300 uint32_t needs_cause_ast_mask = 0x0;
301 spl_t s;
302
303 s = splsched();
304
305 simple_lock(&sfi_lock, LCK_GRP_NULL);
306 if (!sfi_is_enabled) {
307 /* If SFI has been disabled, let all "on" timers drain naturally */
308 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_OFF_TIMER) | DBG_FUNC_NONE, 1, 0, 0, 0, 0);
309
310 simple_unlock(&sfi_lock);
311 splx(s);
312 return;
313 }
314
315 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_OFF_TIMER) | DBG_FUNC_START, 0, 0, 0, 0, 0);
316
317 /* First set all configured classes into the off state, and program their "on" timer */
318 for (i = 0; i < MAX_SFI_CLASS_ID; i++) {
319 if (sfi_classes[i].class_sfi_is_enabled) {
320 uint64_t on_timer_deadline;
321
322 sfi_classes[i].class_in_on_phase = FALSE;
323 sfi_classes[i].on_timer_programmed = TRUE;
324
325 /* Push out on-timer */
326 on_timer_deadline = now + sfi_classes[i].off_time_interval;
327 sfi_classes[i].on_timer_deadline = on_timer_deadline;
328
329 timer_call_enter1(&sfi_classes[i].on_timer, NULL, on_timer_deadline, TIMER_CALL_SYS_CRITICAL);
330 } else {
331 /* If this class no longer needs SFI, make sure the timer is cancelled */
332 sfi_classes[i].class_in_on_phase = TRUE;
333 if (sfi_classes[i].on_timer_programmed) {
334 sfi_classes[i].on_timer_programmed = FALSE;
335 sfi_classes[i].on_timer_deadline = ~0ULL;
336 timer_call_cancel(&sfi_classes[i].on_timer);
337 }
338 }
339 }
340 simple_unlock(&sfi_lock);
341
342 /* Iterate over processors, call cause_ast_check() on ones running a thread that should be in an off phase */
343 processor = processor_list;
344 pset = processor->processor_set;
345
346 pset_lock(pset);
347
348 do {
349 nset = processor->processor_set;
350 if (nset != pset) {
351 pset_unlock(pset);
352 pset = nset;
353 pset_lock(pset);
354 }
355
356 /* "processor" and its pset are locked */
357 if (processor->state == PROCESSOR_RUNNING) {
358 if (AST_NONE != sfi_processor_needs_ast(processor)) {
359 needs_cause_ast_mask |= (1U << processor->cpu_id);
360 }
361 }
362 } while ((processor = processor->processor_list) != NULL);
363
364 pset_unlock(pset);
365
366 for (int cpuid = lsb_first(needs_cause_ast_mask); cpuid >= 0; cpuid = lsb_next(needs_cause_ast_mask, cpuid)) {
367 processor = processor_array[cpuid];
368 if (processor == current_processor()) {
369 ast_on(AST_SFI);
370 } else {
371 cause_ast_check(processor);
372 }
373 }
374
375 /* Re-arm timer if still enabled */
376 simple_lock(&sfi_lock, LCK_GRP_NULL);
377 if (sfi_is_enabled) {
378 clock_deadline_for_periodic_event(sfi_window_interval,
379 now,
380 &sfi_next_off_deadline);
381 timer_call_enter1(&sfi_timer_call_entry,
382 NULL,
383 sfi_next_off_deadline,
384 TIMER_CALL_SYS_CRITICAL);
385 }
386
387 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_OFF_TIMER) | DBG_FUNC_END, 0, 0, 0, 0, 0);
388
389 simple_unlock(&sfi_lock);
390
391 splx(s);
392 }
393
394 static void
sfi_timer_per_class_on(timer_call_param_t param0,timer_call_param_t param1 __unused)395 sfi_timer_per_class_on(
396 timer_call_param_t param0,
397 timer_call_param_t param1 __unused)
398 {
399 sfi_class_id_t sfi_class_id = (sfi_class_id_t)(uintptr_t)param0;
400 struct sfi_class_state *sfi_class = &sfi_classes[sfi_class_id];
401 kern_return_t kret;
402 spl_t s;
403
404 s = splsched();
405
406 simple_lock(&sfi_lock, LCK_GRP_NULL);
407
408 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_ON_TIMER) | DBG_FUNC_START, sfi_class_id, 0, 0, 0, 0);
409
410 /*
411 * Any threads that may have accumulated in the ready queue for this class should get re-enqueued.
412 * Since we have the sfi_lock held and have changed "class_in_on_phase", we expect
413 * no new threads to be put on this wait queue until the global "off timer" has fired.
414 */
415
416 sfi_class->class_in_on_phase = TRUE;
417 sfi_class->on_timer_programmed = FALSE;
418
419 kret = waitq_wakeup64_all(&sfi_class->waitq,
420 CAST_EVENT64_T(sfi_class_id),
421 THREAD_AWAKENED, WAITQ_ALL_PRIORITIES);
422 assert(kret == KERN_SUCCESS || kret == KERN_NOT_WAITING);
423
424 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_ON_TIMER) | DBG_FUNC_END, 0, 0, 0, 0, 0);
425
426 simple_unlock(&sfi_lock);
427
428 splx(s);
429 }
430
431
432 kern_return_t
sfi_set_window(uint64_t window_usecs)433 sfi_set_window(uint64_t window_usecs)
434 {
435 uint64_t interval, deadline;
436 uint64_t now = mach_absolute_time();
437 sfi_class_id_t i;
438 spl_t s;
439 uint64_t largest_class_off_interval = 0;
440
441 if (window_usecs < MIN_SFI_WINDOW_USEC) {
442 window_usecs = MIN_SFI_WINDOW_USEC;
443 }
444
445 if (window_usecs > UINT32_MAX) {
446 return KERN_INVALID_ARGUMENT;
447 }
448
449 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_SET_WINDOW), window_usecs, 0, 0, 0, 0);
450
451 clock_interval_to_absolutetime_interval((uint32_t)window_usecs, NSEC_PER_USEC, &interval);
452 deadline = now + interval;
453
454 s = splsched();
455
456 simple_lock(&sfi_lock, LCK_GRP_NULL);
457
458 /* Check that we are not bringing in the SFI window smaller than any class */
459 for (i = 0; i < MAX_SFI_CLASS_ID; i++) {
460 if (sfi_classes[i].class_sfi_is_enabled) {
461 largest_class_off_interval = MAX(largest_class_off_interval, sfi_classes[i].off_time_interval);
462 }
463 }
464
465 /*
466 * Off window must be strictly greater than all enabled classes,
467 * otherwise threads would build up on ready queue and never be able to run.
468 */
469 if (interval <= largest_class_off_interval) {
470 simple_unlock(&sfi_lock);
471 splx(s);
472 return KERN_INVALID_ARGUMENT;
473 }
474
475 /*
476 * If the new "off" deadline is further out than the current programmed timer,
477 * just let the current one expire (and the new cadence will be established thereafter).
478 * If the new "off" deadline is nearer than the current one, bring it in, so we
479 * can start the new behavior sooner. Note that this may cause the "off" timer to
480 * fire before some of the class "on" timers have fired.
481 */
482 sfi_window_usecs = window_usecs;
483 sfi_window_interval = interval;
484 sfi_window_is_set = TRUE;
485
486 if (os_atomic_load(&sfi_enabled_class_count, relaxed) == 0) {
487 /* Can't program timer yet */
488 } else if (!sfi_is_enabled) {
489 sfi_is_enabled = TRUE;
490 sfi_next_off_deadline = deadline;
491 timer_call_enter1(&sfi_timer_call_entry,
492 NULL,
493 sfi_next_off_deadline,
494 TIMER_CALL_SYS_CRITICAL);
495 } else if (deadline >= sfi_next_off_deadline) {
496 sfi_next_off_deadline = deadline;
497 } else {
498 sfi_next_off_deadline = deadline;
499 timer_call_enter1(&sfi_timer_call_entry,
500 NULL,
501 sfi_next_off_deadline,
502 TIMER_CALL_SYS_CRITICAL);
503 }
504
505 simple_unlock(&sfi_lock);
506 splx(s);
507
508 return KERN_SUCCESS;
509 }
510
511 kern_return_t
sfi_window_cancel(void)512 sfi_window_cancel(void)
513 {
514 spl_t s;
515
516 s = splsched();
517
518 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_CANCEL_WINDOW), 0, 0, 0, 0, 0);
519
520 /* Disable globals so that global "off-timer" is not re-armed */
521 simple_lock(&sfi_lock, LCK_GRP_NULL);
522 sfi_window_is_set = FALSE;
523 sfi_window_usecs = 0;
524 sfi_window_interval = 0;
525 sfi_next_off_deadline = 0;
526 sfi_is_enabled = FALSE;
527 simple_unlock(&sfi_lock);
528
529 splx(s);
530
531 return KERN_SUCCESS;
532 }
533
534 /* Defers SFI off and per-class on timers (if live) by the specified interval
535 * in Mach Absolute Time Units. Currently invoked to align with the global
536 * forced idle mechanism. Making some simplifying assumptions, the iterative GFI
537 * induced SFI on+off deferrals form a geometric series that converges to yield
538 * an effective SFI duty cycle that is scaled by the GFI duty cycle. Initial phase
539 * alignment and congruency of the SFI/GFI periods can distort this to some extent.
540 */
541
542 kern_return_t
sfi_defer(uint64_t sfi_defer_matus)543 sfi_defer(uint64_t sfi_defer_matus)
544 {
545 spl_t s;
546 kern_return_t kr = KERN_FAILURE;
547 s = splsched();
548
549 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_GLOBAL_DEFER), sfi_defer_matus, 0, 0, 0, 0);
550
551 simple_lock(&sfi_lock, LCK_GRP_NULL);
552 if (!sfi_is_enabled) {
553 goto sfi_defer_done;
554 }
555
556 assert(sfi_next_off_deadline != 0);
557
558 sfi_next_off_deadline += sfi_defer_matus;
559 timer_call_enter1(&sfi_timer_call_entry, NULL, sfi_next_off_deadline, TIMER_CALL_SYS_CRITICAL);
560
561 int i;
562 for (i = 0; i < MAX_SFI_CLASS_ID; i++) {
563 if (sfi_classes[i].class_sfi_is_enabled) {
564 if (sfi_classes[i].on_timer_programmed) {
565 uint64_t new_on_deadline = sfi_classes[i].on_timer_deadline + sfi_defer_matus;
566 sfi_classes[i].on_timer_deadline = new_on_deadline;
567 timer_call_enter1(&sfi_classes[i].on_timer, NULL, new_on_deadline, TIMER_CALL_SYS_CRITICAL);
568 }
569 }
570 }
571
572 kr = KERN_SUCCESS;
573 sfi_defer_done:
574 simple_unlock(&sfi_lock);
575
576 splx(s);
577
578 return kr;
579 }
580
581
582 kern_return_t
sfi_get_window(uint64_t * window_usecs)583 sfi_get_window(uint64_t *window_usecs)
584 {
585 spl_t s;
586 uint64_t off_window_us;
587
588 s = splsched();
589 simple_lock(&sfi_lock, LCK_GRP_NULL);
590
591 off_window_us = sfi_window_usecs;
592
593 simple_unlock(&sfi_lock);
594 splx(s);
595
596 *window_usecs = off_window_us;
597
598 return KERN_SUCCESS;
599 }
600
601
602 kern_return_t
sfi_set_class_offtime(sfi_class_id_t class_id,uint64_t offtime_usecs)603 sfi_set_class_offtime(sfi_class_id_t class_id, uint64_t offtime_usecs)
604 {
605 uint64_t interval;
606 spl_t s;
607 uint64_t off_window_interval;
608
609 if (offtime_usecs < MIN_SFI_WINDOW_USEC) {
610 offtime_usecs = MIN_SFI_WINDOW_USEC;
611 }
612
613 if (class_id == SFI_CLASS_UNSPECIFIED || class_id >= MAX_SFI_CLASS_ID) {
614 return KERN_INVALID_ARGUMENT;
615 }
616
617 if (offtime_usecs > UINT32_MAX) {
618 return KERN_INVALID_ARGUMENT;
619 }
620
621 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_SET_CLASS_OFFTIME), offtime_usecs, class_id, 0, 0, 0);
622
623 clock_interval_to_absolutetime_interval((uint32_t)offtime_usecs, NSEC_PER_USEC, &interval);
624
625 s = splsched();
626
627 simple_lock(&sfi_lock, LCK_GRP_NULL);
628 off_window_interval = sfi_window_interval;
629
630 /* Check that we are not bringing in class off-time larger than the SFI window */
631 if (off_window_interval && (interval >= off_window_interval)) {
632 simple_unlock(&sfi_lock);
633 splx(s);
634 return KERN_INVALID_ARGUMENT;
635 }
636
637 /* We never re-program the per-class on-timer, but rather just let it expire naturally */
638 if (!sfi_classes[class_id].class_sfi_is_enabled) {
639 os_atomic_inc(&sfi_enabled_class_count, relaxed);
640 }
641 sfi_classes[class_id].off_time_usecs = offtime_usecs;
642 sfi_classes[class_id].off_time_interval = interval;
643 sfi_classes[class_id].class_sfi_is_enabled = TRUE;
644
645 if (sfi_window_is_set && !sfi_is_enabled) {
646 /* start global off timer */
647 sfi_is_enabled = TRUE;
648 sfi_next_off_deadline = mach_absolute_time() + sfi_window_interval;
649 timer_call_enter1(&sfi_timer_call_entry,
650 NULL,
651 sfi_next_off_deadline,
652 TIMER_CALL_SYS_CRITICAL);
653 }
654
655 simple_unlock(&sfi_lock);
656
657 splx(s);
658
659 return KERN_SUCCESS;
660 }
661
662 kern_return_t
sfi_class_offtime_cancel(sfi_class_id_t class_id)663 sfi_class_offtime_cancel(sfi_class_id_t class_id)
664 {
665 spl_t s;
666
667 if (class_id == SFI_CLASS_UNSPECIFIED || class_id >= MAX_SFI_CLASS_ID) {
668 return KERN_INVALID_ARGUMENT;
669 }
670
671 s = splsched();
672
673 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_CANCEL_CLASS_OFFTIME), class_id, 0, 0, 0, 0);
674
675 simple_lock(&sfi_lock, LCK_GRP_NULL);
676
677 /* We never re-program the per-class on-timer, but rather just let it expire naturally */
678 if (sfi_classes[class_id].class_sfi_is_enabled) {
679 os_atomic_dec(&sfi_enabled_class_count, relaxed);
680 }
681 sfi_classes[class_id].off_time_usecs = 0;
682 sfi_classes[class_id].off_time_interval = 0;
683 sfi_classes[class_id].class_sfi_is_enabled = FALSE;
684
685 if (os_atomic_load(&sfi_enabled_class_count, relaxed) == 0) {
686 sfi_is_enabled = FALSE;
687 }
688
689 simple_unlock(&sfi_lock);
690
691 splx(s);
692
693 return KERN_SUCCESS;
694 }
695
696 kern_return_t
sfi_get_class_offtime(sfi_class_id_t class_id,uint64_t * offtime_usecs)697 sfi_get_class_offtime(sfi_class_id_t class_id, uint64_t *offtime_usecs)
698 {
699 uint64_t off_time_us;
700 spl_t s;
701
702 if (class_id == SFI_CLASS_UNSPECIFIED || class_id >= MAX_SFI_CLASS_ID) {
703 return 0;
704 }
705
706 s = splsched();
707
708 simple_lock(&sfi_lock, LCK_GRP_NULL);
709 off_time_us = sfi_classes[class_id].off_time_usecs;
710 simple_unlock(&sfi_lock);
711
712 splx(s);
713
714 *offtime_usecs = off_time_us;
715
716 return KERN_SUCCESS;
717 }
718
719 /*
720 * sfi_thread_classify and sfi_processor_active_thread_classify perform the critical
721 * role of quickly categorizing a thread into its SFI class so that an AST_SFI can be
722 * set. As the thread is unwinding to userspace, sfi_ast() performs full locking
723 * and determines whether the thread should enter an SFI wait state. Because of
724 * the inherent races between the time the AST is set and when it is evaluated,
725 * thread classification can be inaccurate (but should always be safe). This is
726 * especially the case for sfi_processor_active_thread_classify, which must
727 * classify the active thread on a remote processor without taking the thread lock.
728 * When in doubt, classification should err on the side of *not* classifying a
729 * thread at all, and wait for the thread itself to either hit a quantum expiration
730 * or block inside the kernel.
731 */
732
733 /*
734 * Thread must be locked. Ultimately, the real decision to enter
735 * SFI wait happens at the AST boundary.
736 */
737 sfi_class_id_t
sfi_thread_classify(thread_t thread)738 sfi_thread_classify(thread_t thread)
739 {
740 task_t task = get_threadtask(thread);
741 boolean_t is_kernel_thread = (task == kernel_task);
742 sched_mode_t thmode = thread->sched_mode;
743 boolean_t focal = FALSE;
744
745 /* kernel threads never reach the user AST boundary, and are in a separate world for SFI */
746 if (is_kernel_thread) {
747 return SFI_CLASS_KERNEL;
748 }
749
750 /* no need to re-classify threads unless there is at least one enabled SFI class */
751 if (os_atomic_load(&sfi_enabled_class_count, relaxed) == 0) {
752 return SFI_CLASS_OPTED_OUT;
753 }
754
755 int task_role = proc_get_effective_task_policy(task, TASK_POLICY_ROLE);
756 int latency_qos = proc_get_effective_task_policy(task, TASK_POLICY_LATENCY_QOS);
757 int managed_task = proc_get_effective_task_policy(task, TASK_POLICY_SFI_MANAGED);
758
759 int thread_qos = proc_get_effective_thread_policy(thread, TASK_POLICY_QOS);
760 int thread_bg = proc_get_effective_thread_policy(thread, TASK_POLICY_DARWIN_BG);
761
762 if (thread_qos == THREAD_QOS_MAINTENANCE) {
763 return SFI_CLASS_MAINTENANCE;
764 }
765
766 if (thread_bg || thread_qos == THREAD_QOS_BACKGROUND) {
767 return SFI_CLASS_DARWIN_BG;
768 }
769
770 if (latency_qos != 0) {
771 int latency_qos_wtf = latency_qos - 1;
772
773 if ((latency_qos_wtf >= 4) && (latency_qos_wtf <= 5)) {
774 return SFI_CLASS_APP_NAP;
775 }
776 }
777
778 /*
779 * Realtime and fixed priority threads express their duty cycle constraints
780 * via other mechanisms, and are opted out of (most) forms of SFI
781 */
782 if (thmode == TH_MODE_REALTIME || thmode == TH_MODE_FIXED || task_role == TASK_GRAPHICS_SERVER) {
783 return SFI_CLASS_OPTED_OUT;
784 }
785
786 /*
787 * Threads with unspecified, legacy, or user-initiated QOS class can be individually managed.
788 */
789 switch (task_role) {
790 case TASK_CONTROL_APPLICATION:
791 case TASK_FOREGROUND_APPLICATION:
792 focal = TRUE;
793 break;
794 case TASK_BACKGROUND_APPLICATION:
795 case TASK_DEFAULT_APPLICATION:
796 case TASK_UNSPECIFIED:
797 /* Focal if the task is in a coalition with a FG/focal app */
798 if (task_coalition_focal_count(task) > 0) {
799 focal = TRUE;
800 }
801 break;
802 case TASK_THROTTLE_APPLICATION:
803 case TASK_DARWINBG_APPLICATION:
804 case TASK_NONUI_APPLICATION:
805 /* Definitely not focal */
806 default:
807 break;
808 }
809
810 if (managed_task) {
811 switch (thread_qos) {
812 case THREAD_QOS_UNSPECIFIED:
813 case THREAD_QOS_LEGACY:
814 case THREAD_QOS_USER_INITIATED:
815 if (focal) {
816 return SFI_CLASS_MANAGED_FOCAL;
817 } else {
818 return SFI_CLASS_MANAGED_NONFOCAL;
819 }
820 default:
821 break;
822 }
823 }
824
825 if (thread_qos == THREAD_QOS_UTILITY) {
826 return SFI_CLASS_UTILITY;
827 }
828
829 /*
830 * Classify threads in non-managed tasks
831 */
832 if (focal) {
833 switch (thread_qos) {
834 case THREAD_QOS_USER_INTERACTIVE:
835 return SFI_CLASS_USER_INTERACTIVE_FOCAL;
836 case THREAD_QOS_USER_INITIATED:
837 return SFI_CLASS_USER_INITIATED_FOCAL;
838 case THREAD_QOS_LEGACY:
839 return SFI_CLASS_LEGACY_FOCAL;
840 default:
841 return SFI_CLASS_DEFAULT_FOCAL;
842 }
843 } else {
844 switch (thread_qos) {
845 case THREAD_QOS_USER_INTERACTIVE:
846 return SFI_CLASS_USER_INTERACTIVE_NONFOCAL;
847 case THREAD_QOS_USER_INITIATED:
848 return SFI_CLASS_USER_INITIATED_NONFOCAL;
849 case THREAD_QOS_LEGACY:
850 return SFI_CLASS_LEGACY_NONFOCAL;
851 default:
852 return SFI_CLASS_DEFAULT_NONFOCAL;
853 }
854 }
855 }
856
857 /*
858 * pset must be locked.
859 */
860 sfi_class_id_t
sfi_processor_active_thread_classify(processor_t processor)861 sfi_processor_active_thread_classify(processor_t processor)
862 {
863 return processor->current_sfi_class;
864 }
865
866 /*
867 * thread must be locked. This is inherently racy, with the intent that
868 * at the AST boundary, it will be fully evaluated whether we need to
869 * perform an AST wait
870 */
871 ast_t
sfi_thread_needs_ast(thread_t thread,sfi_class_id_t * out_class)872 sfi_thread_needs_ast(thread_t thread, sfi_class_id_t *out_class)
873 {
874 sfi_class_id_t class_id;
875
876 class_id = sfi_thread_classify(thread);
877
878 if (out_class) {
879 *out_class = class_id;
880 }
881
882 /* No lock taken, so a stale value may be used. */
883 if (!sfi_classes[class_id].class_in_on_phase) {
884 return AST_SFI;
885 } else {
886 return AST_NONE;
887 }
888 }
889
890 /*
891 * pset must be locked. We take the SFI class for
892 * the currently running thread which is cached on
893 * the processor_t, and assume it is accurate. In the
894 * worst case, the processor will get an IPI and be asked
895 * to evaluate if the current running thread at that
896 * later point in time should be in an SFI wait.
897 */
898 ast_t
sfi_processor_needs_ast(processor_t processor)899 sfi_processor_needs_ast(processor_t processor)
900 {
901 sfi_class_id_t class_id;
902
903 class_id = sfi_processor_active_thread_classify(processor);
904
905 /* No lock taken, so a stale value may be used. */
906 if (!sfi_classes[class_id].class_in_on_phase) {
907 return AST_SFI;
908 } else {
909 return AST_NONE;
910 }
911 }
912
913 static inline void
_sfi_wait_cleanup(void)914 _sfi_wait_cleanup(void)
915 {
916 thread_t self = current_thread();
917
918 spl_t s = splsched();
919 simple_lock(&sfi_lock, LCK_GRP_NULL);
920
921 sfi_class_id_t current_sfi_wait_class = self->sfi_wait_class;
922
923 assert((SFI_CLASS_UNSPECIFIED < current_sfi_wait_class) &&
924 (current_sfi_wait_class < MAX_SFI_CLASS_ID));
925
926 self->sfi_wait_class = SFI_CLASS_UNSPECIFIED;
927
928 simple_unlock(&sfi_lock);
929 splx(s);
930
931 /*
932 * It's possible for the thread to be woken up due to the SFI period
933 * ending *before* it finishes blocking. In that case,
934 * wait_sfi_begin_time won't be set.
935 *
936 * Derive the time sacrificed to SFI by looking at when this thread was
937 * awoken by the on-timer, to avoid counting the time this thread spent
938 * waiting to get scheduled.
939 *
940 * Note that last_made_runnable_time could be reset if this thread
941 * gets preempted before we read the value. To fix that, we'd need to
942 * track wait time in a thread timer, sample the timer before blocking,
943 * pass the value through thread->parameter, and subtract that.
944 */
945
946 if (self->wait_sfi_begin_time != 0) {
947 uint64_t made_runnable = os_atomic_load(&self->last_made_runnable_time, relaxed);
948 int64_t sfi_wait_time = made_runnable - self->wait_sfi_begin_time;
949 assert(sfi_wait_time >= 0);
950
951 ledger_credit(get_threadtask(self)->ledger,
952 task_ledgers.sfi_wait_times[current_sfi_wait_class],
953 sfi_wait_time);
954
955 self->wait_sfi_begin_time = 0;
956 }
957 }
958
959 /*
960 * Called at AST context to fully evaluate if the current thread
961 * (which is obviously running) should instead block in an SFI wait.
962 * We must take the sfi_lock to check whether we are in the "off" period
963 * for the class, and if so, block.
964 */
965 void
sfi_ast(thread_t thread)966 sfi_ast(thread_t thread)
967 {
968 sfi_class_id_t class_id;
969 spl_t s;
970 struct sfi_class_state *sfi_class;
971 wait_result_t waitret;
972 boolean_t did_wait = FALSE;
973 thread_continue_t continuation;
974
975 s = splsched();
976
977 simple_lock(&sfi_lock, LCK_GRP_NULL);
978
979 if (!sfi_is_enabled) {
980 /*
981 * SFI is not enabled, or has recently been disabled.
982 * There is no point putting this thread on a deferred ready
983 * queue, even if it were classified as needing it, since
984 * SFI will truly be off at the next global off timer
985 */
986 simple_unlock(&sfi_lock);
987 splx(s);
988
989 return;
990 }
991
992 thread_lock(thread);
993 thread->sfi_class = class_id = sfi_thread_classify(thread);
994 thread_unlock(thread);
995
996 /*
997 * Once the sfi_lock is taken and the thread's ->sfi_class field is updated, we
998 * are committed to transitioning to whatever state is indicated by "->class_in_on_phase".
999 * If another thread tries to call sfi_reevaluate() after this point, it will take the
1000 * sfi_lock and see the thread in this wait state. If another thread calls
1001 * sfi_reevaluate() before this point, it would see a runnable thread and at most
1002 * attempt to send an AST to this processor, but we would have the most accurate
1003 * classification.
1004 */
1005
1006 sfi_class = &sfi_classes[class_id];
1007 if (!sfi_class->class_in_on_phase) {
1008 /* Need to block thread in wait queue */
1009 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_THREAD_DEFER),
1010 thread_tid(thread), class_id, 0, 0, 0);
1011
1012 waitret = waitq_assert_wait64(&sfi_class->waitq,
1013 CAST_EVENT64_T(class_id),
1014 THREAD_INTERRUPTIBLE | THREAD_WAIT_NOREPORT, 0);
1015 if (waitret == THREAD_WAITING) {
1016 thread->sfi_wait_class = class_id;
1017 did_wait = TRUE;
1018 continuation = sfi_class->continuation;
1019 } else {
1020 /* thread may be exiting already, all other errors are unexpected */
1021 assert(waitret == THREAD_INTERRUPTED);
1022 }
1023 }
1024 simple_unlock(&sfi_lock);
1025
1026 splx(s);
1027
1028 if (did_wait) {
1029 assert(thread->wait_sfi_begin_time == 0);
1030
1031 thread_block_reason(continuation, NULL, AST_SFI);
1032 }
1033 }
1034
1035 /* Thread must be unlocked */
1036 void
sfi_reevaluate(thread_t thread)1037 sfi_reevaluate(thread_t thread)
1038 {
1039 kern_return_t kret;
1040 spl_t s;
1041 sfi_class_id_t class_id, current_class_id;
1042 ast_t sfi_ast;
1043
1044 s = splsched();
1045
1046 simple_lock(&sfi_lock, LCK_GRP_NULL);
1047
1048 thread_lock(thread);
1049 sfi_ast = sfi_thread_needs_ast(thread, &class_id);
1050 thread->sfi_class = class_id;
1051
1052 /*
1053 * This routine chiefly exists to boost threads out of an SFI wait
1054 * if their classification changes before the "on" timer fires.
1055 *
1056 * If we calculate that a thread is in a different ->sfi_wait_class
1057 * than we think it should be (including no-SFI-wait), we need to
1058 * correct that:
1059 *
1060 * If the thread is in SFI wait and should not be (or should be waiting
1061 * on a different class' "on" timer), we wake it up. If needed, the
1062 * thread may immediately block again in the different SFI wait state.
1063 *
1064 * If the thread is not in an SFI wait state and it should be, we need
1065 * to get that thread's attention, possibly by sending an AST to another
1066 * processor.
1067 */
1068
1069 if ((current_class_id = thread->sfi_wait_class) != SFI_CLASS_UNSPECIFIED) {
1070 thread_unlock(thread); /* not needed anymore */
1071
1072 assert(current_class_id < MAX_SFI_CLASS_ID);
1073
1074 if ((sfi_ast == AST_NONE) || (class_id != current_class_id)) {
1075 struct sfi_class_state *sfi_class = &sfi_classes[current_class_id];
1076
1077 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SFI, SFI_WAIT_CANCELED), thread_tid(thread), current_class_id, class_id, 0, 0);
1078
1079 kret = waitq_wakeup64_thread(&sfi_class->waitq,
1080 CAST_EVENT64_T(current_class_id),
1081 thread,
1082 THREAD_AWAKENED);
1083 assert(kret == KERN_SUCCESS || kret == KERN_NOT_WAITING);
1084 }
1085 } else {
1086 /*
1087 * Thread's current SFI wait class is not set, and because we
1088 * have the sfi_lock, it won't get set.
1089 */
1090
1091 if ((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN) {
1092 if (sfi_ast != AST_NONE) {
1093 if (thread == current_thread()) {
1094 ast_on(sfi_ast);
1095 } else {
1096 processor_t processor = thread->last_processor;
1097
1098 if (processor != PROCESSOR_NULL &&
1099 processor->state == PROCESSOR_RUNNING &&
1100 processor->active_thread == thread) {
1101 cause_ast_check(processor);
1102 } else {
1103 /*
1104 * Runnable thread that's not on a CPU currently. When a processor
1105 * does context switch to it, the AST will get set based on whether
1106 * the thread is in its "off time".
1107 */
1108 }
1109 }
1110 }
1111 }
1112
1113 thread_unlock(thread);
1114 }
1115
1116 simple_unlock(&sfi_lock);
1117 splx(s);
1118 }
1119
1120 #else /* !CONFIG_SCHED_SFI */
1121
1122 kern_return_t
sfi_set_window(uint64_t window_usecs __unused)1123 sfi_set_window(uint64_t window_usecs __unused)
1124 {
1125 return KERN_NOT_SUPPORTED;
1126 }
1127
1128 kern_return_t
sfi_window_cancel(void)1129 sfi_window_cancel(void)
1130 {
1131 return KERN_NOT_SUPPORTED;
1132 }
1133
1134
1135 kern_return_t
sfi_get_window(uint64_t * window_usecs __unused)1136 sfi_get_window(uint64_t *window_usecs __unused)
1137 {
1138 return KERN_NOT_SUPPORTED;
1139 }
1140
1141
1142 kern_return_t
sfi_set_class_offtime(sfi_class_id_t class_id __unused,uint64_t offtime_usecs __unused)1143 sfi_set_class_offtime(sfi_class_id_t class_id __unused, uint64_t offtime_usecs __unused)
1144 {
1145 return KERN_NOT_SUPPORTED;
1146 }
1147
1148 kern_return_t
sfi_class_offtime_cancel(sfi_class_id_t class_id __unused)1149 sfi_class_offtime_cancel(sfi_class_id_t class_id __unused)
1150 {
1151 return KERN_NOT_SUPPORTED;
1152 }
1153
1154 kern_return_t
sfi_get_class_offtime(sfi_class_id_t class_id __unused,uint64_t * offtime_usecs __unused)1155 sfi_get_class_offtime(sfi_class_id_t class_id __unused, uint64_t *offtime_usecs __unused)
1156 {
1157 return KERN_NOT_SUPPORTED;
1158 }
1159
1160 void
sfi_reevaluate(thread_t thread __unused)1161 sfi_reevaluate(thread_t thread __unused)
1162 {
1163 return;
1164 }
1165
1166 sfi_class_id_t
sfi_thread_classify(thread_t thread)1167 sfi_thread_classify(thread_t thread)
1168 {
1169 task_t task = get_threadtask(thread);
1170 boolean_t is_kernel_thread = (task == kernel_task);
1171
1172 if (is_kernel_thread) {
1173 return SFI_CLASS_KERNEL;
1174 }
1175
1176 return SFI_CLASS_OPTED_OUT;
1177 }
1178
1179 #endif /* !CONFIG_SCHED_SFI */
1180