1 /*
2 * Copyright (c) 2007-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Routines for preemption disablement,
31 * which prevents the current thread from giving up its current CPU.
32 */
33
34 #include <arm/cpu_data.h>
35 #include <arm/cpu_data_internal.h>
36 #include <arm/preemption_disable_internal.h>
37 #include <kern/cpu_data.h>
38 #include <kern/percpu.h>
39 #include <kern/thread.h>
40 #include <mach/machine/sdt.h>
41 #include <os/base.h>
42 #include <stdint.h>
43 #include <sys/kdebug.h>
44
45 #if SCHED_HYGIENE_DEBUG
46 static void
47 _do_disable_preemption_without_measurements(void);
48 #endif
49
50 /*
51 * This function checks whether an AST_URGENT has been pended.
52 *
53 * It is called once the preemption has been reenabled, which means the thread
54 * may have been preempted right before this was called, and when this function
55 * actually performs the check, we've changed CPU.
56 *
57 * This race is however benign: the point of AST_URGENT is to trigger a context
58 * switch, so if one happened, there's nothing left to check for, and AST_URGENT
59 * was cleared in the process.
60 *
61 * It follows that this check cannot have false negatives, which allows us
62 * to avoid fiddling with interrupt state for the vast majority of cases
63 * when the check will actually be negative.
64 */
65 static OS_NOINLINE
66 void
kernel_preempt_check(void)67 kernel_preempt_check(void)
68 {
69 uint64_t state;
70
71 /* If interrupts are masked, we can't take an AST here */
72 state = __builtin_arm_rsr64("DAIF");
73 if (state & DAIF_IRQF) {
74 return;
75 }
76
77 /* disable interrupts (IRQ FIQ ASYNCF) */
78 __builtin_arm_wsr64("DAIFSet", DAIFSC_STANDARD_DISABLE);
79
80 /*
81 * Reload cpu_pending_ast: a context switch would cause it to change.
82 * Now that interrupts are disabled, this will debounce false positives.
83 */
84 if (current_thread()->machine.CpuDatap->cpu_pending_ast & AST_URGENT) {
85 ast_taken_kernel();
86 }
87
88 /* restore the original interrupt mask */
89 __builtin_arm_wsr64("DAIF", state);
90 }
91
92 static inline void
_enable_preemption_write_count(thread_t thread,unsigned int count)93 _enable_preemption_write_count(thread_t thread, unsigned int count)
94 {
95 os_atomic_store(&thread->machine.preemption_count, count, compiler_acq_rel);
96
97 /*
98 * This check is racy and could load from another CPU's pending_ast mask,
99 * but as described above, this can't have false negatives.
100 */
101 if (count == 0) {
102 if (__improbable(thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT)) {
103 return kernel_preempt_check();
104 }
105 }
106 }
107
108 /*
109 * This function is written in a way that the codegen is extremely short.
110 *
111 * LTO isn't smart enough to inline it, yet it is profitable because
112 * the vast majority of callers use current_thread() already.
113 *
114 * /!\ Breaking inlining causes zalloc to be roughly 10% slower /!\
115 */
116 OS_ALWAYS_INLINE
117 void
_disable_preemption(void)118 _disable_preemption(void)
119 {
120 thread_t thread = current_thread();
121 unsigned int count = thread->machine.preemption_count;
122
123 os_atomic_store(&thread->machine.preemption_count,
124 count + 1, compiler_acq_rel);
125
126 #if SCHED_HYGIENE_DEBUG
127 /*
128 * Note that this is not the only place preemption gets disabled,
129 * it also gets modified on ISR and PPL entry/exit. Both of those
130 * events will be treated specially however, and
131 * increment/decrement being paired around their entry/exit means
132 * that collection here is not desynced otherwise.
133 */
134 if (improbable_static_if(sched_debug_preemption_disable)) {
135 if (__improbable(count == 0 &&
136 sched_preemption_disable_debug_mode)) {
137 __attribute__((musttail))
138 return _prepare_preemption_disable_measurement();
139 }
140 }
141 #endif /* SCHED_HYGIENE_DEBUG */
142 }
143
144 /*
145 * This variant of disable_preemption() allows disabling preemption
146 * without taking measurements (and later potentially triggering
147 * actions on those).
148 */
149 OS_ALWAYS_INLINE
150 void
_disable_preemption_without_measurements(void)151 _disable_preemption_without_measurements(void)
152 {
153 thread_t thread = current_thread();
154 unsigned int count = thread->machine.preemption_count;
155
156 #if SCHED_HYGIENE_DEBUG
157 _do_disable_preemption_without_measurements();
158 #endif /* SCHED_HYGIENE_DEBUG */
159
160 os_atomic_store(&thread->machine.preemption_count,
161 count + 1, compiler_acq_rel);
162 }
163
164 /*
165 * To help _enable_preemption() inline everywhere with LTO,
166 * we keep these nice non inlineable functions as the panic()
167 * codegen setup is quite large and for weird reasons causes a frame.
168 */
169 __abortlike
170 static void
_enable_preemption_underflow(void)171 _enable_preemption_underflow(void)
172 {
173 panic("Preemption count underflow");
174 }
175
176 /*
177 * This function is written in a way that the codegen is extremely short.
178 *
179 * LTO isn't smart enough to inline it, yet it is profitable because
180 * the vast majority of callers use current_thread() already.
181 *
182 * The SCHED_HYGIENE_MARKER trick is used so that we do not have to load
183 * unrelated fields of current_thread().
184 *
185 * /!\ Breaking inlining causes zalloc to be roughly 10% slower /!\
186 */
187 OS_ALWAYS_INLINE
188 void
_enable_preemption(void)189 _enable_preemption(void)
190 {
191 thread_t thread = current_thread();
192 unsigned int count = thread->machine.preemption_count;
193
194 if (__improbable(count == 0)) {
195 _enable_preemption_underflow();
196 }
197
198 #if SCHED_HYGIENE_DEBUG
199 if (__improbable(count == SCHED_HYGIENE_MARKER + 1)) {
200 return _collect_preemption_disable_measurement();
201 }
202 #endif /* SCHED_HYGIENE_DEBUG */
203
204 _enable_preemption_write_count(thread, count - 1);
205 }
206
207 OS_ALWAYS_INLINE
208 unsigned int
get_preemption_level_for_thread(thread_t thread)209 get_preemption_level_for_thread(thread_t thread)
210 {
211 unsigned int count = thread->machine.preemption_count;
212
213 #if SCHED_HYGIENE_DEBUG
214 /*
215 * hide this "flag" from callers,
216 * and it would make the count look negative anyway
217 * which some people dislike
218 */
219 count &= ~SCHED_HYGIENE_MARKER;
220 #endif
221 return (int)count;
222 }
223
224 OS_ALWAYS_INLINE
225 int
get_preemption_level(void)226 get_preemption_level(void)
227 {
228 return get_preemption_level_for_thread(current_thread());
229 }
230
231 #if SCHED_HYGIENE_DEBUG
232
233 uint64_t _Atomic PERCPU_DATA_HACK_78750602(preemption_disable_max_mt);
234
235 #if XNU_PLATFORM_iPhoneOS
236 #define DEFAULT_PREEMPTION_TIMEOUT 120000 /* 5ms */
237 #define DEFAULT_PREEMPTION_MODE SCHED_HYGIENE_MODE_PANIC
238 #elif XNU_PLATFORM_XROS
239 #define DEFAULT_PREEMPTION_TIMEOUT 24000 /* 1ms */
240 #define DEFAULT_PREEMPTION_MODE SCHED_HYGIENE_MODE_PANIC
241 #else
242 #define DEFAULT_PREEMPTION_TIMEOUT 0 /* Disabled */
243 #define DEFAULT_PREEMPTION_MODE SCHED_HYGIENE_MODE_OFF
244 #endif /* XNU_PLATFORM_iPhoneOS */
245
246 MACHINE_TIMEOUT_DEV_WRITEABLE(sched_preemption_disable_threshold_mt, "sched-preemption",
247 DEFAULT_PREEMPTION_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE, kprintf_spam_mt_pred);
248 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, sched_preemption_disable_debug_mode,
249 "machine-timeouts",
250 "sched-preemption-disable-mode", /* DT property names have to be 31 chars max */
251 "sched_preemption_disable_debug_mode",
252 DEFAULT_PREEMPTION_MODE,
253 TUNABLE_DT_CHECK_CHOSEN);
254
255 struct _preemption_disable_pcpu PERCPU_DATA(_preemption_disable_pcpu_data);
256
257 /*
258 ** Start a measurement window for the current CPU's preemption disable timeout.
259 *
260 * Interrupts must be disabled when calling this function,
261 * but the assertion has been elided as this is on the fast path.
262 */
263 static void
_preemption_disable_snap_start(void)264 _preemption_disable_snap_start(void)
265 {
266 struct _preemption_disable_pcpu *pcpu = PERCPU_GET(_preemption_disable_pcpu_data);
267 pcpu->pdp_abandon = false;
268 pcpu->pdp_start.pds_mach_time = ml_get_sched_hygiene_timebase();
269 pcpu->pdp_start.pds_int_mach_time = recount_current_processor_interrupt_duration_mach();
270 #if CONFIG_CPU_COUNTERS
271 if (static_if(sched_debug_pmc)) {
272 mt_cur_cpu_cycles_instrs_speculative(&pcpu->pdp_start.pds_cycles,
273 &pcpu->pdp_start.pds_instrs);
274 }
275 #endif /* CONFIG_CPU_COUNTERS */
276 }
277
278 /*
279 **
280 * End a measurement window for the current CPU's preemption disable timeout,
281 * using the snapshot started by _preemption_disable_snap_start().
282 *
283 * @param start An out-parameter for the starting snapshot,
284 * captured while interrupts are disabled.
285 *
286 * @param now An out-parameter for the current times,
287 * captured at the same time as the start and with interrupts disabled.
288 * This is meant for computing a delta.
289 * Even with @link sched_hygiene_debug_pmc , the PMCs will not be read.
290 * This allows their (relatively expensive) reads to happen only if the time threshold has been violated.
291 *
292 * @return Whether to abandon the current measurement due to a call to abandon_preemption_disable_measurement().
293 */
294 static bool
_preemption_disable_snap_end(struct _preemption_disable_snap * start,struct _preemption_disable_snap * now)295 _preemption_disable_snap_end(
296 struct _preemption_disable_snap *start,
297 struct _preemption_disable_snap *now)
298 {
299 struct _preemption_disable_pcpu *pcpu = PERCPU_GET(_preemption_disable_pcpu_data);
300
301 const bool int_masked_debug = false;
302 const bool istate = ml_set_interrupts_enabled_with_debug(false, int_masked_debug);
303 /*
304 * Collect start time and current time with interrupts disabled.
305 * Otherwise an interrupt coming in after grabbing the timestamp
306 * could spuriously inflate the measurement, because it will
307 * adjust preemption_disable_mt only after we already grabbed
308 * it.
309 *
310 * (Even worse if we collected the current time first: Then a
311 * subsequent interrupt could adjust preemption_disable_mt to
312 * make the duration go negative after subtracting the already
313 * grabbed time. With interrupts disabled we don't care much about
314 * the order.)
315 */
316
317 *start = pcpu->pdp_start;
318 uint64_t now_time = ml_get_sched_hygiene_timebase();
319 now->pds_mach_time = now_time;
320 now->pds_int_mach_time = recount_current_processor_interrupt_duration_mach();
321 const bool abandon = pcpu->pdp_abandon;
322 const uint64_t max_duration = os_atomic_load(&pcpu->pdp_max_mach_duration, relaxed);
323
324 pcpu->pdp_start.pds_mach_time = 0;
325
326 /*
327 * Don't need to reset (or even save) pdp_abandon here:
328 * abandon_preemption_disable_measurement is a no-op anyway
329 * if pdp_start.pds_mach_time == 0 (which we just set), and it
330 * will stay that way until the next call to
331 * _collect_preemption_disable_measurement.
332 */
333 ml_set_interrupts_enabled_with_debug(istate, int_masked_debug);
334 if (__probable(!abandon)) {
335 const int64_t gross_duration = now_time - start->pds_mach_time;
336 if (__improbable(gross_duration > max_duration)) {
337 os_atomic_store(&pcpu->pdp_max_mach_duration, gross_duration, relaxed);
338 }
339 }
340 return abandon;
341 }
342
343 OS_NOINLINE
344 void
_prepare_preemption_disable_measurement(void)345 _prepare_preemption_disable_measurement(void)
346 {
347 thread_t thread = current_thread();
348
349 if (thread->machine.inthandler_timestamp == 0) {
350 /*
351 * Only prepare a measurement if not currently in an interrupt
352 * handler.
353 *
354 * We are only interested in the net duration of disabled
355 * preemption, that is: The time in which preemption was
356 * disabled, minus the intervals in which any (likely
357 * unrelated) interrupts were handled.
358 * recount_current_thread_interrupt_time_mach() will remove those
359 * intervals, however we also do not even start measuring
360 * preemption disablement if we are already within handling of
361 * an interrupt when preemption was disabled (the resulting
362 * net time would be 0).
363 *
364 * Interrupt handling duration is handled separately, and any
365 * long intervals of preemption disablement are counted
366 * towards that.
367 */
368
369 bool const int_masked_debug = false;
370 bool istate = ml_set_interrupts_enabled_with_debug(false, int_masked_debug);
371 thread->machine.preemption_count |= SCHED_HYGIENE_MARKER;
372 _preemption_disable_snap_start();
373 ml_set_interrupts_enabled_with_debug(istate, int_masked_debug);
374 }
375 }
376
377 OS_NOINLINE
378 void
_collect_preemption_disable_measurement(void)379 _collect_preemption_disable_measurement(void)
380 {
381 struct _preemption_disable_snap start = { 0 };
382 struct _preemption_disable_snap now = { 0 };
383 const bool abandon = _preemption_disable_snap_end(&start, &now);
384
385 if (__improbable(abandon)) {
386 goto out;
387 }
388
389 int64_t const gross_duration = now.pds_mach_time - start.pds_mach_time;
390 uint64_t const threshold = os_atomic_load(&sched_preemption_disable_threshold_mt, relaxed);
391 if (__improbable(threshold > 0 && gross_duration >= threshold)) {
392 /*
393 * Double check that the time spent not handling interrupts is over the threshold.
394 */
395 int64_t const interrupt_duration = now.pds_int_mach_time - start.pds_int_mach_time;
396 int64_t const net_duration = gross_duration - interrupt_duration;
397 assert3u(net_duration, >=, 0);
398 if (net_duration < threshold) {
399 goto out;
400 }
401
402 uint64_t average_freq = 0;
403 uint64_t average_cpi_whole = 0;
404 uint64_t average_cpi_fractional = 0;
405
406 #if CONFIG_CPU_COUNTERS
407 if (static_if(sched_debug_pmc)) {
408 /*
409 * We're getting these values a bit late, but getting them
410 * is a bit expensive, so we take the slight hit in
411 * accuracy for the reported values (which aren't very
412 * stable anyway).
413 */
414 const bool int_masked_debug = false;
415 const bool istate = ml_set_interrupts_enabled_with_debug(false, int_masked_debug);
416 mt_cur_cpu_cycles_instrs_speculative(&now.pds_cycles, &now.pds_instrs);
417 ml_set_interrupts_enabled_with_debug(istate, int_masked_debug);
418 const uint64_t cycles_elapsed = now.pds_cycles - start.pds_cycles;
419 const uint64_t instrs_retired = now.pds_instrs - start.pds_instrs;
420
421 uint64_t duration_ns;
422 absolutetime_to_nanoseconds(gross_duration, &duration_ns);
423
424 average_freq = cycles_elapsed / (duration_ns / 1000);
425 average_cpi_whole = cycles_elapsed / instrs_retired;
426 average_cpi_fractional =
427 ((cycles_elapsed * 100) / instrs_retired) % 100;
428 }
429 #endif /* CONFIG_CPU_COUNTERS */
430
431 if (__probable(sched_preemption_disable_debug_mode == SCHED_HYGIENE_MODE_PANIC)) {
432 panic("preemption disable timeout exceeded: %llu >= %llu mt ticks (start: %llu, now: %llu, gross: %llu, inttime: %llu), "
433 "freq = %llu MHz, CPI = %llu.%llu",
434 net_duration, threshold, start.pds_mach_time, now.pds_mach_time,
435 gross_duration, interrupt_duration,
436 average_freq, average_cpi_whole, average_cpi_fractional);
437 }
438
439 DTRACE_SCHED4(mach_preemption_expired, uint64_t, net_duration, uint64_t, gross_duration,
440 uint64_t, average_cpi_whole, uint64_t, average_cpi_fractional);
441 KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_PREEMPTION_EXPIRED), net_duration, gross_duration, average_cpi_whole, average_cpi_fractional);
442 }
443
444 out:
445 /*
446 * the preemption count is SCHED_HYGIENE_MARKER, we need to clear it.
447 */
448 _enable_preemption_write_count(current_thread(), 0);
449 }
450
451 /*
452 * Abandon a potential preemption disable measurement. Useful for
453 * example for the idle thread, which would just spuriously
454 * trigger the threshold while actually idling, which we don't
455 * care about.
456 */
457 void
abandon_preemption_disable_measurement(void)458 abandon_preemption_disable_measurement(void)
459 {
460 const bool int_masked_debug = false;
461 bool istate = ml_set_interrupts_enabled_with_debug(false, int_masked_debug);
462 struct _preemption_disable_pcpu *pcpu = PERCPU_GET(_preemption_disable_pcpu_data);
463 if (pcpu->pdp_start.pds_mach_time != 0) {
464 pcpu->pdp_abandon = true;
465 }
466 ml_set_interrupts_enabled_with_debug(istate, int_masked_debug);
467 }
468
469 /* Inner part of disable_preemption_without_measuerments() */
470 OS_ALWAYS_INLINE
471 static void
_do_disable_preemption_without_measurements(void)472 _do_disable_preemption_without_measurements(void)
473 {
474 /*
475 * Inform _collect_preemption_disable_measurement()
476 * that we didn't really care.
477 */
478 struct _preemption_disable_pcpu *pcpu = PERCPU_GET(_preemption_disable_pcpu_data);
479 pcpu->pdp_abandon = true;
480 }
481
482 /**
483 * Reset the max interrupt durations of all CPUs.
484 */
485 void preemption_disable_reset_max_durations(void);
486 void
preemption_disable_reset_max_durations(void)487 preemption_disable_reset_max_durations(void)
488 {
489 percpu_foreach(pcpu, _preemption_disable_pcpu_data) {
490 os_atomic_store(&pcpu->pdp_max_mach_duration, 0, relaxed);
491 }
492 }
493
494 unsigned int preemption_disable_get_max_durations(uint64_t *durations, size_t count);
495 unsigned int
preemption_disable_get_max_durations(uint64_t * durations,size_t count)496 preemption_disable_get_max_durations(uint64_t *durations, size_t count)
497 {
498 int cpu = 0;
499 percpu_foreach(pcpu, _preemption_disable_pcpu_data) {
500 if (cpu < count) {
501 durations[cpu++] = os_atomic_load(&pcpu->pdp_max_mach_duration, relaxed);
502 }
503 }
504 return cpu;
505 }
506
507 /*
508 * Skip predicate for sched_preemption_disable, which would trigger
509 * spuriously when kprintf spam is enabled.
510 */
511 bool
kprintf_spam_mt_pred(struct machine_timeout_spec const __unused * spec)512 kprintf_spam_mt_pred(struct machine_timeout_spec const __unused *spec)
513 {
514 bool const kprintf_spam_enabled = !(disable_kprintf_output || disable_serial_output);
515 return kprintf_spam_enabled;
516 }
517
518 /*
519 * Abandon function exported for AppleCLPC, as a workaround to rdar://91668370.
520 *
521 * Only for AppleCLPC!
522 */
523 void
sched_perfcontrol_abandon_preemption_disable_measurement(void)524 sched_perfcontrol_abandon_preemption_disable_measurement(void)
525 {
526 abandon_preemption_disable_measurement();
527 }
528
529 #else /* SCHED_HYGIENE_DEBUG */
530
531 void
sched_perfcontrol_abandon_preemption_disable_measurement(void)532 sched_perfcontrol_abandon_preemption_disable_measurement(void)
533 {
534 // No-op. Function is exported, so needs to be defined
535 }
536
537 #endif /* SCHED_HYGIENE_DEBUG */
538