1 /*
2 * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #include <kern/ast.h>
58 #include <kern/counter.h>
59 #include <kern/misc_protos.h>
60 #include <kern/queue.h>
61 #include <kern/sched_prim.h>
62 #include <kern/thread.h>
63 #include <kern/processor.h>
64 #include <kern/restartable.h>
65 #include <kern/spl.h>
66 #include <kern/sfi.h>
67 #if CONFIG_TELEMETRY
68 #include <kern/telemetry.h>
69 #endif
70 #include <kern/waitq.h>
71 #include <kern/ledger.h>
72 #include <kern/machine.h>
73 #include <kperf/kperf_kpc.h>
74 #include <mach/policy.h>
75 #include <security/mac_mach_internal.h> // for MACF AST hook
76 #include <stdatomic.h>
77
78 #if CONFIG_ARCADE
79 #include <kern/arcade.h>
80 #endif
81
82 static void __attribute__((noinline, noreturn, disable_tail_calls))
thread_preempted(__unused void * parameter,__unused wait_result_t result)83 thread_preempted(__unused void* parameter, __unused wait_result_t result)
84 {
85 /*
86 * We've been scheduled again after a userspace preemption,
87 * try again to return to userspace.
88 */
89 thread_exception_return();
90 }
91
92 /*
93 * Create a dedicated frame to clarify that this thread has been preempted
94 * while running in kernel space.
95 */
96 static void __attribute__((noinline, disable_tail_calls))
thread_preempted_in_kernel(ast_t urgent_reason)97 thread_preempted_in_kernel(ast_t urgent_reason)
98 {
99 thread_block_reason(THREAD_CONTINUE_NULL, NULL, urgent_reason);
100
101 assert(ml_get_interrupts_enabled() == FALSE);
102 }
103
104 /*
105 * AST_URGENT was detected while in kernel mode
106 * Called with interrupts disabled, returns the same way
107 * Must return to caller
108 */
109 void
ast_taken_kernel(void)110 ast_taken_kernel(void)
111 {
112 assert(ml_get_interrupts_enabled() == FALSE);
113
114 thread_t thread = current_thread();
115
116 /* Idle threads handle preemption themselves */
117 if ((thread->state & TH_IDLE)) {
118 ast_off(AST_PREEMPTION);
119 return;
120 }
121
122 /*
123 * It's possible for this to be called after AST_URGENT
124 * has already been handled, due to races in enable_preemption
125 */
126 if (ast_peek(AST_URGENT) != AST_URGENT) {
127 return;
128 }
129
130 /*
131 * Don't preempt if the thread is already preparing to block.
132 * TODO: the thread can cheese this with clear_wait()
133 */
134 if (waitq_wait_possible(thread) == FALSE) {
135 /* Consume AST_URGENT or the interrupt will call us again */
136 ast_consume(AST_URGENT);
137 return;
138 }
139
140 /* TODO: Should we csw_check again to notice if conditions have changed? */
141
142 ast_t urgent_reason = ast_consume(AST_PREEMPTION);
143
144 assert(urgent_reason & AST_PREEMPT);
145
146 /* We've decided to try context switching */
147 thread_preempted_in_kernel(urgent_reason);
148 }
149
150 /*
151 * An AST flag was set while returning to user mode
152 * Called with interrupts disabled, returns with interrupts enabled
153 * May call continuation instead of returning
154 */
155 void
ast_taken_user(void)156 ast_taken_user(void)
157 {
158 assert(ml_get_interrupts_enabled() == FALSE);
159
160 thread_t thread = current_thread();
161 task_t task = get_threadtask(thread);
162
163 /* We are about to return to userspace, there must not be a pending wait */
164 assert(waitq_wait_possible(thread));
165 assert((thread->state & TH_IDLE) == 0);
166
167 /* TODO: Add more 'return to userspace' assertions here */
168
169 /*
170 * If this thread was urgently preempted in userspace,
171 * take the preemption before processing the ASTs.
172 * The trap handler will call us again if we have more ASTs, so it's
173 * safe to block in a continuation here.
174 */
175 if (ast_peek(AST_URGENT) == AST_URGENT) {
176 ast_t urgent_reason = ast_consume(AST_PREEMPTION);
177
178 assert(urgent_reason & AST_PREEMPT);
179
180 /* TODO: Should we csw_check again to notice if conditions have changed? */
181
182 thread_block_reason(thread_preempted, NULL, urgent_reason);
183 /* NOTREACHED */
184 }
185
186 /*
187 * AST_KEVENT does not send an IPI when setting the ast for a thread running in parallel
188 * on a different processor. Only the ast bit on the thread will be set.
189 *
190 * Force a propagate for concurrent updates without an IPI.
191 */
192 ast_propagate(thread);
193
194 /*
195 * Consume all non-preemption processor ASTs matching reasons
196 * because we're handling them here.
197 *
198 * If one of the AST handlers blocks in a continuation,
199 * we'll reinstate the unserviced thread-level AST flags
200 * from the thread to the processor on context switch.
201 * If one of the AST handlers sets another AST,
202 * the trap handler will call ast_taken_user again.
203 *
204 * We expect the AST handlers not to thread_exception_return
205 * without an ast_propagate or context switch to reinstate
206 * the per-processor ASTs.
207 *
208 * TODO: Why are AST_DTRACE and AST_KPERF not per-thread ASTs?
209 */
210 ast_t reasons = ast_consume(AST_PER_THREAD | AST_KPERF | AST_DTRACE);
211
212 ml_set_interrupts_enabled(TRUE);
213
214 #if CONFIG_DTRACE
215 if (reasons & AST_DTRACE) {
216 dtrace_ast();
217 }
218 #endif
219
220 #ifdef MACH_BSD
221 if (reasons & AST_BSD) {
222 thread_ast_clear(thread, AST_BSD);
223 bsd_ast(thread);
224 }
225 #endif
226
227 #if CONFIG_MACF
228 if (reasons & AST_MACF) {
229 thread_ast_clear(thread, AST_MACF);
230 mac_thread_userret(thread);
231 }
232 #endif
233
234 #if CONFIG_ARCADE
235 if (reasons & AST_ARCADE) {
236 thread_ast_clear(thread, AST_ARCADE);
237 arcade_ast(thread);
238 }
239 #endif
240
241 if (reasons & AST_APC) {
242 thread_ast_clear(thread, AST_APC);
243 thread_apc_ast(thread);
244 }
245
246 if (reasons & AST_GUARD) {
247 thread_ast_clear(thread, AST_GUARD);
248 guard_ast(thread);
249 }
250
251 if (reasons & AST_LEDGER) {
252 thread_ast_clear(thread, AST_LEDGER);
253 ledger_ast(thread);
254 }
255
256 if (reasons & AST_KPERF) {
257 thread_ast_clear(thread, AST_KPERF);
258 #if CONFIG_CPU_COUNTERS
259 kperf_kpc_thread_ast(thread);
260 #endif /* CONFIG_CPU_COUNTERS */
261 }
262
263 if (reasons & AST_RESET_PCS) {
264 thread_ast_clear(thread, AST_RESET_PCS);
265 thread_reset_pcs_ast(task, thread);
266 }
267
268 if (reasons & AST_KEVENT) {
269 thread_ast_clear(thread, AST_KEVENT);
270 uint16_t bits = atomic_exchange(&thread->kevent_ast_bits, 0);
271 if (bits) {
272 kevent_ast(thread, bits);
273 }
274 }
275
276 if (reasons & AST_PROC_RESOURCE) {
277 thread_ast_clear(thread, AST_PROC_RESOURCE);
278 task_port_space_ast(task);
279 #if MACH_BSD
280 proc_filedesc_ast(task);
281 #endif /* MACH_BSD */
282 }
283
284 #if CONFIG_TELEMETRY
285 if (reasons & AST_TELEMETRY_ALL) {
286 ast_t telemetry_reasons = reasons & AST_TELEMETRY_ALL;
287 thread_ast_clear(thread, AST_TELEMETRY_ALL);
288 telemetry_ast(thread, telemetry_reasons);
289 }
290 #endif
291
292 #if MACH_ASSERT
293 if (reasons & AST_DEBUG_ASSERT) {
294 thread_ast_clear(thread, AST_DEBUG_ASSERT);
295 thread_debug_return_to_user_ast(thread);
296 }
297 #endif
298
299 spl_t s = splsched();
300
301 #if CONFIG_SCHED_SFI
302 /*
303 * SFI is currently a per-processor AST, not a per-thread AST
304 * TODO: SFI should be a per-thread AST
305 */
306 if (ast_consume(AST_SFI) == AST_SFI) {
307 sfi_ast(thread);
308 }
309 #endif
310
311 /* We are about to return to userspace, there must not be a pending wait */
312 assert(waitq_wait_possible(thread));
313
314 /*
315 * We've handled all per-thread ASTs, time to handle non-urgent preemption.
316 *
317 * We delay reading the preemption bits until now in case the thread
318 * blocks while handling per-thread ASTs.
319 *
320 * If one of the AST handlers had managed to set a new AST bit,
321 * thread_exception_return will call ast_taken again.
322 */
323 ast_t preemption_reasons = ast_consume(AST_PREEMPTION);
324
325 if (preemption_reasons & AST_PREEMPT) {
326 /* Conditions may have changed from when the AST_PREEMPT was originally set, so re-check. */
327
328 thread_lock(thread);
329 preemption_reasons = csw_check(thread, current_processor(), (preemption_reasons & AST_QUANTUM));
330 thread_unlock(thread);
331
332 #if CONFIG_SCHED_SFI
333 /* csw_check might tell us that SFI is needed */
334 if (preemption_reasons & AST_SFI) {
335 sfi_ast(thread);
336 }
337 #endif
338
339 if (preemption_reasons & AST_PREEMPT) {
340 /* switching to a continuation implicitly re-enables interrupts */
341 thread_block_reason(thread_preempted, NULL, preemption_reasons);
342 /* NOTREACHED */
343 }
344
345 /*
346 * We previously had a pending AST_PREEMPT, but csw_check
347 * decided that it should no longer be set, and to keep
348 * executing the current thread instead.
349 * Clear the pending preemption timer as we no longer
350 * have a pending AST_PREEMPT to time out.
351 *
352 * TODO: just do the thread block if we see AST_PREEMPT
353 * to avoid taking the pset lock twice.
354 * To do that thread block needs to be smarter
355 * about not context switching when it's not necessary
356 * e.g. the first-timeslice check for queue has priority
357 */
358 clear_pending_nonurgent_preemption(current_processor());
359 }
360
361 splx(s);
362
363 /*
364 * Here's a good place to put assertions of things which must be true
365 * upon return to userspace.
366 */
367 assert(thread->kern_promotion_schedpri == 0);
368 if (thread->rwlock_count > 0) {
369 panic("rwlock_count is %d for thread %p, possibly it still holds a rwlock", thread->rwlock_count, thread);
370 }
371 assert(thread->priority_floor_count == 0);
372
373 assert3u(0, ==, thread->sched_flags &
374 (TH_SFLAG_WAITQ_PROMOTED |
375 TH_SFLAG_RW_PROMOTED |
376 TH_SFLAG_EXEC_PROMOTED |
377 TH_SFLAG_FLOOR_PROMOTED |
378 TH_SFLAG_PROMOTED |
379 TH_SFLAG_DEPRESS));
380 }
381
382 /*
383 * Set AST flags on current processor
384 * Called at splsched
385 */
386 void
ast_on(ast_t reasons)387 ast_on(ast_t reasons)
388 {
389 ast_t *pending_ast = ast_pending();
390
391 *pending_ast |= reasons;
392 }
393
394 /*
395 * Clear AST flags on current processor
396 * Called at splsched
397 */
398 void
ast_off(ast_t reasons)399 ast_off(ast_t reasons)
400 {
401 ast_t *pending_ast = ast_pending();
402
403 *pending_ast &= ~reasons;
404 }
405
406 /*
407 * Consume the requested subset of the AST flags set on the processor
408 * Return the bits that were set
409 * Called at splsched
410 */
411 ast_t
ast_consume(ast_t reasons)412 ast_consume(ast_t reasons)
413 {
414 ast_t *pending_ast = ast_pending();
415
416 reasons &= *pending_ast;
417 *pending_ast &= ~reasons;
418
419 return reasons;
420 }
421
422 /*
423 * Read the requested subset of the AST flags set on the processor
424 * Return the bits that were set, don't modify the processor
425 * Called at splsched
426 */
427 ast_t
ast_peek(ast_t reasons)428 ast_peek(ast_t reasons)
429 {
430 ast_t *pending_ast = ast_pending();
431
432 reasons &= *pending_ast;
433
434 return reasons;
435 }
436
437 /*
438 * Re-set current processor's per-thread AST flags to those set on thread
439 * Called at splsched
440 */
441 void
ast_context(thread_t thread)442 ast_context(thread_t thread)
443 {
444 ast_t *pending_ast = ast_pending();
445
446 *pending_ast = (*pending_ast & ~AST_PER_THREAD) | thread_ast_get(thread);
447 }
448
449 /*
450 * Propagate ASTs set on a thread to the current processor
451 * Called at splsched
452 */
453 void
ast_propagate(thread_t thread)454 ast_propagate(thread_t thread)
455 {
456 ast_on(thread_ast_get(thread));
457 }
458
459 void
ast_dtrace_on(void)460 ast_dtrace_on(void)
461 {
462 ast_on(AST_DTRACE);
463 }
464