1 /*
2 * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #include <kern/ast.h>
58 #include <kern/counter.h>
59 #include <kern/cpu_quiesce.h>
60 #include <kern/misc_protos.h>
61 #include <kern/queue.h>
62 #include <kern/sched_prim.h>
63 #include <kern/thread.h>
64 #include <kern/processor.h>
65 #include <kern/restartable.h>
66 #include <kern/spl.h>
67 #include <kern/sfi.h>
68 #if CONFIG_TELEMETRY
69 #include <kern/telemetry.h>
70 #endif
71 #include <kern/waitq.h>
72 #include <kern/ledger.h>
73 #include <kern/machine.h>
74 #include <kperf/kperf_kpc.h>
75 #include <mach/policy.h>
76 #include <security/mac_mach_internal.h> // for MACF AST hook
77 #include <stdatomic.h>
78
79 #if CONFIG_ARCADE
80 #include <kern/arcade.h>
81 #endif
82
83 static void __attribute__((noinline, noreturn, disable_tail_calls))
thread_preempted(__unused void * parameter,__unused wait_result_t result)84 thread_preempted(__unused void* parameter, __unused wait_result_t result)
85 {
86 /*
87 * We've been scheduled again after a userspace preemption,
88 * try again to return to userspace.
89 */
90 thread_exception_return();
91 }
92
93 /*
94 * Create a dedicated frame to clarify that this thread has been preempted
95 * while running in kernel space.
96 */
97 static void __attribute__((noinline, disable_tail_calls))
thread_preempted_in_kernel(ast_t urgent_reason)98 thread_preempted_in_kernel(ast_t urgent_reason)
99 {
100 thread_block_reason(THREAD_CONTINUE_NULL, NULL, urgent_reason);
101
102 assert(ml_get_interrupts_enabled() == FALSE);
103 }
104
105 /*
106 * AST_URGENT was detected while in kernel mode
107 * Called with interrupts disabled, returns the same way
108 * Must return to caller
109 */
110 void
ast_taken_kernel(void)111 ast_taken_kernel(void)
112 {
113 assert(ml_get_interrupts_enabled() == FALSE);
114
115 thread_t thread = current_thread();
116
117 /* Idle threads handle preemption themselves */
118 if ((thread->state & TH_IDLE)) {
119 ast_off(AST_PREEMPTION);
120 return;
121 }
122
123 /*
124 * It's possible for this to be called after AST_URGENT
125 * has already been handled, due to races in enable_preemption
126 */
127 if (ast_peek(AST_URGENT) != AST_URGENT) {
128 return;
129 }
130
131 /*
132 * Don't preempt if the thread is already preparing to block.
133 * TODO: the thread can cheese this with clear_wait()
134 */
135 if (waitq_wait_possible(thread) == FALSE) {
136 /* Consume AST_URGENT or the interrupt will call us again */
137 ast_consume(AST_URGENT);
138 return;
139 }
140
141 /* TODO: Should we csw_check again to notice if conditions have changed? */
142
143 ast_t urgent_reason = ast_consume(AST_PREEMPTION);
144
145 assert(urgent_reason & AST_PREEMPT);
146
147 /* We've decided to try context switching */
148 thread_preempted_in_kernel(urgent_reason);
149 }
150
151 /*
152 * An AST flag was set while returning to user mode
153 * Called with interrupts disabled, returns with interrupts enabled
154 * May call continuation instead of returning
155 */
156 void
ast_taken_user(void)157 ast_taken_user(void)
158 {
159 assert(ml_get_interrupts_enabled() == FALSE);
160
161 thread_t thread = current_thread();
162 task_t task = get_threadtask(thread);
163
164 /* We are about to return to userspace, there must not be a pending wait */
165 assert(waitq_wait_possible(thread));
166 assert((thread->state & TH_IDLE) == 0);
167
168 /* TODO: Add more 'return to userspace' assertions here */
169
170 /*
171 * If this thread was urgently preempted in userspace,
172 * take the preemption before processing the ASTs.
173 * The trap handler will call us again if we have more ASTs, so it's
174 * safe to block in a continuation here.
175 */
176 if (ast_peek(AST_URGENT) == AST_URGENT) {
177 ast_t urgent_reason = ast_consume(AST_PREEMPTION);
178
179 assert(urgent_reason & AST_PREEMPT);
180
181 /* TODO: Should we csw_check again to notice if conditions have changed? */
182
183 thread_block_reason(thread_preempted, NULL, urgent_reason);
184 /* NOTREACHED */
185 }
186
187 /*
188 * AST_KEVENT does not send an IPI when setting the ast for a thread running in parallel
189 * on a different processor. Only the ast bit on the thread will be set.
190 *
191 * Force a propagate for concurrent updates without an IPI.
192 */
193 ast_propagate(thread);
194
195 /*
196 * Consume all non-preemption processor ASTs matching reasons
197 * because we're handling them here.
198 *
199 * If one of the AST handlers blocks in a continuation,
200 * we'll reinstate the unserviced thread-level AST flags
201 * from the thread to the processor on context switch.
202 * If one of the AST handlers sets another AST,
203 * the trap handler will call ast_taken_user again.
204 *
205 * We expect the AST handlers not to thread_exception_return
206 * without an ast_propagate or context switch to reinstate
207 * the per-processor ASTs.
208 *
209 * TODO: Why are AST_DTRACE and AST_KPERF not per-thread ASTs?
210 */
211 ast_t reasons = ast_consume(AST_PER_THREAD | AST_KPERF | AST_DTRACE);
212
213 ml_set_interrupts_enabled(TRUE);
214
215 #if CONFIG_DTRACE
216 if (reasons & AST_DTRACE) {
217 dtrace_ast();
218 }
219 #endif
220
221 #ifdef MACH_BSD
222 if (reasons & AST_BSD) {
223 thread_ast_clear(thread, AST_BSD);
224 bsd_ast(thread);
225 }
226 #endif
227
228 #if CONFIG_MACF
229 if (reasons & AST_MACF) {
230 thread_ast_clear(thread, AST_MACF);
231 mac_thread_userret(thread);
232 }
233 #endif
234
235 #if CONFIG_ARCADE
236 if (reasons & AST_ARCADE) {
237 thread_ast_clear(thread, AST_ARCADE);
238 arcade_ast(thread);
239 }
240 #endif
241
242 if (reasons & AST_APC) {
243 thread_ast_clear(thread, AST_APC);
244 thread_apc_ast(thread);
245 }
246
247 if (reasons & AST_GUARD) {
248 thread_ast_clear(thread, AST_GUARD);
249 guard_ast(thread);
250 }
251
252 if (reasons & AST_LEDGER) {
253 thread_ast_clear(thread, AST_LEDGER);
254 ledger_ast(thread);
255 }
256
257 if (reasons & AST_KPERF) {
258 thread_ast_clear(thread, AST_KPERF);
259 kperf_kpc_thread_ast(thread);
260 }
261
262 if (reasons & AST_RESET_PCS) {
263 thread_ast_clear(thread, AST_RESET_PCS);
264 thread_reset_pcs_ast(task, thread);
265 }
266
267 if (reasons & AST_KEVENT) {
268 thread_ast_clear(thread, AST_KEVENT);
269 uint16_t bits = atomic_exchange(&thread->kevent_ast_bits, 0);
270 if (bits) {
271 kevent_ast(thread, bits);
272 }
273 }
274
275 if (reasons & AST_PROC_RESOURCE) {
276 thread_ast_clear(thread, AST_PROC_RESOURCE);
277 task_port_space_ast(task);
278 #if MACH_BSD
279 proc_filedesc_ast(task);
280 #endif /* MACH_BSD */
281 }
282
283 #if CONFIG_TELEMETRY
284 if (reasons & AST_TELEMETRY_ALL) {
285 ast_t telemetry_reasons = reasons & AST_TELEMETRY_ALL;
286 thread_ast_clear(thread, AST_TELEMETRY_ALL);
287 telemetry_ast(thread, telemetry_reasons);
288 }
289 #endif
290
291 #if MACH_ASSERT
292 if (reasons & AST_DEBUG_ASSERT) {
293 thread_ast_clear(thread, AST_DEBUG_ASSERT);
294 thread_debug_return_to_user_ast(thread);
295 }
296 #endif
297
298 spl_t s = splsched();
299
300 #if CONFIG_SCHED_SFI
301 /*
302 * SFI is currently a per-processor AST, not a per-thread AST
303 * TODO: SFI should be a per-thread AST
304 */
305 if (ast_consume(AST_SFI) == AST_SFI) {
306 sfi_ast(thread);
307 }
308 #endif
309
310 /* We are about to return to userspace, there must not be a pending wait */
311 assert(waitq_wait_possible(thread));
312
313 /*
314 * We've handled all per-thread ASTs, time to handle non-urgent preemption.
315 *
316 * We delay reading the preemption bits until now in case the thread
317 * blocks while handling per-thread ASTs.
318 *
319 * If one of the AST handlers had managed to set a new AST bit,
320 * thread_exception_return will call ast_taken again.
321 */
322 ast_t preemption_reasons = ast_consume(AST_PREEMPTION);
323
324 if (preemption_reasons & AST_PREEMPT) {
325 /* Conditions may have changed from when the AST_PREEMPT was originally set, so re-check. */
326
327 thread_lock(thread);
328 preemption_reasons = csw_check(thread, current_processor(), (preemption_reasons & AST_QUANTUM));
329 thread_unlock(thread);
330
331 #if CONFIG_SCHED_SFI
332 /* csw_check might tell us that SFI is needed */
333 if (preemption_reasons & AST_SFI) {
334 sfi_ast(thread);
335 }
336 #endif
337
338 if (preemption_reasons & AST_PREEMPT) {
339 /* switching to a continuation implicitly re-enables interrupts */
340 thread_block_reason(thread_preempted, NULL, preemption_reasons);
341 /* NOTREACHED */
342 }
343 }
344
345 if (ast_consume(AST_UNQUIESCE) == AST_UNQUIESCE) {
346 cpu_quiescent_counter_ast();
347 }
348
349 cpu_quiescent_counter_assert_ast();
350
351 splx(s);
352
353 /*
354 * Here's a good place to put assertions of things which must be true
355 * upon return to userspace.
356 */
357 assert(thread->kern_promotion_schedpri == 0);
358 if (thread->rwlock_count > 0) {
359 panic("rwlock_count is %d for thread %p, possibly it still holds a rwlock", thread->rwlock_count, thread);
360 }
361 assert(thread->priority_floor_count == 0);
362
363 assert3u(0, ==, thread->sched_flags &
364 (TH_SFLAG_WAITQ_PROMOTED |
365 TH_SFLAG_RW_PROMOTED |
366 TH_SFLAG_EXEC_PROMOTED |
367 TH_SFLAG_FLOOR_PROMOTED |
368 TH_SFLAG_PROMOTED |
369 TH_SFLAG_DEPRESS));
370 }
371
372 /*
373 * Set AST flags on current processor
374 * Called at splsched
375 */
376 void
ast_on(ast_t reasons)377 ast_on(ast_t reasons)
378 {
379 ast_t *pending_ast = ast_pending();
380
381 *pending_ast |= reasons;
382 }
383
384 /*
385 * Clear AST flags on current processor
386 * Called at splsched
387 */
388 void
ast_off(ast_t reasons)389 ast_off(ast_t reasons)
390 {
391 ast_t *pending_ast = ast_pending();
392
393 *pending_ast &= ~reasons;
394 }
395
396 /*
397 * Consume the requested subset of the AST flags set on the processor
398 * Return the bits that were set
399 * Called at splsched
400 */
401 ast_t
ast_consume(ast_t reasons)402 ast_consume(ast_t reasons)
403 {
404 ast_t *pending_ast = ast_pending();
405
406 reasons &= *pending_ast;
407 *pending_ast &= ~reasons;
408
409 return reasons;
410 }
411
412 /*
413 * Read the requested subset of the AST flags set on the processor
414 * Return the bits that were set, don't modify the processor
415 * Called at splsched
416 */
417 ast_t
ast_peek(ast_t reasons)418 ast_peek(ast_t reasons)
419 {
420 ast_t *pending_ast = ast_pending();
421
422 reasons &= *pending_ast;
423
424 return reasons;
425 }
426
427 /*
428 * Re-set current processor's per-thread AST flags to those set on thread
429 * Called at splsched
430 */
431 void
ast_context(thread_t thread)432 ast_context(thread_t thread)
433 {
434 ast_t *pending_ast = ast_pending();
435
436 *pending_ast = (*pending_ast & ~AST_PER_THREAD) | thread_ast_get(thread);
437 }
438
439 /*
440 * Propagate ASTs set on a thread to the current processor
441 * Called at splsched
442 */
443 void
ast_propagate(thread_t thread)444 ast_propagate(thread_t thread)
445 {
446 ast_on(thread_ast_get(thread));
447 }
448
449 void
ast_dtrace_on(void)450 ast_dtrace_on(void)
451 {
452 ast_on(AST_DTRACE);
453 }
454