1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * processor.h: Processor and processor-related definitions.
61 */
62
63 #ifndef _KERN_PROCESSOR_H_
64 #define _KERN_PROCESSOR_H_
65
66 #include <mach/boolean.h>
67 #include <mach/kern_return.h>
68 #include <kern/kern_types.h>
69
70 #include <sys/cdefs.h>
71
72 #if defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS
73 #include <kern/bits.h>
74 #include <kern/sched_common.h>
75 #include <kern/sched_urgency.h>
76 #include <mach/sfi_class.h>
77 #include <kern/circle_queue.h>
78 #endif /* defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS */
79
80 #ifdef MACH_KERNEL_PRIVATE
81 #include <mach/mach_types.h>
82 #include <kern/ast.h>
83 #include <kern/cpu_number.h>
84 #include <kern/smp.h>
85 #include <kern/simple_lock.h>
86 #include <kern/locks.h>
87 #include <kern/percpu.h>
88 #include <kern/queue.h>
89 #include <kern/recount.h>
90 #include <kern/sched.h>
91 #include <kern/timer.h>
92 #include <kern/sched_clutch.h>
93 #include <kern/timer_call.h>
94 #include <kern/assert.h>
95 #include <machine/limits.h>
96 #endif
97
98 __BEGIN_DECLS __ASSUME_PTR_ABI_SINGLE_BEGIN
99
100 #if defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS
101
102 /*
103 * Processor state is accessed by locking the scheduling lock
104 * for the assigned processor set.
105 *
106 * --- PENDING_OFFLINE <
107 * / \
108 * _/ \
109 * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING
110 * \_________________^ ^ ^______/ /
111 * \__________________/
112 *
113 * The transition from offline to start and idle to dispatching
114 * is externally driven as a a directive. However these
115 * are paired with a handshake by the processor itself
116 * to indicate that it has completed a transition of indeterminate
117 * length (for example, the DISPATCHING->RUNNING or START->RUNNING
118 * transitions must occur on the processor itself).
119 *
120 * The boot processor has some special cases, and skips the START state,
121 * since it has already bootstrapped and is ready to context switch threads.
122 *
123 * When a processor is in DISPATCHING or RUNNING state, the current_pri,
124 * current_thmode, and deadline fields should be set, so that other
125 * processors can evaluate if it is an appropriate candidate for preemption.
126 */
127 #if defined(CONFIG_SCHED_DEFERRED_AST)
128 /*
129 * --- PENDING_OFFLINE <
130 * / \
131 * _/ \
132 * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING
133 * \_________________^ ^ ^______/ ^_____ / /
134 * \__________________/
135 *
136 * A DISPATCHING processor may be put back into IDLE, if another
137 * processor determines that the target processor will have nothing to do
138 * upon reaching the RUNNING state. This is racy, but if the target
139 * responds and becomes RUNNING, it will not break the processor state
140 * machine.
141 *
142 * This change allows us to cancel an outstanding signal/AST on a processor
143 * (if such an operation is supported through hardware or software), and
144 * push the processor back into the IDLE state as a power optimization.
145 */
146 #endif /* defined(CONFIG_SCHED_DEFERRED_AST) */
147
148 typedef enum {
149 PROCESSOR_OFF_LINE = 0, /* Not booted or off-line */
150 /* PROCESSOR_SHUTDOWN = 1, Going off-line, but schedulable. No longer used. */
151 PROCESSOR_START = 2, /* Being started */
152 PROCESSOR_PENDING_OFFLINE = 3, /* Going off-line, not schedulable */
153 PROCESSOR_IDLE = 4, /* Idle (available) */
154 PROCESSOR_DISPATCHING = 5, /* Dispatching (idle -> active) */
155 PROCESSOR_RUNNING = 6, /* Normal execution */
156 PROCESSOR_STATE_LEN = (PROCESSOR_RUNNING + 1)
157 } processor_state_t;
158
159 typedef enum {
160 PSET_SMP = 0,
161 #if __AMP__
162 PSET_AMP_E = 1,
163 PSET_AMP_P = 2,
164 #endif /* __AMP__ */
165 MAX_PSET_TYPES,
166 } pset_cluster_type_t;
167
168 #if __AMP__
169
170 #define MAX_AMP_CLUSTER_TYPES (MAX_PSET_TYPES - 1)
171
172 typedef enum {
173 SCHED_PERFCTL_POLICY_DEFAULT, /* static policy: set at boot */
174 SCHED_PERFCTL_POLICY_FOLLOW_GROUP, /* dynamic policy: perfctl_class follows thread group across amp clusters */
175 SCHED_PERFCTL_POLICY_RESTRICT_E, /* dynamic policy: limits perfctl_class to amp e cluster */
176 } sched_perfctl_class_policy_t;
177
178 extern _Atomic sched_perfctl_class_policy_t sched_perfctl_policy_util;
179 extern _Atomic sched_perfctl_class_policy_t sched_perfctl_policy_bg;
180
181 #endif /* __AMP__ */
182
183 typedef bitmap_t cpumap_t;
184
185 struct pulled_thread_queue {
186 circle_queue_head_t ptq_threadq;
187 cpumap_t ptq_needs_smr_cpu_down;
188 bool ptq_queue_active;
189 };
190
191 extern __result_use_check struct pulled_thread_queue *
192 pulled_thread_queue_prepare(void);
193
194 /* Ensure the correct caller is blamed for preemption hygiene panics */
195 __not_tail_called
196 extern void
197 pulled_thread_queue_flush(struct pulled_thread_queue * threadq);
198
199 extern void
200 pulled_thread_queue_enqueue(
201 struct pulled_thread_queue * threadq,
202 thread_t thread);
203
204 extern void
205 pulled_thread_queue_needs_smr_cpu_down(
206 struct pulled_thread_queue * threadq,
207 int cpu_id);
208
209 #if __arm64__
210
211 extern cluster_type_t pset_cluster_type_to_cluster_type(pset_cluster_type_t pset_cluster_type);
212 extern pset_cluster_type_t cluster_type_to_pset_cluster_type(cluster_type_t cluster_type);
213
214 /*
215 * pset_execution_time_t
216 *
217 * The pset_execution_time_t type is used to maintain the average
218 * execution time of threads on a pset. Since the avg. execution time is
219 * updated from contexts where the pset lock is not held, it uses a
220 * double-wide RMW loop to update these values atomically.
221 */
222 typedef union {
223 struct {
224 uint64_t pset_avg_thread_execution_time;
225 uint64_t pset_execution_time_last_update;
226 };
227 unsigned __int128 pset_execution_time_packed;
228 } pset_execution_time_t;
229
230 #endif /* __arm64__ */
231
232 struct processor_set {
233 int pset_id;
234 int online_processor_count;
235 int cpu_set_low, cpu_set_hi;
236 int cpu_set_count;
237 int last_chosen;
238
239 #if CONFIG_SCHED_EDGE
240 uint64_t pset_load_average[TH_BUCKET_SCHED_MAX];
241 /*
242 * Count of threads running or enqueued on the cluster (not including threads enqueued in a processor-bound runq).
243 * Updated atomically per scheduling bucket, around the same time as pset_load_average
244 */
245 uint32_t pset_runnable_depth[TH_BUCKET_SCHED_MAX];
246 #elif __AMP__
247 uint64_t load_average;
248 #endif /* !CONFIG_SCHED_EDGE && __AMP__ */
249 uint64_t pset_load_last_update;
250 cpumap_t cpu_bitmask;
251 cpumap_t recommended_bitmask;
252 cpumap_t cpu_state_map[PROCESSOR_STATE_LEN];
253 #if CONFIG_SCHED_SMT
254 cpumap_t primary_map;
255 #endif /* CONFIG_SCHED_SMT */
256 cpumap_t realtime_map;
257 cpumap_t cpu_available_map;
258
259 #define SCHED_PSET_TLOCK (1)
260 #if defined(SCHED_PSET_TLOCK)
261 /* TODO: reorder struct for temporal cache locality */
262 __attribute__((aligned(128))) lck_ticket_t sched_lock;
263 #else /* SCHED_PSET_TLOCK*/
264 __attribute__((aligned(128))) lck_spin_t sched_lock; /* lock for above */
265 #endif /* SCHED_PSET_TLOCK*/
266
267 struct run_queue pset_runq; /* runq for this processor set, used by the amp and dualq scheduler policies */
268 struct rt_queue rt_runq; /* realtime runq for this processor set */
269 /*
270 * stealable_rt_threads_earliest_deadline stores the earliest deadline of
271 * the rt_runq if this pset has stealable RT threads, and RT_DEADLINE_NONE
272 * otherwise.
273 *
274 * It can only be read outside of the pset lock in sched_rt_steal_thread as
275 * a hint for which pset to lock. It must be re-checked under the lock
276 * before relying on its value to dequeue a thread.
277 *
278 * Updates are made under the pset lock by pset_update_rt_stealable_state.
279 */
280 _Atomic uint64_t stealable_rt_threads_earliest_deadline;
281 #if CONFIG_SCHED_CLUTCH
282 struct sched_clutch_root pset_clutch_root; /* clutch hierarchy root */
283 #endif /* CONFIG_SCHED_CLUTCH */
284
285 /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */
286 cpumap_t pending_AST_URGENT_cpu_mask;
287 cpumap_t pending_AST_PREEMPT_cpu_mask;
288 #if defined(CONFIG_SCHED_DEFERRED_AST)
289 /*
290 * A separate mask, for ASTs that we may be able to cancel. This is dependent on
291 * some level of support for requesting an AST on a processor, and then quashing
292 * that request later.
293 *
294 * The purpose of this field (and the associated codepaths) is to infer when we
295 * no longer need a processor that is DISPATCHING to come up, and to prevent it
296 * from coming out of IDLE if possible. This should serve to decrease the number
297 * of spurious ASTs in the system, and let processors spend longer periods in
298 * IDLE.
299 */
300 cpumap_t pending_deferred_AST_cpu_mask;
301 #endif /* defined(CONFIG_SCHED_DEFERRED_AST) */
302 cpumap_t pending_spill_cpu_mask;
303 cpumap_t rt_pending_spill_cpu_mask;
304
305 struct ipc_port * pset_self; /* port for operations */
306 struct ipc_port * pset_name_self; /* port for information */
307
308 processor_set_t pset_list; /* chain of associated psets */
309 pset_node_t node;
310 uint32_t pset_cluster_id;
311
312 /*
313 * Currently the scheduler uses a mix of pset_cluster_type_t & cluster_type_t
314 * for recommendations etc. It might be useful to unify these as a single type.
315 */
316 pset_cluster_type_t pset_cluster_type;
317 /*
318 * For scheduler use only:
319 * The type that this pset will be treated like for scheduling purposes
320 */
321 cluster_type_t pset_type;
322
323 #if CONFIG_SCHED_EDGE
324 cpumap_t cpu_running_foreign;
325 cpumap_t cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_COUNT];
326 sched_bucket_t cpu_running_buckets[MAX_CPUS];
327
328 bitmap_t foreign_psets[BITMAP_LEN(MAX_PSETS)];
329 bitmap_t native_psets[BITMAP_LEN(MAX_PSETS)];
330 bitmap_t local_psets[BITMAP_LEN(MAX_PSETS)];
331 bitmap_t remote_psets[BITMAP_LEN(MAX_PSETS)];
332 pset_execution_time_t pset_execution_time[TH_BUCKET_SCHED_MAX];
333 uint64_t pset_cluster_shared_rsrc_load[CLUSTER_SHARED_RSRC_TYPE_COUNT];
334 _Atomic sched_clutch_edge sched_edges[MAX_PSETS][TH_BUCKET_SCHED_MAX];
335 sched_pset_search_order_t spill_search_order[TH_BUCKET_SCHED_MAX];
336 /*
337 * Recommended width of threads (one per core) or shared resource threads
338 * (one per cluster), if this is the preferred pset.
339 */
340 uint8_t max_parallel_cores[TH_BUCKET_SCHED_MAX];
341 uint8_t max_parallel_clusters[TH_BUCKET_SCHED_MAX];
342 #endif /* CONFIG_SCHED_EDGE */
343
344 #if __AMP__
345 /* Writes to sched_rt_* fields are guarded by sched_available_cores_lock to
346 * prevent concurrent updates. Reads are not guaranteed to be consistent
347 * except atomicity of specific fields, as noted below */
348
349 /* sched_rt_edges controls realtime thread scheduling policies like migration and steal. */
350 sched_clutch_edge sched_rt_edges[MAX_PSETS];
351 sched_pset_search_order_t sched_rt_spill_search_order; /* should be stored/accessed atomically */
352 #if CONFIG_SCHED_EDGE
353 sched_pset_search_order_t sched_rt_steal_search_order; /* should be stored/accessed atomically */
354 #endif /* CONFIG_SCHED_EDGE */
355 #endif /* __AMP__ */
356 cpumap_t perfcontrol_cpu_preferred_bitmask;
357 cpumap_t perfcontrol_cpu_migration_bitmask;
358 int cpu_preferred_last_chosen;
359 #if CONFIG_SCHED_SMT
360 bool is_SMT; /* pset contains SMT processors */
361 #endif /* CONFIG_SCHED_SMT */
362 };
363
364 typedef bitmap_t pset_map_t;
365
366 struct pset_node {
367 processor_set_t psets; /* list of associated psets */
368
369 pset_node_t node_list; /* chain of associated nodes */
370
371 pset_cluster_type_t pset_cluster_type; /* Same as the type of all psets in this node */
372
373 pset_map_t pset_map; /* map of associated psets */
374 _Atomic pset_map_t pset_idle_map; /* psets with at least one IDLE CPU */
375 _Atomic pset_map_t pset_non_rt_map; /* psets with at least one available CPU not running a realtime thread */
376 #if CONFIG_SCHED_SMT
377 _Atomic pset_map_t pset_non_rt_primary_map;/* psets with at least one available primary CPU not running a realtime thread */
378 #endif /* CONFIG_SCHED_SMT */
379 _Atomic pset_map_t pset_recommended_map; /* psets with at least one recommended processor */
380 };
381
382 /* Boot pset node and head of the pset node linked list */
383 extern struct pset_node pset_node0;
384
385 #if __AMP__
386
387 /* Boot pset node */
388 #define pset_node0 (pset_nodes[0])
389 extern struct pset_node pset_nodes[MAX_AMP_CLUSTER_TYPES];
390 extern pset_node_t pset_node_for_pset_cluster_type(pset_cluster_type_t pset_cluster_type);
391
392 #else /* !__AMP__ */
393
394 /* Boot pset node and head of the pset node linked list */
395 extern struct pset_node pset_node0;
396
397 #endif /* !__AMP__ */
398
399 extern queue_head_t tasks, threads, corpse_tasks;
400 extern int tasks_count, terminated_tasks_count, threads_count, terminated_threads_count;
401 decl_lck_mtx_data(extern, tasks_threads_lock);
402 decl_lck_mtx_data(extern, tasks_corpse_lock);
403
404 /*
405 * The terminated tasks queue should only be inspected elsewhere by stackshot.
406 */
407 extern queue_head_t terminated_tasks;
408
409 extern queue_head_t terminated_threads;
410
411 /*
412 * Valid state transitions:
413 * not booted -> starting
414 * starting -> started not running
415 * starting -> started not waited
416 * started not running | not waited -> running
417 * running -> begin shutdown
418 * begin shutdown -> pending offline
419 * pending offline -> system sleep
420 * system sleep -> running
421 * pending offline -> cpu offline -> fully offline
422 * fully offline -> starting
423 */
424 __enum_closed_decl(processor_offline_state_t, uint8_t, {
425 /* Before it's ever booted */
426 PROCESSOR_OFFLINE_NOT_BOOTED = 0,
427
428 /* cpu_start is going to be sent */
429 PROCESSOR_OFFLINE_STARTING = 1,
430
431 /* cpu_start has been sent, but it hasn't started up yet */
432 PROCESSOR_OFFLINE_STARTED_NOT_RUNNING = 2,
433
434 /* processor has started up and began running, but nobody has wait-for-start-ed it */
435 PROCESSOR_OFFLINE_STARTED_NOT_WAITED = 3,
436
437 /* processor is running and someone confirmed this with wait for start, no state change operations are in flight */
438 PROCESSOR_OFFLINE_RUNNING = 4, /* This is the 'normal' state */
439
440 /* someone is working on asking to shut this processor down */
441 PROCESSOR_OFFLINE_BEGIN_SHUTDOWN = 5,
442
443 /* this processor has started itself on its way to offline */
444 PROCESSOR_OFFLINE_PENDING_OFFLINE = 6,
445
446 /* another processor has confirmed the processor has powered down */
447 PROCESSOR_OFFLINE_CPU_OFFLINE = 7,
448
449 /* cluster power has been disabled for this processor if it's going to be */
450 PROCESSOR_OFFLINE_FULLY_OFFLINE = 8, /* This is the finished powering down state */
451
452 /* This processor is the boot processor, and it's in the final system sleep */
453 PROCESSOR_OFFLINE_FINAL_SYSTEM_SLEEP = 9,
454
455 PROCESSOR_OFFLINE_MAX = 10,
456 });
457
458 /* Locked under the sched_available_cores_lock */
459 extern cpumap_t processor_offline_state_map[PROCESSOR_OFFLINE_MAX];
460
461
462 struct processor {
463 processor_state_t state; /* See above */
464 #if CONFIG_SCHED_SMT
465 bool is_SMT;
466 bool current_is_NO_SMT; /* cached TH_SFLAG_NO_SMT of current thread */
467 #endif /* CONFIG_SCHED_SMT */
468 bool is_recommended;
469 bool current_is_bound; /* current thread is bound to this processor */
470 bool current_is_eagerpreempt;/* current thread is TH_SFLAG_EAGERPREEMPT */
471 bool pending_nonurgent_preemption; /* RUNNING_TIMER_PREEMPT is armed */
472 struct thread *active_thread; /* thread running on processor */
473 struct thread *idle_thread; /* this processor's idle thread. */
474 struct thread *startup_thread;
475
476 processor_set_t processor_set; /* assigned set */
477
478 /*
479 * XXX All current_* fields should be grouped together, as they're
480 * updated at the same time.
481 */
482 int current_pri; /* priority of current thread */
483 sfi_class_id_t current_sfi_class; /* SFI class of current thread */
484 perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */
485 /*
486 * The cluster type recommended for the current thread, used by AMP scheduler
487 */
488 pset_cluster_type_t current_recommended_pset_type;
489 thread_urgency_t current_urgency; /* cached urgency of current thread */
490
491 #if CONFIG_THREAD_GROUPS
492 struct thread_group *current_thread_group; /* thread_group of current thread */
493 #endif /* CONFIG_THREAD_GROUPS */
494 int starting_pri; /* priority of current thread as it was when scheduled */
495 int cpu_id; /* platform numeric id */
496
497 uint64_t quantum_end; /* time when current quantum ends */
498 uint64_t last_dispatch; /* time of last dispatch */
499
500 #if KPERF
501 uint64_t kperf_last_sample_time; /* time of last kperf sample */
502 #endif /* KPERF */
503
504 uint64_t deadline; /* for next realtime thread */
505 bool first_timeslice; /* has the quantum expired since context switch */
506
507 bool must_idle; /* Needs to be forced idle as next selected thread is allowed on this processor */
508 bool next_idle_short; /* Expecting a response IPI soon, so the next idle period is likely very brief */
509
510 #if !SCHED_TEST_HARNESS
511 bool running_timers_active; /* whether the running timers should fire */
512 struct timer_call running_timers[RUNNING_TIMER_MAX];
513 #endif /* !SCHED_TEST_HARNESS */
514
515 struct run_queue runq; /* runq for this processor */
516
517 #if !SCHED_TEST_HARNESS
518 struct recount_processor pr_recount;
519 #endif /* !SCHED_TEST_HARNESS */
520
521 #if CONFIG_SCHED_SMT
522 /*
523 * Pointer to primary processor for secondary SMT processors, or a
524 * pointer to ourselves for primaries or non-SMT.
525 */
526 processor_t processor_primary;
527 processor_t processor_secondary;
528 #endif /* CONFIG_SCHED_SMT */
529 struct ipc_port *processor_self; /* port for operations */
530
531 processor_t processor_list; /* all existing processors */
532
533 uint64_t timer_call_ttd; /* current timer call time-to-deadline */
534 processor_reason_t last_startup_reason;
535 processor_reason_t last_shutdown_reason;
536 processor_reason_t last_recommend_reason;
537 processor_reason_t last_derecommend_reason;
538
539 struct pulled_thread_queue processor_threadq; /* queue of threads pulled from runq */
540 struct pulled_thread_queue processor_threadq_interrupt; /* queue of threads pulled from runq when in an interrupt handler */
541
542 /* locked by processor_start_state_lock */
543 bool processor_instartup; /* between dostartup and up */
544
545 /* Locked by the processor_updown_lock */
546 bool processor_booted; /* Has gone through processor_boot */
547
548 /* Locked by sched_available_cores_lock */
549 bool shutdown_temporary; /* Shutdown should be transparent to user - don't update CPU counts */
550 bool processor_online; /* between mark-online and mark-offline, tracked in sched_online_processors */
551
552 bool processor_inshutdown; /* is the processor between processor_shutdown and processor_startup */
553 processor_offline_state_t processor_offline_state;
554
555 #if CONFIG_SCHED_EDGE
556 _Atomic int stir_the_pot_inbox_cpu; /* ID of P-core available to be preempted for stir-the-pot */
557 #endif /* CONFIG_SCHED_EDGE */
558 };
559
560 extern bool sched_all_cpus_offline(void);
561 extern void sched_assert_not_last_online_cpu(int cpu_id);
562
563 extern processor_t processor_list;
564 decl_simple_lock_data(extern, processor_list_lock);
565
566 decl_simple_lock_data(extern, processor_start_state_lock);
567
568 /*
569 * Maximum number of CPUs supported by the scheduler. bits.h bitmap macros
570 * need to be used to support greater than 64.
571 */
572 #define MAX_SCHED_CPUS 64
573 extern processor_t __single processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */
574 extern processor_set_t __single pset_array[MAX_PSETS]; /* array indexed by pset_id */
575
576 /* Returns the processor set for the given ID, asserting on its existence. */
577 processor_set_t
578 pset_for_id_checked(pset_id_t id);
579
580 /* Returns the processor set for the given ID. */
581 OS_INLINE
582 processor_set_t
pset_for_id(pset_id_t id)583 pset_for_id(pset_id_t id)
584 {
585 extern struct processor_set pset_array_actual[MAX_PSETS];
586 return &pset_array_actual[id];
587 }
588
589 /* Boot (and default) pset */
590 extern processor_set_t sched_boot_pset;
591
592 extern uint32_t processor_avail_count;
593 extern uint32_t processor_avail_count_user;
594 #if CONFIG_SCHED_SMT
595 extern uint32_t primary_processor_avail_count_user;
596 #endif /* CONFIG_SCHED_SMT */
597
598 #define cpumap_foreach(cpu_id, cpumap) \
599 for (int cpu_id = lsb_first(cpumap); \
600 (cpu_id) >= 0; \
601 cpu_id = lsb_next((cpumap), cpu_id))
602
603 #define foreach_node(node) \
604 for (pset_node_t node = &pset_node0; node != NULL; node = node->node_list)
605
606 #define foreach_pset_id(pset_id, node) \
607 for (int pset_id = lsb_first((node)->pset_map); \
608 pset_id >= 0; \
609 pset_id = lsb_next((node)->pset_map, pset_id))
610
611 cpumap_t pset_available_cpumap(processor_set_t pset);
612
613 /*
614 * All of the operations on a processor that change the processor count
615 * published to userspace and kernel.
616 */
617 __enum_closed_decl(processor_mode_t, uint8_t, {
618 PCM_RECOMMENDED = 0, /* processor->is_recommended */
619 PCM_TEMPORARY = 1, /* processor->shutdown_temporary */
620 PCM_ONLINE = 2, /* processor->processor_online */
621 });
622
623 extern void sched_processor_change_mode_locked(processor_t processor, processor_mode_t pcm_mode, bool value);
624
625 extern processor_t current_processor(void);
626
627 #if !SCHED_TEST_HARNESS
628
629 #define master_processor PERCPU_GET_MASTER(processor)
630 PERCPU_DECL(struct processor, processor);
631
632 /* Lock macros, always acquired and released with interrupts disabled (splsched()) */
633
634 extern lck_grp_t pset_lck_grp;
635
636 #if defined(SCHED_PSET_TLOCK)
637 #define pset_lock_init(p) lck_ticket_init(&(p)->sched_lock, &pset_lck_grp)
638 #define pset_lock(p) lck_ticket_lock(&(p)->sched_lock, &pset_lck_grp)
639 #define pset_unlock(p) lck_ticket_unlock(&(p)->sched_lock)
640 #define pset_assert_locked(p) lck_ticket_assert_owned(&(p)->sched_lock)
641 #else /* SCHED_PSET_TLOCK*/
642 #define pset_lock_init(p) lck_spin_init(&(p)->sched_lock, &pset_lck_grp, NULL)
643 #define pset_lock(p) lck_spin_lock_grp(&(p)->sched_lock, &pset_lck_grp)
644 #define pset_unlock(p) lck_spin_unlock(&(p)->sched_lock)
645 #define pset_assert_locked(p) LCK_SPIN_ASSERT(&(p)->sched_lock, LCK_ASSERT_OWNED)
646 #endif /*!SCHED_PSET_TLOCK*/
647
648 inline static processor_set_t
change_locked_pset(processor_set_t current_pset,processor_set_t new_pset)649 change_locked_pset(processor_set_t current_pset, processor_set_t new_pset)
650 {
651 if (current_pset != new_pset) {
652 pset_unlock(current_pset);
653 pset_lock(new_pset);
654 }
655
656 return new_pset;
657 }
658
659 #endif /* !SCHED_TEST_HARNESS */
660
661 extern void processor_bootstrap(void);
662
663 extern void processor_init(
664 processor_t processor,
665 int cpu_id,
666 processor_set_t processor_set);
667
668 #if CONFIG_SCHED_SMT
669 extern void processor_set_primary(
670 processor_t processor,
671 processor_t primary);
672 #endif /* CONFIG_SCHED_SMT */
673
674 extern void
675 processor_update_offline_state(processor_t processor, processor_offline_state_t new_state);
676 extern void
677 processor_update_offline_state_locked(processor_t processor, processor_offline_state_t new_state);
678
679 extern void processor_doshutdown(
680 processor_t processor,
681 bool is_final_system_sleep);
682
683 __enum_closed_decl(processor_start_kind_t, uint8_t, {
684 PROCESSOR_FIRST_BOOT = 0,
685 PROCESSOR_BEFORE_ENTERING_SLEEP = 1,
686 PROCESSOR_WAKE_FROM_SLEEP = 2,
687 PROCESSOR_CLUSTER_POWERDOWN_SUSPEND = 3,
688 PROCESSOR_CLUSTER_POWERDOWN_RESUME = 4,
689 PROCESSOR_POWERED_CORES_CHANGE = 5,
690 });
691
692 extern void processor_wait_for_start(
693 processor_t processor,
694 processor_start_kind_t start_kind);
695
696 extern kern_return_t processor_start_from_user(
697 processor_t processor);
698 extern kern_return_t processor_start_from_kext(
699 processor_t processor);
700 extern kern_return_t processor_exit_from_kext(
701 processor_t processor);
702
703
704 extern void processor_start_reason(
705 processor_t processor,
706 processor_reason_t reason);
707 extern void processor_exit_reason(
708 processor_t processor,
709 processor_reason_t reason,
710 bool is_system_sleep);
711
712 extern kern_return_t sched_processor_exit_user(processor_t processor);
713 extern kern_return_t sched_processor_start_user(processor_t processor);
714
715 extern bool sched_mark_processor_online(processor_t processor, processor_reason_t reason);
716 extern void sched_mark_processor_offline(processor_t processor, bool is_final_system_sleep);
717
718 extern processor_set_t processor_pset(
719 processor_t processor);
720
721 extern pset_node_t pset_node_root(void);
722
723 extern processor_set_t pset_create(
724 cluster_type_t cluster_type,
725 uint32_t pset_cluster_id,
726 int pset_id);
727
728 extern void pset_init(
729 processor_set_t pset,
730 pset_node_t node);
731
732 #if !SCHED_TEST_HARNESS
733
734 extern lck_mtx_t cluster_powerdown_lock;
735 extern lck_mtx_t processor_updown_lock;
736
737 extern bool sched_is_in_sleep(void);
738 extern bool sched_is_cpu_init_completed(void);
739
740 extern void processor_queue_shutdown(
741 processor_t processor);
742
743 extern kern_return_t processor_info_count(
744 processor_flavor_t flavor,
745 mach_msg_type_number_t *count);
746
747 extern void processor_cpu_load_info(
748 processor_t processor,
749 natural_t ticks[static CPU_STATE_MAX]);
750
751 extern void machine_run_count(
752 uint32_t count);
753
754 #if defined(__x86_64__)
755 extern processor_t machine_choose_processor(
756 processor_set_t pset,
757 processor_t processor);
758 #endif /* __x86_64__ */
759
760 #endif /* !SCHED_TEST_HARNESS */
761
762 inline static processor_set_t
next_pset(processor_set_t pset)763 next_pset(processor_set_t pset)
764 {
765 pset_map_t map = pset->node->pset_map;
766
767 int pset_id = lsb_next(map, pset->pset_id);
768 if (pset_id == -1) {
769 pset_id = lsb_first(map);
770 }
771
772 return pset_for_id((pset_id_t)pset_id);
773 }
774
775 #define PSET_THING_TASK 0
776 #define PSET_THING_THREAD 1
777
778 extern pset_cluster_type_t recommended_pset_type(
779 thread_t thread);
780
781 extern void processor_state_update_idle(
782 processor_t processor);
783
784 extern void processor_state_update_from_thread(
785 processor_t processor,
786 thread_t thread,
787 boolean_t pset_lock_held);
788
789 #if CONFIG_SCHED_EDGE
790 extern cluster_type_t pset_type_for_id(uint32_t cluster_id);
791 #endif /* CONFIG_SCHED_EDGE */
792
793 extern void
794 pset_update_processor_state(processor_set_t pset, processor_t processor, uint new_state);
795
796 decl_simple_lock_data(extern, sched_available_cores_lock);
797
798 #endif /* defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS */
799
800 #ifdef KERNEL_PRIVATE
801
802 /* Private KPI */
803 extern processor_t cpu_to_processor(int cpu);
804
805 /*!
806 * @function sched_enable_acc_rail
807 * @abstract Enable shared voltage rail for a single ACC block.
808 * @param die_id 0-based die number indicating which die the ACC is on.
809 * @param die_cluster_id 0 for the first cluster on the die, 1 for the second, ...
810 * @discussion Called from the PMGR driver. On systems where ANE and PACC
811 * share a voltage rail, the PMGR driver calls into XNU prior to
812 * accessing the ANE hardware, to ensure that the ANE block
813 * is powered. This will block until the rail has been enabled,
814 * and it must be called from a schedulable context.
815 *
816 * This should not be called on systems without a shared ANE/ACC rail.
817 * The caller is responsible for knowing which die/cluster needs to
818 * be forced on, in order to allow access to the ANE block.
819 */
820 extern void sched_enable_acc_rail(unsigned int die_id, unsigned int die_cluster_id);
821
822 /*!
823 * @function sched_disable_acc_rail
824 * @abstract Disable voltage rail for a single ACC block.
825 * @param die_id 0-based die number indicating which die the ACC is on.
826 * @param die_cluster_id 0 for the first cluster on the die, 1 for the second, ...
827 * @discussion Tells XNU that the shared ACC voltage rail can be safely disabled.
828 * This may or may not cut voltage immediately. Must be called from a
829 * schedulable context.
830 */
831 extern void sched_disable_acc_rail(unsigned int die_id, unsigned int die_cluster_id);
832
833 /*
834 * Private KPI with CLPC
835 *
836 * Update the scheduler with the set of cores that should be used to dispatch new threads.
837 * Non-recommended cores can still be used to field interrupts or run bound threads.
838 * This should be called with interrupts enabled and no scheduler locks held.
839 */
840 #define ALL_CORES_RECOMMENDED (~(uint64_t)0)
841 #define ALL_CORES_POWERED (~(uint64_t)0)
842
843 extern void sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores);
844 extern void sched_perfcontrol_update_recommended_cores_reason(uint64_t recommended_cores, processor_reason_t reason, uint32_t flags);
845
846 /* Request a change to the powered cores mask that CLPC wants. Does not block waiting for completion. */
847 extern void sched_perfcontrol_update_powered_cores(uint64_t powered_cores, processor_reason_t reason, uint32_t flags);
848
849 /* Reevaluate the thread placement decision on cpu_id and force a preemption if necessary. */
850 extern bool sched_perfcontrol_check_oncore_thread_preemption(uint64_t flags, int cpu_id);
851
852 #endif /* KERNEL_PRIVATE */
853
854 #ifdef XNU_KERNEL_PRIVATE
855
856 extern bool support_bootcpu_shutdown;
857 extern bool enable_processor_exit;
858 extern unsigned int processor_count;
859
860 #if CONFIG_SCHED_SMT
861 extern int sched_enable_smt;
862
863 extern kern_return_t enable_smt_processors(bool enable);
864 #endif /* CONFIG_SCHED_SMT */
865
866 extern void sched_override_available_cores_for_sleep(void);
867 extern void sched_restore_available_cores_after_sleep(void);
868 extern bool processor_should_kprintf(processor_t processor, bool starting);
869 extern void suspend_cluster_powerdown(void);
870 extern void resume_cluster_powerdown(void);
871 extern kern_return_t suspend_cluster_powerdown_from_user(void);
872 extern kern_return_t resume_cluster_powerdown_from_user(void);
873 extern int get_cluster_powerdown_user_suspended(void);
874
875 extern void processor_wake(
876 processor_t processor);
877 extern void processor_sleep(
878 processor_t processor);
879 extern void processor_boot(
880 processor_t processor);
881 extern kern_return_t processor_exit_from_user(
882 processor_t processor);
883
884 #endif /* XNU_KERNEL_PRIVATE */
885
886 __ASSUME_PTR_ABI_SINGLE_END __END_DECLS
887
888 #endif /* _KERN_PROCESSOR_H_ */
889