1 /* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 /* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or [email protected] 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 /* 57 */ 58 /* 59 * File: sched_prim.h 60 * Author: David Golub 61 * 62 * Scheduling primitive definitions file 63 * 64 */ 65 66 #ifndef _KERN_SCHED_PRIM_H_ 67 #define _KERN_SCHED_PRIM_H_ 68 69 #include <sys/cdefs.h> 70 #include <mach/boolean.h> 71 #include <mach/machine/vm_types.h> 72 #include <mach/kern_return.h> 73 #include <kern/clock.h> 74 #include <kern/kern_types.h> 75 #include <kern/percpu.h> 76 #include <kern/thread.h> 77 #include <kern/block_hint.h> 78 79 extern int thread_get_current_cpuid(void); 80 81 #ifdef MACH_KERNEL_PRIVATE 82 83 #include <kern/sched_urgency.h> 84 #include <kern/thread_group.h> 85 #include <kern/waitq.h> 86 87 /* Initialization */ 88 extern void sched_init(void); 89 90 extern void sched_startup(void); 91 92 extern void sched_timebase_init(void); 93 94 extern void pset_rt_init(processor_set_t pset); 95 96 extern void sched_rtlocal_init(processor_set_t pset); 97 98 extern rt_queue_t sched_rtlocal_runq(processor_set_t pset); 99 100 extern void sched_rtlocal_queue_shutdown(processor_t processor); 101 102 extern int64_t sched_rtlocal_runq_count_sum(void); 103 104 extern thread_t sched_rtlocal_steal_thread(processor_set_t stealing_pset, uint64_t earliest_deadline); 105 106 extern thread_t sched_rt_choose_thread(processor_set_t pset); 107 108 extern void sched_check_spill(processor_set_t pset, thread_t thread); 109 110 extern bool sched_thread_should_yield(processor_t processor, thread_t thread); 111 112 extern bool sched_steal_thread_DISABLED(processor_set_t pset); 113 extern bool sched_steal_thread_enabled(processor_set_t pset); 114 115 /* Force a preemption point for a thread and wait for it to stop running */ 116 extern boolean_t thread_stop( 117 thread_t thread, 118 boolean_t until_not_runnable); 119 120 /* Release a previous stop request */ 121 extern void thread_unstop( 122 thread_t thread); 123 124 /* Wait for a thread to stop running */ 125 extern void thread_wait( 126 thread_t thread, 127 boolean_t until_not_runnable); 128 129 /* Unblock thread on wake up */ 130 extern boolean_t thread_unblock( 131 thread_t thread, 132 wait_result_t wresult); 133 134 /* Unblock and dispatch thread */ 135 extern kern_return_t thread_go( 136 thread_t thread, 137 wait_result_t wresult, 138 bool try_handoff); 139 140 /* Check if direct handoff is allowed */ 141 extern boolean_t 142 thread_allowed_for_handoff( 143 thread_t thread); 144 145 /* Handle threads at context switch */ 146 extern void thread_dispatch( 147 thread_t old_thread, 148 thread_t new_thread); 149 150 /* Switch directly to a particular thread */ 151 extern int thread_run( 152 thread_t self, 153 thread_continue_t continuation, 154 void *parameter, 155 thread_t new_thread); 156 157 /* Resume thread with new stack */ 158 extern __dead2 void thread_continue(thread_t old_thread); 159 160 /* Invoke continuation */ 161 extern __dead2 void call_continuation( 162 thread_continue_t continuation, 163 void *parameter, 164 wait_result_t wresult, 165 boolean_t enable_interrupts); 166 167 /* 168 * Flags that can be passed to set_sched_pri 169 * to skip side effects 170 */ 171 __options_decl(set_sched_pri_options_t, uint32_t, { 172 SETPRI_DEFAULT = 0x0, 173 SETPRI_LAZY = 0x1, /* Avoid setting AST flags or sending IPIs */ 174 }); 175 176 /* Set the current scheduled priority */ 177 extern void set_sched_pri( 178 thread_t thread, 179 int16_t priority, 180 set_sched_pri_options_t options); 181 182 /* Set base priority of the specified thread */ 183 extern void sched_set_thread_base_priority( 184 thread_t thread, 185 int priority); 186 187 /* Set absolute base priority of the specified thread */ 188 extern void sched_set_kernel_thread_priority( 189 thread_t thread, 190 int priority); 191 192 193 /* Set the thread's true scheduling mode */ 194 extern void sched_set_thread_mode(thread_t thread, 195 sched_mode_t mode); 196 197 /* 198 * Set the thread's scheduling mode taking into account that the thread may have 199 * been demoted. 200 * */ 201 extern void sched_set_thread_mode_user(thread_t thread, 202 sched_mode_t mode); 203 204 /* Demote the true scheduler mode */ 205 extern void sched_thread_mode_demote(thread_t thread, 206 uint32_t reason); 207 /* Un-demote the true scheduler mode */ 208 extern void sched_thread_mode_undemote(thread_t thread, 209 uint32_t reason); 210 211 extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); 212 extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); 213 214 /* Re-evaluate base priority of thread (thread locked) */ 215 void thread_recompute_priority(thread_t thread); 216 217 /* Re-evaluate scheduled priority of thread (thread locked) */ 218 extern void thread_recompute_sched_pri( 219 thread_t thread, 220 set_sched_pri_options_t options); 221 222 /* Periodic scheduler activity */ 223 extern void sched_init_thread(void); 224 225 /* Perform sched_tick housekeeping activities */ 226 extern boolean_t can_update_priority( 227 thread_t thread); 228 229 extern void update_priority( 230 thread_t thread); 231 232 extern void lightweight_update_priority( 233 thread_t thread); 234 235 extern void sched_default_quantum_expire(thread_t thread); 236 237 /* Idle processor thread continuation */ 238 extern void idle_thread( 239 void* parameter, 240 wait_result_t result); 241 242 extern kern_return_t idle_thread_create( 243 processor_t processor); 244 245 /* Continuation return from syscall */ 246 extern void thread_syscall_return( 247 kern_return_t ret); 248 249 /* Context switch */ 250 extern wait_result_t thread_block_reason( 251 thread_continue_t continuation, 252 void *parameter, 253 ast_t reason); 254 255 __options_decl(sched_options_t, uint32_t, { 256 SCHED_NONE = 0x0, 257 SCHED_TAILQ = 0x1, 258 SCHED_HEADQ = 0x2, 259 SCHED_PREEMPT = 0x4, 260 SCHED_REBALANCE = 0x8, 261 }); 262 263 /* Reschedule thread for execution */ 264 extern void thread_setrun( 265 thread_t thread, 266 sched_options_t options); 267 268 extern processor_set_t task_choose_pset( 269 task_t task); 270 271 /* Bind the current thread to a particular processor */ 272 extern processor_t thread_bind( 273 processor_t processor); 274 275 extern bool pset_has_stealable_threads( 276 processor_set_t pset); 277 278 extern bool pset_has_stealable_rt_threads( 279 processor_set_t pset); 280 281 extern processor_set_t choose_starting_pset( 282 pset_node_t node, 283 thread_t thread, 284 processor_t *processor_hint); 285 286 extern int pset_available_cpu_count( 287 processor_set_t pset); 288 289 extern bool pset_is_recommended( 290 processor_set_t pset); 291 292 extern pset_node_t sched_choose_node( 293 thread_t thread); 294 295 /* Choose the best processor to run a thread */ 296 extern processor_t choose_processor( 297 processor_set_t pset, 298 processor_t processor, 299 thread_t thread); 300 301 extern void sched_SMT_balance( 302 processor_t processor, 303 processor_set_t pset); 304 305 extern void thread_quantum_init( 306 thread_t thread); 307 308 extern void run_queue_init( 309 run_queue_t runq); 310 311 extern thread_t run_queue_dequeue( 312 run_queue_t runq, 313 sched_options_t options); 314 315 extern boolean_t run_queue_enqueue( 316 run_queue_t runq, 317 thread_t thread, 318 sched_options_t options); 319 320 extern void run_queue_remove( 321 run_queue_t runq, 322 thread_t thread); 323 324 extern thread_t run_queue_peek( 325 run_queue_t runq); 326 327 struct sched_update_scan_context { 328 uint64_t earliest_bg_make_runnable_time; 329 uint64_t earliest_normal_make_runnable_time; 330 uint64_t earliest_rt_make_runnable_time; 331 uint64_t sched_tick_last_abstime; 332 }; 333 typedef struct sched_update_scan_context *sched_update_scan_context_t; 334 335 extern void sched_rtlocal_runq_scan(sched_update_scan_context_t scan_context); 336 337 extern void sched_pset_made_schedulable( 338 processor_t processor, 339 processor_set_t pset, 340 boolean_t drop_lock); 341 342 extern void sched_cpu_init_completed(void); 343 344 /* 345 * Enum to define various events which need IPIs. The IPI policy 346 * engine decides what kind of IPI to use based on destination 347 * processor state, thread and one of the following scheduling events. 348 */ 349 typedef enum { 350 SCHED_IPI_EVENT_BOUND_THR = 0x1, 351 SCHED_IPI_EVENT_PREEMPT = 0x2, 352 SCHED_IPI_EVENT_SMT_REBAL = 0x3, 353 SCHED_IPI_EVENT_SPILL = 0x4, 354 SCHED_IPI_EVENT_REBALANCE = 0x5, 355 SCHED_IPI_EVENT_RT_PREEMPT = 0x6, 356 } sched_ipi_event_t; 357 358 359 /* Enum to define various IPI types used by the scheduler */ 360 typedef enum { 361 SCHED_IPI_NONE = 0x0, 362 SCHED_IPI_IMMEDIATE = 0x1, 363 SCHED_IPI_IDLE = 0x2, 364 SCHED_IPI_DEFERRED = 0x3, 365 } sched_ipi_type_t; 366 367 /* The IPI policy engine behaves in the following manner: 368 * - All scheduler events which need an IPI invoke sched_ipi_action() with 369 * the appropriate destination processor, thread and event. 370 * - sched_ipi_action() performs basic checks, invokes the scheduler specific 371 * ipi_policy routine and sets pending_AST bits based on the result. 372 * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform() 373 * routine which actually sends the appropriate IPI to the destination core. 374 */ 375 extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, sched_ipi_event_t event); 376 extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); 377 378 /* sched_ipi_policy() is the global default IPI policy for all schedulers */ 379 extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, 380 boolean_t dst_idle, sched_ipi_event_t event); 381 382 /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ 383 extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, 384 processor_t dst, thread_t thread, sched_ipi_event_t event); 385 386 #if defined(CONFIG_SCHED_TIMESHARE_CORE) 387 388 extern boolean_t thread_update_add_thread(thread_t thread); 389 extern void thread_update_process_threads(void); 390 extern boolean_t runq_scan(run_queue_t runq, sched_update_scan_context_t scan_context); 391 392 #if CONFIG_SCHED_CLUTCH 393 extern boolean_t sched_clutch_timeshare_scan(queue_t thread_queue, uint16_t count, sched_update_scan_context_t scan_context); 394 #endif /* CONFIG_SCHED_CLUTCH */ 395 396 extern void sched_timeshare_init(void); 397 extern void sched_timeshare_timebase_init(void); 398 extern void sched_timeshare_maintenance_continue(void); 399 400 extern boolean_t priority_is_urgent(int priority); 401 extern uint32_t sched_timeshare_initial_quantum_size(thread_t thread); 402 403 extern int sched_compute_timeshare_priority(thread_t thread); 404 405 #endif /* CONFIG_SCHED_TIMESHARE_CORE */ 406 407 /* Remove thread from its run queue */ 408 extern boolean_t thread_run_queue_remove(thread_t thread); 409 thread_t thread_run_queue_remove_for_handoff(thread_t thread); 410 411 /* Put a thread back in the run queue after being yanked */ 412 extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options); 413 414 extern void thread_timer_expire( 415 void *thread, 416 void *p1); 417 418 extern bool thread_is_eager_preempt(thread_t thread); 419 420 extern boolean_t sched_generic_direct_dispatch_to_idle_processors; 421 422 /* Set the maximum interrupt level for the thread */ 423 __private_extern__ wait_interrupt_t thread_interrupt_level( 424 wait_interrupt_t interruptible); 425 426 __private_extern__ wait_result_t thread_mark_wait_locked( 427 thread_t thread, 428 wait_interrupt_t interruptible); 429 430 /* Wake up locked thread directly, passing result */ 431 __private_extern__ kern_return_t clear_wait_internal( 432 thread_t thread, 433 wait_result_t result); 434 435 struct sched_statistics { 436 uint32_t csw_count; 437 uint32_t preempt_count; 438 uint32_t preempted_rt_count; 439 uint32_t preempted_by_rt_count; 440 uint32_t rt_sched_count; 441 uint32_t interrupt_count; 442 uint32_t ipi_count; 443 uint32_t timer_pop_count; 444 uint32_t idle_transitions; 445 uint32_t quantum_timer_expirations; 446 }; 447 PERCPU_DECL(struct sched_statistics, sched_stats); 448 extern bool sched_stats_active; 449 450 extern void sched_stats_handle_csw( 451 processor_t processor, 452 int reasons, 453 int selfpri, 454 int otherpri); 455 456 extern void sched_stats_handle_runq_change( 457 struct runq_stats *stats, 458 int old_count); 459 460 #define SCHED_STATS_INC(field) \ 461 MACRO_BEGIN \ 462 if (__improbable(sched_stats_active)) { \ 463 PERCPU_GET(sched_stats)->field++; \ 464 } \ 465 MACRO_END 466 467 #if DEBUG 468 469 #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ 470 MACRO_BEGIN \ 471 if (__improbable(sched_stats_active)) { \ 472 sched_stats_handle_csw((processor), \ 473 (reasons), (selfpri), (otherpri)); \ 474 } \ 475 MACRO_END 476 477 478 #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ 479 MACRO_BEGIN \ 480 if (__improbable(sched_stats_active)) { \ 481 sched_stats_handle_runq_change((stats), (old_count)); \ 482 } \ 483 MACRO_END 484 485 #else /* DEBUG */ 486 487 #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0) 488 #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0) 489 490 #endif /* DEBUG */ 491 492 extern uint32_t sched_debug_flags; 493 #define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 494 #define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 495 496 #define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) \ 497 MACRO_BEGIN \ 498 if (__improbable(sched_debug_flags & \ 499 SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ 500 KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ 501 } \ 502 MACRO_END 503 504 #define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) \ 505 MACRO_BEGIN \ 506 if (__improbable(sched_debug_flags & \ 507 SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ 508 KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ 509 } \ 510 MACRO_END 511 512 /* Tells if there are "active" RT threads in the system (provided by CPU PM) */ 513 extern void active_rt_threads( 514 boolean_t active); 515 516 /* Returns the perfcontrol attribute for the thread */ 517 extern perfcontrol_class_t thread_get_perfcontrol_class( 518 thread_t thread); 519 520 /* Generic routine for Non-AMP schedulers to calculate parallelism */ 521 extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); 522 523 extern void check_monotonic_time(uint64_t ctime); 524 525 #endif /* MACH_KERNEL_PRIVATE */ 526 527 __BEGIN_DECLS 528 529 #ifdef XNU_KERNEL_PRIVATE 530 531 extern void thread_bind_cluster_type(thread_t, char cluster_type, bool soft_bind); 532 533 __options_decl(thread_bind_option_t, uint64_t, { 534 /* Unbind a previously cluster bound thread */ 535 THREAD_UNBIND = 0x1, 536 /* 537 * Soft bind the thread to the cluster; soft binding means the thread will be 538 * moved to an available cluster if the bound cluster is de-recommended/offline. 539 */ 540 THREAD_BIND_SOFT = 0x2, 541 /* 542 * Bind thread to the cluster only if it is eligible to run on that cluster. If 543 * the thread is not eligible to run on the cluster, thread_bind_cluster_id() 544 * returns KERN_INVALID_POLICY. 545 */ 546 THREAD_BIND_ELIGIBLE_ONLY = 0x4, 547 }); 548 extern kern_return_t thread_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options); 549 550 extern int sched_get_rt_n_backup_processors(void); 551 extern void sched_set_rt_n_backup_processors(int n); 552 553 extern int sched_get_rt_deadline_epsilon(void); 554 extern void sched_set_rt_deadline_epsilon(int new_epsilon_us); 555 556 /* Toggles a global override to turn off CPU Throttling */ 557 extern void sys_override_cpu_throttle(boolean_t enable_override); 558 559 extern int sched_get_powered_cores(void); 560 extern void sched_set_powered_cores(int n); 561 562 /* 563 ****************** Only exported until BSD stops using ******************** 564 */ 565 566 extern void thread_vm_bind_group_add(void); 567 568 /* Wake up thread directly, passing result */ 569 extern kern_return_t clear_wait( 570 thread_t thread, 571 wait_result_t result); 572 573 /* Start thread running */ 574 extern void thread_bootstrap_return(void) __attribute__((noreturn)); 575 576 /* Return from exception (BSD-visible interface) */ 577 extern void thread_exception_return(void) __dead2; 578 579 #define SCHED_STRING_MAX_LENGTH (48) 580 /* String declaring the name of the current scheduler */ 581 extern char sched_string[SCHED_STRING_MAX_LENGTH]; 582 583 __options_decl(thread_handoff_option_t, uint32_t, { 584 THREAD_HANDOFF_NONE = 0, 585 THREAD_HANDOFF_SETRUN_NEEDED = 0x1, 586 }); 587 588 /* Remove thread from its run queue */ 589 thread_t thread_prepare_for_handoff(thread_t thread, thread_handoff_option_t option); 590 591 /* Attempt to context switch to a specific runnable thread */ 592 extern wait_result_t thread_handoff_deallocate(thread_t thread, thread_handoff_option_t option); 593 594 __attribute__((nonnull(2))) 595 extern void thread_handoff_parameter(thread_t thread, 596 thread_continue_t continuation, void *parameter, thread_handoff_option_t) __dead2; 597 598 extern struct waitq *assert_wait_queue(event_t event); 599 600 extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority); 601 602 extern thread_t thread_wakeup_identify(event_t event, int priority); 603 604 /* 605 * sched_cond_t: 606 * 607 * A atomic condition variable used to synchronize wake/block operations on threads. 608 * Bits defined below are reserved for use by sched_prim. Remaining 609 * bits may be used by caller for additional synchronization semantics. 610 */ 611 __options_decl(sched_cond_t, uint32_t, { 612 SCHED_COND_INIT = 0x0000, /* initialize all bits to zero (inactive and not awoken) */ 613 SCHED_COND_ACTIVE = 0x0001, /* target thread is active */ 614 SCHED_COND_WAKEUP = 0x0002 /* wakeup has been issued for target thread */ 615 }); 616 typedef _Atomic sched_cond_t sched_cond_atomic_t; 617 618 /* 619 * sched_cond_init: 620 * 621 * Initialize an atomic condition variable. Note that this does not occur atomically and should be 622 * performed during thread initialization, before the condition is observable by other threads. 623 */ 624 extern void sched_cond_init( 625 sched_cond_atomic_t *cond); 626 627 /* 628 * sched_cond_signal: 629 * 630 * Wakeup the specified thread if it is waiting on this event and it has not already been issued a wakeup. 631 * 632 * parameters: 633 * thread thread to awaken 634 * cond atomic condition variable 635 */ 636 extern kern_return_t sched_cond_signal( 637 sched_cond_atomic_t *cond, 638 thread_t thread); 639 640 /* 641 * sched_cond_wait: 642 * 643 * Assert wait and block on cond if no wakeup has been issued. 644 * If a wakeup has been issued on cond since the last `sched_cond_ack`, clear_wait and 645 * return `THREAD_AWAKENED`. 646 * 647 * `sched_cond_wait` must be paired with `sched_cond_ack`. 648 * 649 * NOTE: `continuation` will only be jumped to if a wakeup has not been issued 650 * 651 * parameters: 652 * cond atomic condition variable to synchronize on 653 * interruptible interruptible value to pass to assert_wait 654 * continuation continuation if block succeeds 655 */ 656 extern wait_result_t sched_cond_wait( 657 sched_cond_atomic_t *cond, 658 wait_interrupt_t interruptible, 659 thread_continue_t continuation); 660 661 /* 662 * sched_cond_ack: 663 * 664 * Acknowledge an issued wakeup by clearing WAKEUP and setting ACTIVE (via XOR). 665 * It is the callers responsibility to ensure that the ACTIVE bit is always low prior to calling 666 * (i.e. by calling `sched_cond_wait` prior to any rerun or block). 667 * Synchronization schemes that allow for WAKEUP bit to be reset prior to wakeup 668 * (e.g. a cancellation mechanism) should check that WAKEUP was indeed cleared. 669 * 670 * e.g. 671 * ``` 672 * if (sched_cond_ack(&my_state) & SCHED_THREAD_WAKEUP) { 673 * // WAKEUP bit was no longer set by the time this thread woke up 674 * do_cancellation_policy(); 675 * } 676 * ``` 677 * 678 * parameters: 679 * cond: atomic condition variable 680 */ 681 extern sched_cond_t sched_cond_ack( 682 sched_cond_atomic_t *cond); 683 684 #endif /* XNU_KERNEL_PRIVATE */ 685 686 #ifdef KERNEL_PRIVATE 687 /* Set pending block hint for a particular object before we go into a wait state */ 688 extern void thread_set_pending_block_hint( 689 thread_t thread, 690 block_hint_t block_hint); 691 692 #define QOS_PARALLELISM_COUNT_LOGICAL 0x1 693 #define QOS_PARALLELISM_REALTIME 0x2 694 #define QOS_PARALLELISM_CLUSTER_SHARED_RESOURCE 0x4 695 696 extern uint32_t qos_max_parallelism(int qos, uint64_t options); 697 #endif /* KERNEL_PRIVATE */ 698 699 #if XNU_KERNEL_PRIVATE 700 extern void thread_yield_with_continuation( 701 thread_continue_t continuation, 702 void *parameter) __dead2; 703 #endif 704 705 /* Context switch */ 706 extern wait_result_t thread_block( 707 thread_continue_t continuation); 708 709 extern wait_result_t thread_block_parameter( 710 thread_continue_t continuation, 711 void *parameter); 712 713 /* Declare thread will wait on a particular event */ 714 extern wait_result_t assert_wait( 715 event_t event, 716 wait_interrupt_t interruptible); 717 718 /* Assert that the thread intends to wait with a timeout */ 719 extern wait_result_t assert_wait_timeout( 720 event_t event, 721 wait_interrupt_t interruptible, 722 uint32_t interval, 723 uint32_t scale_factor); 724 725 /* Assert that the thread intends to wait with an urgency, timeout and leeway */ 726 extern wait_result_t assert_wait_timeout_with_leeway( 727 event_t event, 728 wait_interrupt_t interruptible, 729 wait_timeout_urgency_t urgency, 730 uint32_t interval, 731 uint32_t leeway, 732 uint32_t scale_factor); 733 734 extern wait_result_t assert_wait_deadline( 735 event_t event, 736 wait_interrupt_t interruptible, 737 uint64_t deadline); 738 739 /* Assert that the thread intends to wait with an urgency, deadline, and leeway */ 740 extern wait_result_t assert_wait_deadline_with_leeway( 741 event_t event, 742 wait_interrupt_t interruptible, 743 wait_timeout_urgency_t urgency, 744 uint64_t deadline, 745 uint64_t leeway); 746 747 748 /* Wake up thread (or threads) waiting on a particular event */ 749 extern kern_return_t thread_wakeup_prim( 750 event_t event, 751 boolean_t one_thread, 752 wait_result_t result); 753 754 #define thread_wakeup(x) \ 755 thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) 756 #define thread_wakeup_with_result(x, z) \ 757 thread_wakeup_prim((x), FALSE, (z)) 758 #define thread_wakeup_one(x) \ 759 thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) 760 761 /* Wakeup the specified thread if it is waiting on this event */ 762 extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread); 763 764 extern boolean_t preemption_enabled(void); 765 766 #ifdef MACH_KERNEL_PRIVATE 767 768 /* 769 * Scheduler algorithm indirection. If only one algorithm is 770 * enabled at compile-time, a direction function call is used. 771 * If more than one is enabled, calls are dispatched through 772 * a function pointer table. 773 */ 774 775 #if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) && !defined(CONFIG_SCHED_CLUTCH) && !defined(CONFIG_SCHED_EDGE) 776 #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX 777 #endif 778 779 #if __AMP__ 780 781 #if CONFIG_SCHED_EDGE 782 extern const struct sched_dispatch_table sched_edge_dispatch; 783 #define SCHED(f) (sched_edge_dispatch.f) 784 #else /* CONFIG_SCHED_EDGE */ 785 extern const struct sched_dispatch_table sched_amp_dispatch; 786 #define SCHED(f) (sched_amp_dispatch.f) 787 #endif /* CONFIG_SCHED_EDGE */ 788 789 #else /* __AMP__ */ 790 791 #if CONFIG_SCHED_CLUTCH 792 extern const struct sched_dispatch_table sched_clutch_dispatch; 793 #define SCHED(f) (sched_clutch_dispatch.f) 794 #else /* CONFIG_SCHED_CLUTCH */ 795 extern const struct sched_dispatch_table sched_dualq_dispatch; 796 #define SCHED(f) (sched_dualq_dispatch.f) 797 #endif /* CONFIG_SCHED_CLUTCH */ 798 799 #endif /* __AMP__ */ 800 801 struct sched_dispatch_table { 802 const char *sched_name; 803 void (*init)(void); /* Init global state */ 804 void (*timebase_init)(void); /* Timebase-dependent initialization */ 805 void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ 806 void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ 807 808 void (*maintenance_continuation)(void); /* Function called regularly */ 809 810 /* 811 * Choose a thread of greater or equal priority from the per-processor 812 * runqueue for timeshare/fixed threads 813 */ 814 thread_t (*choose_thread)( 815 processor_t processor, 816 int priority, 817 ast_t reason); 818 819 /* True if scheduler supports stealing threads for this pset */ 820 bool (*steal_thread_enabled)(processor_set_t pset); 821 822 /* 823 * Steal a thread from another processor in the pset so that it can run 824 * immediately 825 */ 826 thread_t (*steal_thread)( 827 processor_set_t pset); 828 829 /* 830 * Compute priority for a timeshare thread based on base priority. 831 */ 832 int (*compute_timeshare_priority)(thread_t thread); 833 834 /* 835 * Pick the best node for a thread to run on. 836 */ 837 pset_node_t (*choose_node)( 838 thread_t thread); 839 840 /* 841 * Pick the best processor for a thread (any kind of thread) to run on. 842 */ 843 processor_t (*choose_processor)( 844 processor_set_t pset, 845 processor_t processor, 846 thread_t thread); 847 /* 848 * Enqueue a timeshare or fixed priority thread onto the per-processor 849 * runqueue 850 */ 851 boolean_t (*processor_enqueue)( 852 processor_t processor, 853 thread_t thread, 854 sched_options_t options); 855 856 /* Migrate threads away in preparation for processor shutdown */ 857 void (*processor_queue_shutdown)( 858 processor_t processor); 859 860 /* Remove the specific thread from the per-processor runqueue */ 861 boolean_t (*processor_queue_remove)( 862 processor_t processor, 863 thread_t thread); 864 865 /* 866 * Does the per-processor runqueue have any timeshare or fixed priority 867 * threads on it? Called without pset lock held, so should 868 * not assume immutability while executing. 869 */ 870 boolean_t (*processor_queue_empty)(processor_t processor); 871 872 /* 873 * Would this priority trigger an urgent preemption if it's sitting 874 * on the per-processor runqueue? 875 */ 876 boolean_t (*priority_is_urgent)(int priority); 877 878 /* 879 * Does the per-processor runqueue contain runnable threads that 880 * should cause the currently-running thread to be preempted? 881 */ 882 ast_t (*processor_csw_check)(processor_t processor); 883 884 /* 885 * Does the per-processor runqueue contain a runnable thread 886 * of > or >= priority, as a preflight for choose_thread() or other 887 * thread selection 888 */ 889 boolean_t (*processor_queue_has_priority)(processor_t processor, 890 int priority, 891 boolean_t gte); 892 893 /* Quantum size for the specified non-realtime thread. */ 894 uint32_t (*initial_quantum_size)(thread_t thread); 895 896 /* Scheduler mode for a new thread */ 897 sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); 898 899 /* 900 * Is it safe to call update_priority, which may change a thread's 901 * runqueue or other state. This can be used to throttle changes 902 * to dynamic priority. 903 */ 904 boolean_t (*can_update_priority)(thread_t thread); 905 906 /* 907 * Update both scheduled priority and other persistent state. 908 * Side effects may including migration to another processor's runqueue. 909 */ 910 void (*update_priority)(thread_t thread); 911 912 /* Lower overhead update to scheduled priority and state. */ 913 void (*lightweight_update_priority)(thread_t thread); 914 915 /* Callback for non-realtime threads when the quantum timer fires */ 916 void (*quantum_expire)(thread_t thread); 917 918 /* 919 * Runnable threads on per-processor runqueue. Should only 920 * be used for relative comparisons of load between processors. 921 */ 922 int (*processor_runq_count)(processor_t processor); 923 924 /* Aggregate runcount statistics for per-processor runqueue */ 925 uint64_t (*processor_runq_stats_count_sum)(processor_t processor); 926 927 boolean_t (*processor_bound_count)(processor_t processor); 928 929 void (*thread_update_scan)(sched_update_scan_context_t scan_context); 930 931 /* Supports more than one pset */ 932 boolean_t multiple_psets_enabled; 933 /* Supports scheduler groups */ 934 boolean_t sched_groups_enabled; 935 936 /* Supports avoid-processor */ 937 boolean_t avoid_processor_enabled; 938 939 /* Returns true if this processor should avoid running this thread. */ 940 bool (*thread_avoid_processor)(processor_t processor, thread_t thread); 941 942 /* 943 * Invoked when a processor is about to choose the idle thread 944 * Used to send IPIs to a processor which would be preferred to be idle instead. 945 * Called with pset lock held, returns pset lock unlocked. 946 */ 947 void (*processor_balance)(processor_t processor, processor_set_t pset); 948 rt_queue_t (*rt_runq)(processor_set_t pset); 949 void (*rt_init)(processor_set_t pset); 950 void (*rt_queue_shutdown)(processor_t processor); 951 void (*rt_runq_scan)(sched_update_scan_context_t scan_context); 952 int64_t (*rt_runq_count_sum)(void); 953 thread_t (*rt_steal_thread)(processor_set_t pset, uint64_t earliest_deadline); 954 955 uint32_t (*qos_max_parallelism)(int qos, uint64_t options); 956 void (*check_spill)(processor_set_t pset, thread_t thread); 957 sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); 958 bool (*thread_should_yield)(processor_t processor, thread_t thread); 959 960 /* Routine to update run counts */ 961 uint32_t (*run_count_incr)(thread_t thread); 962 uint32_t (*run_count_decr)(thread_t thread); 963 964 /* Routine to update scheduling bucket for a thread */ 965 void (*update_thread_bucket)(thread_t thread); 966 967 /* Routine to inform the scheduler when a new pset becomes schedulable */ 968 void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock); 969 #if CONFIG_THREAD_GROUPS 970 /* Routine to inform the scheduler when CLPC changes a thread group recommendation */ 971 void (*thread_group_recommendation_change)(struct thread_group *tg, cluster_type_t new_recommendation); 972 #endif 973 /* Routine to inform the scheduler when all CPUs have finished initializing */ 974 void (*cpu_init_completed)(void); 975 /* Routine to check if a thread is eligible to execute on a specific pset */ 976 bool (*thread_eligible_for_pset)(thread_t thread, processor_set_t pset); 977 }; 978 979 #if defined(CONFIG_SCHED_TRADITIONAL) 980 extern const struct sched_dispatch_table sched_traditional_dispatch; 981 extern const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch; 982 #endif 983 984 #if defined(CONFIG_SCHED_MULTIQ) 985 extern const struct sched_dispatch_table sched_multiq_dispatch; 986 extern const struct sched_dispatch_table sched_dualq_dispatch; 987 #if __AMP__ 988 extern const struct sched_dispatch_table sched_amp_dispatch; 989 #endif 990 #endif 991 992 #if defined(CONFIG_SCHED_PROTO) 993 extern const struct sched_dispatch_table sched_proto_dispatch; 994 #endif 995 996 #if defined(CONFIG_SCHED_GRRR) 997 extern const struct sched_dispatch_table sched_grrr_dispatch; 998 #endif 999 1000 #if defined(CONFIG_SCHED_CLUTCH) 1001 extern const struct sched_dispatch_table sched_clutch_dispatch; 1002 #endif 1003 1004 #if defined(CONFIG_SCHED_EDGE) 1005 extern const struct sched_dispatch_table sched_edge_dispatch; 1006 #endif 1007 1008 extern void sched_set_max_unsafe_rt_quanta(int max); 1009 extern void sched_set_max_unsafe_fixed_quanta(int max); 1010 1011 #endif /* MACH_KERNEL_PRIVATE */ 1012 1013 __END_DECLS 1014 1015 #endif /* _KERN_SCHED_PRIM_H_ */ 1016