1 /* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 /* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or [email protected] 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 /* 57 */ 58 /* 59 * File: sched_prim.h 60 * Author: David Golub 61 * 62 * Scheduling primitive definitions file 63 * 64 */ 65 66 #ifndef _KERN_SCHED_PRIM_H_ 67 #define _KERN_SCHED_PRIM_H_ 68 69 #include <sys/cdefs.h> 70 #include <mach/boolean.h> 71 #include <mach/machine/vm_types.h> 72 #include <mach/kern_return.h> 73 #include <kern/clock.h> 74 #include <kern/kern_types.h> 75 #include <kern/percpu.h> 76 #include <kern/thread.h> 77 #include <kern/block_hint.h> 78 79 extern int thread_get_current_cpuid(void); 80 81 #ifdef MACH_KERNEL_PRIVATE 82 83 #include <kern/sched_urgency.h> 84 #include <kern/thread_group.h> 85 #include <kern/waitq.h> 86 87 /* Initialization */ 88 extern void sched_init(void); 89 90 extern void sched_startup(void); 91 92 extern void sched_timebase_init(void); 93 94 extern void pset_rt_init(processor_set_t pset); 95 96 extern void sched_rtlocal_init(processor_set_t pset); 97 98 extern rt_queue_t sched_rtlocal_runq(processor_set_t pset); 99 100 extern void sched_rtlocal_queue_shutdown(processor_t processor); 101 102 extern int64_t sched_rtlocal_runq_count_sum(void); 103 104 extern thread_t sched_rtlocal_steal_thread(processor_set_t stealing_pset, uint64_t earliest_deadline); 105 106 extern thread_t sched_rt_choose_thread(processor_set_t pset); 107 108 extern void sched_check_spill(processor_set_t pset, thread_t thread); 109 110 extern bool sched_thread_should_yield(processor_t processor, thread_t thread); 111 112 extern bool sched_steal_thread_DISABLED(processor_set_t pset); 113 extern bool sched_steal_thread_enabled(processor_set_t pset); 114 115 /* Force a preemption point for a thread and wait for it to stop running */ 116 extern boolean_t thread_stop( 117 thread_t thread, 118 boolean_t until_not_runnable); 119 120 /* Release a previous stop request */ 121 extern void thread_unstop( 122 thread_t thread); 123 124 /* Wait for a thread to stop running */ 125 extern void thread_wait( 126 thread_t thread, 127 boolean_t until_not_runnable); 128 129 /* Unblock thread on wake up */ 130 extern boolean_t thread_unblock( 131 thread_t thread, 132 wait_result_t wresult); 133 134 /* Unblock and dispatch thread */ 135 extern kern_return_t thread_go( 136 thread_t thread, 137 wait_result_t wresult, 138 waitq_options_t option); 139 140 /* Check if direct handoff is allowed */ 141 extern boolean_t 142 thread_allowed_for_handoff( 143 thread_t thread); 144 145 /* Handle threads at context switch */ 146 extern void thread_dispatch( 147 thread_t old_thread, 148 thread_t new_thread); 149 150 /* Switch directly to a particular thread */ 151 extern int thread_run( 152 thread_t self, 153 thread_continue_t continuation, 154 void *parameter, 155 thread_t new_thread); 156 157 /* Resume thread with new stack */ 158 extern __dead2 void thread_continue(thread_t old_thread); 159 160 /* Invoke continuation */ 161 extern __dead2 void call_continuation( 162 thread_continue_t continuation, 163 void *parameter, 164 wait_result_t wresult, 165 boolean_t enable_interrupts); 166 167 /* 168 * Flags that can be passed to set_sched_pri 169 * to skip side effects 170 */ 171 __options_decl(set_sched_pri_options_t, uint32_t, { 172 SETPRI_DEFAULT = 0x0, 173 SETPRI_LAZY = 0x1, /* Avoid setting AST flags or sending IPIs */ 174 }); 175 176 /* Set the current scheduled priority */ 177 extern void set_sched_pri( 178 thread_t thread, 179 int16_t priority, 180 set_sched_pri_options_t options); 181 182 /* Set base priority of the specified thread */ 183 extern void sched_set_thread_base_priority( 184 thread_t thread, 185 int priority); 186 187 /* Set absolute base priority of the specified thread */ 188 extern void sched_set_kernel_thread_priority( 189 thread_t thread, 190 int priority); 191 192 193 /* Set the thread's true scheduling mode */ 194 extern void sched_set_thread_mode(thread_t thread, 195 sched_mode_t mode); 196 /* Demote the true scheduler mode */ 197 extern void sched_thread_mode_demote(thread_t thread, 198 uint32_t reason); 199 /* Un-demote the true scheduler mode */ 200 extern void sched_thread_mode_undemote(thread_t thread, 201 uint32_t reason); 202 203 extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); 204 extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); 205 206 /* Re-evaluate base priority of thread (thread locked) */ 207 void thread_recompute_priority(thread_t thread); 208 209 /* Re-evaluate scheduled priority of thread (thread locked) */ 210 extern void thread_recompute_sched_pri( 211 thread_t thread, 212 set_sched_pri_options_t options); 213 214 /* Periodic scheduler activity */ 215 extern void sched_init_thread(void); 216 217 /* Perform sched_tick housekeeping activities */ 218 extern boolean_t can_update_priority( 219 thread_t thread); 220 221 extern void update_priority( 222 thread_t thread); 223 224 extern void lightweight_update_priority( 225 thread_t thread); 226 227 extern void sched_default_quantum_expire(thread_t thread); 228 229 /* Idle processor thread continuation */ 230 extern void idle_thread( 231 void* parameter, 232 wait_result_t result); 233 234 extern kern_return_t idle_thread_create( 235 processor_t processor); 236 237 /* Continuation return from syscall */ 238 extern void thread_syscall_return( 239 kern_return_t ret); 240 241 /* Context switch */ 242 extern wait_result_t thread_block_reason( 243 thread_continue_t continuation, 244 void *parameter, 245 ast_t reason); 246 247 __options_decl(sched_options_t, uint32_t, { 248 SCHED_NONE = 0x0, 249 SCHED_TAILQ = 0x1, 250 SCHED_HEADQ = 0x2, 251 SCHED_PREEMPT = 0x4, 252 SCHED_REBALANCE = 0x8, 253 }); 254 255 /* Reschedule thread for execution */ 256 extern void thread_setrun( 257 thread_t thread, 258 sched_options_t options); 259 260 extern processor_set_t task_choose_pset( 261 task_t task); 262 263 /* Bind the current thread to a particular processor */ 264 extern processor_t thread_bind( 265 processor_t processor); 266 267 extern bool pset_has_stealable_threads( 268 processor_set_t pset); 269 270 extern bool pset_has_stealable_rt_threads( 271 processor_set_t pset); 272 273 extern processor_set_t choose_starting_pset( 274 pset_node_t node, 275 thread_t thread, 276 processor_t *processor_hint); 277 278 extern int pset_available_cpu_count( 279 processor_set_t pset); 280 281 extern pset_node_t sched_choose_node( 282 thread_t thread); 283 284 /* Choose the best processor to run a thread */ 285 extern processor_t choose_processor( 286 processor_set_t pset, 287 processor_t processor, 288 thread_t thread); 289 290 extern void sched_SMT_balance( 291 processor_t processor, 292 processor_set_t pset); 293 294 extern void thread_quantum_init( 295 thread_t thread); 296 297 extern void run_queue_init( 298 run_queue_t runq); 299 300 extern thread_t run_queue_dequeue( 301 run_queue_t runq, 302 sched_options_t options); 303 304 extern boolean_t run_queue_enqueue( 305 run_queue_t runq, 306 thread_t thread, 307 sched_options_t options); 308 309 extern void run_queue_remove( 310 run_queue_t runq, 311 thread_t thread); 312 313 extern thread_t run_queue_peek( 314 run_queue_t runq); 315 316 struct sched_update_scan_context { 317 uint64_t earliest_bg_make_runnable_time; 318 uint64_t earliest_normal_make_runnable_time; 319 uint64_t earliest_rt_make_runnable_time; 320 uint64_t sched_tick_last_abstime; 321 }; 322 typedef struct sched_update_scan_context *sched_update_scan_context_t; 323 324 extern void sched_rtlocal_runq_scan(sched_update_scan_context_t scan_context); 325 326 extern void sched_pset_made_schedulable( 327 processor_t processor, 328 processor_set_t pset, 329 boolean_t drop_lock); 330 331 extern void sched_cpu_init_completed(void); 332 333 /* 334 * Enum to define various events which need IPIs. The IPI policy 335 * engine decides what kind of IPI to use based on destination 336 * processor state, thread and one of the following scheduling events. 337 */ 338 typedef enum { 339 SCHED_IPI_EVENT_BOUND_THR = 0x1, 340 SCHED_IPI_EVENT_PREEMPT = 0x2, 341 SCHED_IPI_EVENT_SMT_REBAL = 0x3, 342 SCHED_IPI_EVENT_SPILL = 0x4, 343 SCHED_IPI_EVENT_REBALANCE = 0x5, 344 SCHED_IPI_EVENT_RT_PREEMPT = 0x6, 345 } sched_ipi_event_t; 346 347 348 /* Enum to define various IPI types used by the scheduler */ 349 typedef enum { 350 SCHED_IPI_NONE = 0x0, 351 SCHED_IPI_IMMEDIATE = 0x1, 352 SCHED_IPI_IDLE = 0x2, 353 SCHED_IPI_DEFERRED = 0x3, 354 } sched_ipi_type_t; 355 356 /* The IPI policy engine behaves in the following manner: 357 * - All scheduler events which need an IPI invoke sched_ipi_action() with 358 * the appropriate destination processor, thread and event. 359 * - sched_ipi_action() performs basic checks, invokes the scheduler specific 360 * ipi_policy routine and sets pending_AST bits based on the result. 361 * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform() 362 * routine which actually sends the appropriate IPI to the destination core. 363 */ 364 extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, sched_ipi_event_t event); 365 extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); 366 367 /* sched_ipi_policy() is the global default IPI policy for all schedulers */ 368 extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, 369 boolean_t dst_idle, sched_ipi_event_t event); 370 371 /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ 372 extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, 373 processor_t dst, thread_t thread, sched_ipi_event_t event); 374 375 #if defined(CONFIG_SCHED_TIMESHARE_CORE) 376 377 extern boolean_t thread_update_add_thread(thread_t thread); 378 extern void thread_update_process_threads(void); 379 extern boolean_t runq_scan(run_queue_t runq, sched_update_scan_context_t scan_context); 380 381 #if CONFIG_SCHED_CLUTCH 382 extern boolean_t sched_clutch_timeshare_scan(queue_t thread_queue, uint16_t count, sched_update_scan_context_t scan_context); 383 #endif /* CONFIG_SCHED_CLUTCH */ 384 385 extern void sched_timeshare_init(void); 386 extern void sched_timeshare_timebase_init(void); 387 extern void sched_timeshare_maintenance_continue(void); 388 389 extern boolean_t priority_is_urgent(int priority); 390 extern uint32_t sched_timeshare_initial_quantum_size(thread_t thread); 391 392 extern int sched_compute_timeshare_priority(thread_t thread); 393 394 #endif /* CONFIG_SCHED_TIMESHARE_CORE */ 395 396 /* Remove thread from its run queue */ 397 extern boolean_t thread_run_queue_remove(thread_t thread); 398 thread_t thread_run_queue_remove_for_handoff(thread_t thread); 399 400 /* Put a thread back in the run queue after being yanked */ 401 extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options); 402 403 extern void thread_timer_expire( 404 void *thread, 405 void *p1); 406 407 extern bool thread_is_eager_preempt(thread_t thread); 408 409 extern boolean_t sched_generic_direct_dispatch_to_idle_processors; 410 411 /* Set the maximum interrupt level for the thread */ 412 __private_extern__ wait_interrupt_t thread_interrupt_level( 413 wait_interrupt_t interruptible); 414 415 __private_extern__ wait_result_t thread_mark_wait_locked( 416 thread_t thread, 417 wait_interrupt_t interruptible); 418 419 /* Wake up locked thread directly, passing result */ 420 __private_extern__ kern_return_t clear_wait_internal( 421 thread_t thread, 422 wait_result_t result); 423 424 struct sched_statistics { 425 uint32_t csw_count; 426 uint32_t preempt_count; 427 uint32_t preempted_rt_count; 428 uint32_t preempted_by_rt_count; 429 uint32_t rt_sched_count; 430 uint32_t interrupt_count; 431 uint32_t ipi_count; 432 uint32_t timer_pop_count; 433 uint32_t idle_transitions; 434 uint32_t quantum_timer_expirations; 435 }; 436 PERCPU_DECL(struct sched_statistics, sched_stats); 437 extern bool sched_stats_active; 438 439 extern void sched_stats_handle_csw( 440 processor_t processor, 441 int reasons, 442 int selfpri, 443 int otherpri); 444 445 extern void sched_stats_handle_runq_change( 446 struct runq_stats *stats, 447 int old_count); 448 449 #define SCHED_STATS_INC(field) \ 450 MACRO_BEGIN \ 451 if (__improbable(sched_stats_active)) { \ 452 PERCPU_GET(sched_stats)->field++; \ 453 } \ 454 MACRO_END 455 456 #if DEBUG 457 458 #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ 459 MACRO_BEGIN \ 460 if (__improbable(sched_stats_active)) { \ 461 sched_stats_handle_csw((processor), \ 462 (reasons), (selfpri), (otherpri)); \ 463 } \ 464 MACRO_END 465 466 467 #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ 468 MACRO_BEGIN \ 469 if (__improbable(sched_stats_active)) { \ 470 sched_stats_handle_runq_change((stats), (old_count)); \ 471 } \ 472 MACRO_END 473 474 #else /* DEBUG */ 475 476 #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0) 477 #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0) 478 479 #endif /* DEBUG */ 480 481 extern uint32_t sched_debug_flags; 482 #define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 483 #define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 484 485 #define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) \ 486 MACRO_BEGIN \ 487 if (__improbable(sched_debug_flags & \ 488 SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ 489 KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ 490 } \ 491 MACRO_END 492 493 #define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) \ 494 MACRO_BEGIN \ 495 if (__improbable(sched_debug_flags & \ 496 SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ 497 KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ 498 } \ 499 MACRO_END 500 501 /* Tells if there are "active" RT threads in the system (provided by CPU PM) */ 502 extern void active_rt_threads( 503 boolean_t active); 504 505 /* Returns the perfcontrol attribute for the thread */ 506 extern perfcontrol_class_t thread_get_perfcontrol_class( 507 thread_t thread); 508 509 /* Generic routine for Non-AMP schedulers to calculate parallelism */ 510 extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); 511 512 extern void check_monotonic_time(uint64_t ctime); 513 514 #endif /* MACH_KERNEL_PRIVATE */ 515 516 __BEGIN_DECLS 517 518 #ifdef XNU_KERNEL_PRIVATE 519 520 extern void thread_bind_cluster_type(thread_t, char cluster_type, bool soft_bind); 521 522 __options_decl(thread_bind_option_t, uint64_t, { 523 /* Unbind a previously cluster bound thread */ 524 THREAD_UNBIND = 0x1, 525 /* 526 * Soft bind the thread to the cluster; soft binding means the thread will be 527 * moved to an available cluster if the bound cluster is de-recommended/offline. 528 */ 529 THREAD_BIND_SOFT = 0x2, 530 /* 531 * Bind thread to the cluster only if it is eligible to run on that cluster. If 532 * the thread is not eligible to run on the cluster, thread_bind_cluster_id() 533 * returns KERN_INVALID_POLICY. 534 */ 535 THREAD_BIND_ELIGIBLE_ONLY = 0x4, 536 }); 537 extern kern_return_t thread_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options); 538 539 extern int sched_get_rt_n_backup_processors(void); 540 extern void sched_set_rt_n_backup_processors(int n); 541 542 extern int sched_get_rt_constraint_ll(void); 543 extern void sched_set_rt_constraint_ll(int new_constraint_us); 544 545 extern int sched_get_rt_deadline_epsilon(void); 546 extern void sched_set_rt_deadline_epsilon(int new_epsilon_us); 547 548 /* Toggles a global override to turn off CPU Throttling */ 549 extern void sys_override_cpu_throttle(boolean_t enable_override); 550 551 /* 552 ****************** Only exported until BSD stops using ******************** 553 */ 554 555 extern void thread_vm_bind_group_add(void); 556 557 /* Wake up thread directly, passing result */ 558 extern kern_return_t clear_wait( 559 thread_t thread, 560 wait_result_t result); 561 562 /* Start thread running */ 563 extern void thread_bootstrap_return(void) __attribute__((noreturn)); 564 565 /* Return from exception (BSD-visible interface) */ 566 extern void thread_exception_return(void) __dead2; 567 568 #define SCHED_STRING_MAX_LENGTH (48) 569 /* String declaring the name of the current scheduler */ 570 extern char sched_string[SCHED_STRING_MAX_LENGTH]; 571 572 __options_decl(thread_handoff_option_t, uint32_t, { 573 THREAD_HANDOFF_NONE = 0, 574 THREAD_HANDOFF_SETRUN_NEEDED = 0x1, 575 }); 576 577 /* Remove thread from its run queue */ 578 thread_t thread_prepare_for_handoff(thread_t thread, thread_handoff_option_t option); 579 580 /* Attempt to context switch to a specific runnable thread */ 581 extern wait_result_t thread_handoff_deallocate(thread_t thread, thread_handoff_option_t option); 582 583 __attribute__((nonnull(2))) 584 extern void thread_handoff_parameter(thread_t thread, 585 thread_continue_t continuation, void *parameter, thread_handoff_option_t) __dead2; 586 587 extern struct waitq *assert_wait_queue(event_t event); 588 589 extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority); 590 591 extern thread_t thread_wakeup_identify(event_t event, int priority); 592 593 #endif /* XNU_KERNEL_PRIVATE */ 594 595 #ifdef KERNEL_PRIVATE 596 /* Set pending block hint for a particular object before we go into a wait state */ 597 extern void thread_set_pending_block_hint( 598 thread_t thread, 599 block_hint_t block_hint); 600 601 #define QOS_PARALLELISM_COUNT_LOGICAL 0x1 602 #define QOS_PARALLELISM_REALTIME 0x2 603 #define QOS_PARALLELISM_CLUSTER_SHARED_RESOURCE 0x4 604 605 extern uint32_t qos_max_parallelism(int qos, uint64_t options); 606 #endif /* KERNEL_PRIVATE */ 607 608 #if XNU_KERNEL_PRIVATE 609 extern void thread_yield_with_continuation( 610 thread_continue_t continuation, 611 void *parameter) __dead2; 612 #endif 613 614 /* Context switch */ 615 extern wait_result_t thread_block( 616 thread_continue_t continuation); 617 618 extern wait_result_t thread_block_parameter( 619 thread_continue_t continuation, 620 void *parameter); 621 622 /* Declare thread will wait on a particular event */ 623 extern wait_result_t assert_wait( 624 event_t event, 625 wait_interrupt_t interruptible); 626 627 /* Assert that the thread intends to wait with a timeout */ 628 extern wait_result_t assert_wait_timeout( 629 event_t event, 630 wait_interrupt_t interruptible, 631 uint32_t interval, 632 uint32_t scale_factor); 633 634 /* Assert that the thread intends to wait with an urgency, timeout and leeway */ 635 extern wait_result_t assert_wait_timeout_with_leeway( 636 event_t event, 637 wait_interrupt_t interruptible, 638 wait_timeout_urgency_t urgency, 639 uint32_t interval, 640 uint32_t leeway, 641 uint32_t scale_factor); 642 643 extern wait_result_t assert_wait_deadline( 644 event_t event, 645 wait_interrupt_t interruptible, 646 uint64_t deadline); 647 648 /* Assert that the thread intends to wait with an urgency, deadline, and leeway */ 649 extern wait_result_t assert_wait_deadline_with_leeway( 650 event_t event, 651 wait_interrupt_t interruptible, 652 wait_timeout_urgency_t urgency, 653 uint64_t deadline, 654 uint64_t leeway); 655 656 /* Wake up thread (or threads) waiting on a particular event */ 657 extern kern_return_t thread_wakeup_prim( 658 event_t event, 659 boolean_t one_thread, 660 wait_result_t result); 661 662 #define thread_wakeup(x) \ 663 thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) 664 #define thread_wakeup_with_result(x, z) \ 665 thread_wakeup_prim((x), FALSE, (z)) 666 #define thread_wakeup_one(x) \ 667 thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) 668 669 /* Wakeup the specified thread if it is waiting on this event */ 670 extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread); 671 672 extern boolean_t preemption_enabled(void); 673 674 #ifdef MACH_KERNEL_PRIVATE 675 676 /* 677 * Scheduler algorithm indirection. If only one algorithm is 678 * enabled at compile-time, a direction function call is used. 679 * If more than one is enabled, calls are dispatched through 680 * a function pointer table. 681 */ 682 683 #if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) && !defined(CONFIG_SCHED_CLUTCH) && !defined(CONFIG_SCHED_EDGE) 684 #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX 685 #endif 686 687 #if __AMP__ 688 689 #if CONFIG_SCHED_EDGE 690 extern const struct sched_dispatch_table sched_edge_dispatch; 691 #define SCHED(f) (sched_edge_dispatch.f) 692 #else /* CONFIG_SCHED_EDGE */ 693 extern const struct sched_dispatch_table sched_amp_dispatch; 694 #define SCHED(f) (sched_amp_dispatch.f) 695 #endif /* CONFIG_SCHED_EDGE */ 696 697 #else /* __AMP__ */ 698 699 #if CONFIG_SCHED_CLUTCH 700 extern const struct sched_dispatch_table sched_clutch_dispatch; 701 #define SCHED(f) (sched_clutch_dispatch.f) 702 #else /* CONFIG_SCHED_CLUTCH */ 703 extern const struct sched_dispatch_table sched_dualq_dispatch; 704 #define SCHED(f) (sched_dualq_dispatch.f) 705 #endif /* CONFIG_SCHED_CLUTCH */ 706 707 #endif /* __AMP__ */ 708 709 struct sched_dispatch_table { 710 const char *sched_name; 711 void (*init)(void); /* Init global state */ 712 void (*timebase_init)(void); /* Timebase-dependent initialization */ 713 void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ 714 void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ 715 716 void (*maintenance_continuation)(void); /* Function called regularly */ 717 718 /* 719 * Choose a thread of greater or equal priority from the per-processor 720 * runqueue for timeshare/fixed threads 721 */ 722 thread_t (*choose_thread)( 723 processor_t processor, 724 int priority, 725 ast_t reason); 726 727 /* True if scheduler supports stealing threads for this pset */ 728 bool (*steal_thread_enabled)(processor_set_t pset); 729 730 /* 731 * Steal a thread from another processor in the pset so that it can run 732 * immediately 733 */ 734 thread_t (*steal_thread)( 735 processor_set_t pset); 736 737 /* 738 * Compute priority for a timeshare thread based on base priority. 739 */ 740 int (*compute_timeshare_priority)(thread_t thread); 741 742 /* 743 * Pick the best node for a thread to run on. 744 */ 745 pset_node_t (*choose_node)( 746 thread_t thread); 747 748 /* 749 * Pick the best processor for a thread (any kind of thread) to run on. 750 */ 751 processor_t (*choose_processor)( 752 processor_set_t pset, 753 processor_t processor, 754 thread_t thread); 755 /* 756 * Enqueue a timeshare or fixed priority thread onto the per-processor 757 * runqueue 758 */ 759 boolean_t (*processor_enqueue)( 760 processor_t processor, 761 thread_t thread, 762 sched_options_t options); 763 764 /* Migrate threads away in preparation for processor shutdown */ 765 void (*processor_queue_shutdown)( 766 processor_t processor); 767 768 /* Remove the specific thread from the per-processor runqueue */ 769 boolean_t (*processor_queue_remove)( 770 processor_t processor, 771 thread_t thread); 772 773 /* 774 * Does the per-processor runqueue have any timeshare or fixed priority 775 * threads on it? Called without pset lock held, so should 776 * not assume immutability while executing. 777 */ 778 boolean_t (*processor_queue_empty)(processor_t processor); 779 780 /* 781 * Would this priority trigger an urgent preemption if it's sitting 782 * on the per-processor runqueue? 783 */ 784 boolean_t (*priority_is_urgent)(int priority); 785 786 /* 787 * Does the per-processor runqueue contain runnable threads that 788 * should cause the currently-running thread to be preempted? 789 */ 790 ast_t (*processor_csw_check)(processor_t processor); 791 792 /* 793 * Does the per-processor runqueue contain a runnable thread 794 * of > or >= priority, as a preflight for choose_thread() or other 795 * thread selection 796 */ 797 boolean_t (*processor_queue_has_priority)(processor_t processor, 798 int priority, 799 boolean_t gte); 800 801 /* Quantum size for the specified non-realtime thread. */ 802 uint32_t (*initial_quantum_size)(thread_t thread); 803 804 /* Scheduler mode for a new thread */ 805 sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); 806 807 /* 808 * Is it safe to call update_priority, which may change a thread's 809 * runqueue or other state. This can be used to throttle changes 810 * to dynamic priority. 811 */ 812 boolean_t (*can_update_priority)(thread_t thread); 813 814 /* 815 * Update both scheduled priority and other persistent state. 816 * Side effects may including migration to another processor's runqueue. 817 */ 818 void (*update_priority)(thread_t thread); 819 820 /* Lower overhead update to scheduled priority and state. */ 821 void (*lightweight_update_priority)(thread_t thread); 822 823 /* Callback for non-realtime threads when the quantum timer fires */ 824 void (*quantum_expire)(thread_t thread); 825 826 /* 827 * Runnable threads on per-processor runqueue. Should only 828 * be used for relative comparisons of load between processors. 829 */ 830 int (*processor_runq_count)(processor_t processor); 831 832 /* Aggregate runcount statistics for per-processor runqueue */ 833 uint64_t (*processor_runq_stats_count_sum)(processor_t processor); 834 835 boolean_t (*processor_bound_count)(processor_t processor); 836 837 void (*thread_update_scan)(sched_update_scan_context_t scan_context); 838 839 /* Supports more than one pset */ 840 boolean_t multiple_psets_enabled; 841 /* Supports scheduler groups */ 842 boolean_t sched_groups_enabled; 843 844 /* Supports avoid-processor */ 845 boolean_t avoid_processor_enabled; 846 847 /* Returns true if this processor should avoid running this thread. */ 848 bool (*thread_avoid_processor)(processor_t processor, thread_t thread); 849 850 /* 851 * Invoked when a processor is about to choose the idle thread 852 * Used to send IPIs to a processor which would be preferred to be idle instead. 853 * Called with pset lock held, returns pset lock unlocked. 854 */ 855 void (*processor_balance)(processor_t processor, processor_set_t pset); 856 rt_queue_t (*rt_runq)(processor_set_t pset); 857 void (*rt_init)(processor_set_t pset); 858 void (*rt_queue_shutdown)(processor_t processor); 859 void (*rt_runq_scan)(sched_update_scan_context_t scan_context); 860 int64_t (*rt_runq_count_sum)(void); 861 thread_t (*rt_steal_thread)(processor_set_t pset, uint64_t earliest_deadline); 862 863 uint32_t (*qos_max_parallelism)(int qos, uint64_t options); 864 void (*check_spill)(processor_set_t pset, thread_t thread); 865 sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); 866 bool (*thread_should_yield)(processor_t processor, thread_t thread); 867 868 /* Routine to update run counts */ 869 uint32_t (*run_count_incr)(thread_t thread); 870 uint32_t (*run_count_decr)(thread_t thread); 871 872 /* Routine to update scheduling bucket for a thread */ 873 void (*update_thread_bucket)(thread_t thread); 874 875 /* Routine to inform the scheduler when a new pset becomes schedulable */ 876 void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock); 877 #if CONFIG_THREAD_GROUPS 878 /* Routine to inform the scheduler when CLPC changes a thread group recommendation */ 879 void (*thread_group_recommendation_change)(struct thread_group *tg, cluster_type_t new_recommendation); 880 #endif 881 /* Routine to inform the scheduler when all CPUs have finished initializing */ 882 void (*cpu_init_completed)(void); 883 /* Routine to check if a thread is eligible to execute on a specific pset */ 884 bool (*thread_eligible_for_pset)(thread_t thread, processor_set_t pset); 885 }; 886 887 #if defined(CONFIG_SCHED_TRADITIONAL) 888 extern const struct sched_dispatch_table sched_traditional_dispatch; 889 extern const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch; 890 #endif 891 892 #if defined(CONFIG_SCHED_MULTIQ) 893 extern const struct sched_dispatch_table sched_multiq_dispatch; 894 extern const struct sched_dispatch_table sched_dualq_dispatch; 895 #if __AMP__ 896 extern const struct sched_dispatch_table sched_amp_dispatch; 897 #endif 898 #endif 899 900 #if defined(CONFIG_SCHED_PROTO) 901 extern const struct sched_dispatch_table sched_proto_dispatch; 902 #endif 903 904 #if defined(CONFIG_SCHED_GRRR) 905 extern const struct sched_dispatch_table sched_grrr_dispatch; 906 #endif 907 908 #if defined(CONFIG_SCHED_CLUTCH) 909 extern const struct sched_dispatch_table sched_clutch_dispatch; 910 #endif 911 912 #if defined(CONFIG_SCHED_EDGE) 913 extern const struct sched_dispatch_table sched_edge_dispatch; 914 #endif 915 916 917 #endif /* MACH_KERNEL_PRIVATE */ 918 919 __END_DECLS 920 921 #endif /* _KERN_SCHED_PRIM_H_ */ 922