/* * Copyright (c) 2000-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ /* */ /* * File: sched_prim.h * Author: David Golub * * Scheduling primitive definitions file * */ #ifndef _KERN_SCHED_PRIM_H_ #define _KERN_SCHED_PRIM_H_ #include #include #include #include #include #include #include #include #include extern int thread_get_current_cpuid(void); #ifdef MACH_KERNEL_PRIVATE #include #include #include /* Initialization */ extern void sched_init(void); extern void sched_startup(void); extern void sched_timebase_init(void); extern void pset_rt_init(processor_set_t pset); extern void sched_rtlocal_init(processor_set_t pset); extern rt_queue_t sched_rtlocal_runq(processor_set_t pset); extern void sched_rtlocal_queue_shutdown(processor_t processor); extern int64_t sched_rtlocal_runq_count_sum(void); extern thread_t sched_rtlocal_steal_thread(processor_set_t stealing_pset, uint64_t earliest_deadline); extern thread_t sched_rt_choose_thread(processor_set_t pset); extern void sched_check_spill(processor_set_t pset, thread_t thread); extern bool sched_thread_should_yield(processor_t processor, thread_t thread); extern bool sched_steal_thread_DISABLED(processor_set_t pset); extern bool sched_steal_thread_enabled(processor_set_t pset); /* Force a preemption point for a thread and wait for it to stop running */ extern boolean_t thread_stop( thread_t thread, boolean_t until_not_runnable); /* Release a previous stop request */ extern void thread_unstop( thread_t thread); /* Wait for a thread to stop running */ extern void thread_wait( thread_t thread, boolean_t until_not_runnable); /* Unblock thread on wake up */ extern boolean_t thread_unblock( thread_t thread, wait_result_t wresult); /* Unblock and dispatch thread */ extern kern_return_t thread_go( thread_t thread, wait_result_t wresult, bool try_handoff); /* Check if direct handoff is allowed */ extern boolean_t thread_allowed_for_handoff( thread_t thread); /* Handle threads at context switch */ extern void thread_dispatch( thread_t old_thread, thread_t new_thread); /* Switch directly to a particular thread */ extern int thread_run( thread_t self, thread_continue_t continuation, void *parameter, thread_t new_thread); /* Resume thread with new stack */ extern __dead2 void thread_continue(thread_t old_thread); /* Invoke continuation */ extern __dead2 void call_continuation( thread_continue_t continuation, void *parameter, wait_result_t wresult, boolean_t enable_interrupts); /* * Flags that can be passed to set_sched_pri * to skip side effects */ __options_decl(set_sched_pri_options_t, uint32_t, { SETPRI_DEFAULT = 0x0, SETPRI_LAZY = 0x1, /* Avoid setting AST flags or sending IPIs */ }); /* Set the current scheduled priority */ extern void set_sched_pri( thread_t thread, int16_t priority, set_sched_pri_options_t options); /* Set base priority of the specified thread */ extern void sched_set_thread_base_priority( thread_t thread, int priority); /* Set absolute base priority of the specified thread */ extern void sched_set_kernel_thread_priority( thread_t thread, int priority); /* Set the thread's true scheduling mode */ extern void sched_set_thread_mode(thread_t thread, sched_mode_t mode); /* * Set the thread's scheduling mode taking into account that the thread may have * been demoted. * */ extern void sched_set_thread_mode_user(thread_t thread, sched_mode_t mode); /* Demote the true scheduler mode */ extern void sched_thread_mode_demote(thread_t thread, uint32_t reason); /* Un-demote the true scheduler mode */ extern void sched_thread_mode_undemote(thread_t thread, uint32_t reason); extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj); /* Re-evaluate base priority of thread (thread locked) */ void thread_recompute_priority(thread_t thread); /* Re-evaluate scheduled priority of thread (thread locked) */ extern void thread_recompute_sched_pri( thread_t thread, set_sched_pri_options_t options); /* Periodic scheduler activity */ extern void sched_init_thread(void); /* Perform sched_tick housekeeping activities */ extern boolean_t can_update_priority( thread_t thread); extern void update_priority( thread_t thread); extern void lightweight_update_priority( thread_t thread); extern void sched_default_quantum_expire(thread_t thread); /* Idle processor thread continuation */ extern void idle_thread( void* parameter, wait_result_t result); extern kern_return_t idle_thread_create( processor_t processor); /* Continuation return from syscall */ extern void thread_syscall_return( kern_return_t ret); /* Context switch */ extern wait_result_t thread_block_reason( thread_continue_t continuation, void *parameter, ast_t reason); __options_decl(sched_options_t, uint32_t, { SCHED_NONE = 0x0, SCHED_TAILQ = 0x1, SCHED_HEADQ = 0x2, SCHED_PREEMPT = 0x4, SCHED_REBALANCE = 0x8, }); /* Reschedule thread for execution */ extern void thread_setrun( thread_t thread, sched_options_t options); extern processor_set_t task_choose_pset( task_t task); /* Bind the current thread to a particular processor */ extern processor_t thread_bind( processor_t processor); extern bool pset_has_stealable_threads( processor_set_t pset); extern bool pset_has_stealable_rt_threads( processor_set_t pset); extern processor_set_t choose_starting_pset( pset_node_t node, thread_t thread, processor_t *processor_hint); extern int pset_available_cpu_count( processor_set_t pset); extern bool pset_is_recommended( processor_set_t pset); extern pset_node_t sched_choose_node( thread_t thread); /* Choose the best processor to run a thread */ extern processor_t choose_processor( processor_set_t pset, processor_t processor, thread_t thread); extern void sched_SMT_balance( processor_t processor, processor_set_t pset); extern void thread_quantum_init( thread_t thread); extern void run_queue_init( run_queue_t runq); extern thread_t run_queue_dequeue( run_queue_t runq, sched_options_t options); extern boolean_t run_queue_enqueue( run_queue_t runq, thread_t thread, sched_options_t options); extern void run_queue_remove( run_queue_t runq, thread_t thread); extern thread_t run_queue_peek( run_queue_t runq); struct sched_update_scan_context { uint64_t earliest_bg_make_runnable_time; uint64_t earliest_normal_make_runnable_time; uint64_t earliest_rt_make_runnable_time; uint64_t sched_tick_last_abstime; }; typedef struct sched_update_scan_context *sched_update_scan_context_t; extern void sched_rtlocal_runq_scan(sched_update_scan_context_t scan_context); extern void sched_pset_made_schedulable( processor_t processor, processor_set_t pset, boolean_t drop_lock); extern void sched_cpu_init_completed(void); /* * Enum to define various events which need IPIs. The IPI policy * engine decides what kind of IPI to use based on destination * processor state, thread and one of the following scheduling events. */ typedef enum { SCHED_IPI_EVENT_BOUND_THR = 0x1, SCHED_IPI_EVENT_PREEMPT = 0x2, SCHED_IPI_EVENT_SMT_REBAL = 0x3, SCHED_IPI_EVENT_SPILL = 0x4, SCHED_IPI_EVENT_REBALANCE = 0x5, SCHED_IPI_EVENT_RT_PREEMPT = 0x6, } sched_ipi_event_t; /* Enum to define various IPI types used by the scheduler */ typedef enum { SCHED_IPI_NONE = 0x0, SCHED_IPI_IMMEDIATE = 0x1, SCHED_IPI_IDLE = 0x2, SCHED_IPI_DEFERRED = 0x3, } sched_ipi_type_t; /* The IPI policy engine behaves in the following manner: * - All scheduler events which need an IPI invoke sched_ipi_action() with * the appropriate destination processor, thread and event. * - sched_ipi_action() performs basic checks, invokes the scheduler specific * ipi_policy routine and sets pending_AST bits based on the result. * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform() * routine which actually sends the appropriate IPI to the destination core. */ extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, sched_ipi_event_t event); extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi); /* sched_ipi_policy() is the global default IPI policy for all schedulers */ extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); /* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */ extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, processor_t dst, thread_t thread, sched_ipi_event_t event); #if defined(CONFIG_SCHED_TIMESHARE_CORE) extern boolean_t thread_update_add_thread(thread_t thread); extern void thread_update_process_threads(void); extern boolean_t runq_scan(run_queue_t runq, sched_update_scan_context_t scan_context); #if CONFIG_SCHED_CLUTCH extern boolean_t sched_clutch_timeshare_scan(queue_t thread_queue, uint16_t count, sched_update_scan_context_t scan_context); #endif /* CONFIG_SCHED_CLUTCH */ extern void sched_timeshare_init(void); extern void sched_timeshare_timebase_init(void); extern void sched_timeshare_maintenance_continue(void); extern boolean_t priority_is_urgent(int priority); extern uint32_t sched_timeshare_initial_quantum_size(thread_t thread); extern int sched_compute_timeshare_priority(thread_t thread); #endif /* CONFIG_SCHED_TIMESHARE_CORE */ /* Remove thread from its run queue */ extern boolean_t thread_run_queue_remove(thread_t thread); thread_t thread_run_queue_remove_for_handoff(thread_t thread); /* Put a thread back in the run queue after being yanked */ extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options); extern void thread_timer_expire( void *thread, void *p1); extern bool thread_is_eager_preempt(thread_t thread); extern boolean_t sched_generic_direct_dispatch_to_idle_processors; /* Set the maximum interrupt level for the thread */ __private_extern__ wait_interrupt_t thread_interrupt_level( wait_interrupt_t interruptible); __private_extern__ wait_result_t thread_mark_wait_locked( thread_t thread, wait_interrupt_t interruptible); /* Wake up locked thread directly, passing result */ __private_extern__ kern_return_t clear_wait_internal( thread_t thread, wait_result_t result); struct sched_statistics { uint32_t csw_count; uint32_t preempt_count; uint32_t preempted_rt_count; uint32_t preempted_by_rt_count; uint32_t rt_sched_count; uint32_t interrupt_count; uint32_t ipi_count; uint32_t timer_pop_count; uint32_t idle_transitions; uint32_t quantum_timer_expirations; }; PERCPU_DECL(struct sched_statistics, sched_stats); extern bool sched_stats_active; extern void sched_stats_handle_csw( processor_t processor, int reasons, int selfpri, int otherpri); extern void sched_stats_handle_runq_change( struct runq_stats *stats, int old_count); #define SCHED_STATS_INC(field) \ MACRO_BEGIN \ if (__improbable(sched_stats_active)) { \ PERCPU_GET(sched_stats)->field++; \ } \ MACRO_END #if DEBUG #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) \ MACRO_BEGIN \ if (__improbable(sched_stats_active)) { \ sched_stats_handle_csw((processor), \ (reasons), (selfpri), (otherpri)); \ } \ MACRO_END #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) \ MACRO_BEGIN \ if (__improbable(sched_stats_active)) { \ sched_stats_handle_runq_change((stats), (old_count)); \ } \ MACRO_END #else /* DEBUG */ #define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0) #define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0) #endif /* DEBUG */ extern uint32_t sched_debug_flags; #define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS 0x00000001 #define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS 0x00000002 #define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...) \ MACRO_BEGIN \ if (__improbable(sched_debug_flags & \ SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) { \ KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ } \ MACRO_END #define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(...) \ MACRO_BEGIN \ if (__improbable(sched_debug_flags & \ SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) { \ KERNEL_DEBUG_CONSTANT(__VA_ARGS__); \ } \ MACRO_END /* Tells if there are "active" RT threads in the system (provided by CPU PM) */ extern void active_rt_threads( boolean_t active); /* Returns the perfcontrol attribute for the thread */ extern perfcontrol_class_t thread_get_perfcontrol_class( thread_t thread); /* Generic routine for Non-AMP schedulers to calculate parallelism */ extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options); extern void check_monotonic_time(uint64_t ctime); #endif /* MACH_KERNEL_PRIVATE */ __BEGIN_DECLS #ifdef XNU_KERNEL_PRIVATE extern void thread_bind_cluster_type(thread_t, char cluster_type, bool soft_bind); __options_decl(thread_bind_option_t, uint64_t, { /* Unbind a previously cluster bound thread */ THREAD_UNBIND = 0x1, /* * Soft bind the thread to the cluster; soft binding means the thread will be * moved to an available cluster if the bound cluster is de-recommended/offline. */ THREAD_BIND_SOFT = 0x2, /* * Bind thread to the cluster only if it is eligible to run on that cluster. If * the thread is not eligible to run on the cluster, thread_bind_cluster_id() * returns KERN_INVALID_POLICY. */ THREAD_BIND_ELIGIBLE_ONLY = 0x4, }); extern kern_return_t thread_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options); extern int sched_get_rt_n_backup_processors(void); extern void sched_set_rt_n_backup_processors(int n); extern int sched_get_rt_deadline_epsilon(void); extern void sched_set_rt_deadline_epsilon(int new_epsilon_us); /* Toggles a global override to turn off CPU Throttling */ extern void sys_override_cpu_throttle(boolean_t enable_override); extern int sched_get_powered_cores(void); extern void sched_set_powered_cores(int n); /* ****************** Only exported until BSD stops using ******************** */ extern void thread_vm_bind_group_add(void); /* Wake up thread directly, passing result */ extern kern_return_t clear_wait( thread_t thread, wait_result_t result); /* Start thread running */ extern void thread_bootstrap_return(void) __attribute__((noreturn)); /* Return from exception (BSD-visible interface) */ extern void thread_exception_return(void) __dead2; #define SCHED_STRING_MAX_LENGTH (48) /* String declaring the name of the current scheduler */ extern char sched_string[SCHED_STRING_MAX_LENGTH]; __options_decl(thread_handoff_option_t, uint32_t, { THREAD_HANDOFF_NONE = 0, THREAD_HANDOFF_SETRUN_NEEDED = 0x1, }); /* Remove thread from its run queue */ thread_t thread_prepare_for_handoff(thread_t thread, thread_handoff_option_t option); /* Attempt to context switch to a specific runnable thread */ extern wait_result_t thread_handoff_deallocate(thread_t thread, thread_handoff_option_t option); __attribute__((nonnull(2))) extern void thread_handoff_parameter(thread_t thread, thread_continue_t continuation, void *parameter, thread_handoff_option_t) __dead2; extern struct waitq *assert_wait_queue(event_t event); extern kern_return_t thread_wakeup_one_with_pri(event_t event, int priority); extern thread_t thread_wakeup_identify(event_t event, int priority); /* * sched_cond_t: * * A atomic condition variable used to synchronize wake/block operations on threads. * Bits defined below are reserved for use by sched_prim. Remaining * bits may be used by caller for additional synchronization semantics. */ __options_decl(sched_cond_t, uint32_t, { SCHED_COND_INIT = 0x0000, /* initialize all bits to zero (inactive and not awoken) */ SCHED_COND_ACTIVE = 0x0001, /* target thread is active */ SCHED_COND_WAKEUP = 0x0002 /* wakeup has been issued for target thread */ }); typedef _Atomic sched_cond_t sched_cond_atomic_t; /* * sched_cond_init: * * Initialize an atomic condition variable. Note that this does not occur atomically and should be * performed during thread initialization, before the condition is observable by other threads. */ extern void sched_cond_init( sched_cond_atomic_t *cond); /* * sched_cond_signal: * * Wakeup the specified thread if it is waiting on this event and it has not already been issued a wakeup. * * parameters: * thread thread to awaken * cond atomic condition variable */ extern kern_return_t sched_cond_signal( sched_cond_atomic_t *cond, thread_t thread); /* * sched_cond_wait: * * Assert wait and block on cond if no wakeup has been issued. * If a wakeup has been issued on cond since the last `sched_cond_ack`, clear_wait and * return `THREAD_AWAKENED`. * * `sched_cond_wait` must be paired with `sched_cond_ack`. * * NOTE: `continuation` will only be jumped to if a wakeup has not been issued * * parameters: * cond atomic condition variable to synchronize on * interruptible interruptible value to pass to assert_wait * continuation continuation if block succeeds */ extern wait_result_t sched_cond_wait( sched_cond_atomic_t *cond, wait_interrupt_t interruptible, thread_continue_t continuation); /* * sched_cond_ack: * * Acknowledge an issued wakeup by clearing WAKEUP and setting ACTIVE (via XOR). * It is the callers responsibility to ensure that the ACTIVE bit is always low prior to calling * (i.e. by calling `sched_cond_wait` prior to any rerun or block). * Synchronization schemes that allow for WAKEUP bit to be reset prior to wakeup * (e.g. a cancellation mechanism) should check that WAKEUP was indeed cleared. * * e.g. * ``` * if (sched_cond_ack(&my_state) & SCHED_THREAD_WAKEUP) { * // WAKEUP bit was no longer set by the time this thread woke up * do_cancellation_policy(); * } * ``` * * parameters: * cond: atomic condition variable */ extern sched_cond_t sched_cond_ack( sched_cond_atomic_t *cond); #endif /* XNU_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE /* Set pending block hint for a particular object before we go into a wait state */ extern void thread_set_pending_block_hint( thread_t thread, block_hint_t block_hint); #define QOS_PARALLELISM_COUNT_LOGICAL 0x1 #define QOS_PARALLELISM_REALTIME 0x2 #define QOS_PARALLELISM_CLUSTER_SHARED_RESOURCE 0x4 extern uint32_t qos_max_parallelism(int qos, uint64_t options); #endif /* KERNEL_PRIVATE */ #if XNU_KERNEL_PRIVATE extern void thread_yield_with_continuation( thread_continue_t continuation, void *parameter) __dead2; #endif /* Context switch */ extern wait_result_t thread_block( thread_continue_t continuation); extern wait_result_t thread_block_parameter( thread_continue_t continuation, void *parameter); /* Declare thread will wait on a particular event */ extern wait_result_t assert_wait( event_t event, wait_interrupt_t interruptible); /* Assert that the thread intends to wait with a timeout */ extern wait_result_t assert_wait_timeout( event_t event, wait_interrupt_t interruptible, uint32_t interval, uint32_t scale_factor); /* Assert that the thread intends to wait with an urgency, timeout and leeway */ extern wait_result_t assert_wait_timeout_with_leeway( event_t event, wait_interrupt_t interruptible, wait_timeout_urgency_t urgency, uint32_t interval, uint32_t leeway, uint32_t scale_factor); extern wait_result_t assert_wait_deadline( event_t event, wait_interrupt_t interruptible, uint64_t deadline); /* Assert that the thread intends to wait with an urgency, deadline, and leeway */ extern wait_result_t assert_wait_deadline_with_leeway( event_t event, wait_interrupt_t interruptible, wait_timeout_urgency_t urgency, uint64_t deadline, uint64_t leeway); /* Wake up thread (or threads) waiting on a particular event */ extern kern_return_t thread_wakeup_prim( event_t event, boolean_t one_thread, wait_result_t result); #define thread_wakeup(x) \ thread_wakeup_prim((x), FALSE, THREAD_AWAKENED) #define thread_wakeup_with_result(x, z) \ thread_wakeup_prim((x), FALSE, (z)) #define thread_wakeup_one(x) \ thread_wakeup_prim((x), TRUE, THREAD_AWAKENED) /* Wakeup the specified thread if it is waiting on this event */ extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread); extern boolean_t preemption_enabled(void); #ifdef MACH_KERNEL_PRIVATE /* * Scheduler algorithm indirection. If only one algorithm is * enabled at compile-time, a direction function call is used. * If more than one is enabled, calls are dispatched through * a function pointer table. */ #if !defined(CONFIG_SCHED_TRADITIONAL) && !defined(CONFIG_SCHED_PROTO) && !defined(CONFIG_SCHED_GRRR) && !defined(CONFIG_SCHED_MULTIQ) && !defined(CONFIG_SCHED_CLUTCH) && !defined(CONFIG_SCHED_EDGE) #error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX #endif #if __AMP__ #if CONFIG_SCHED_EDGE extern const struct sched_dispatch_table sched_edge_dispatch; #define SCHED(f) (sched_edge_dispatch.f) #else /* CONFIG_SCHED_EDGE */ extern const struct sched_dispatch_table sched_amp_dispatch; #define SCHED(f) (sched_amp_dispatch.f) #endif /* CONFIG_SCHED_EDGE */ #else /* __AMP__ */ #if CONFIG_SCHED_CLUTCH extern const struct sched_dispatch_table sched_clutch_dispatch; #define SCHED(f) (sched_clutch_dispatch.f) #else /* CONFIG_SCHED_CLUTCH */ extern const struct sched_dispatch_table sched_dualq_dispatch; #define SCHED(f) (sched_dualq_dispatch.f) #endif /* CONFIG_SCHED_CLUTCH */ #endif /* __AMP__ */ struct sched_dispatch_table { const char *sched_name; void (*init)(void); /* Init global state */ void (*timebase_init)(void); /* Timebase-dependent initialization */ void (*processor_init)(processor_t processor); /* Per-processor scheduler init */ void (*pset_init)(processor_set_t pset); /* Per-processor set scheduler init */ void (*maintenance_continuation)(void); /* Function called regularly */ /* * Choose a thread of greater or equal priority from the per-processor * runqueue for timeshare/fixed threads */ thread_t (*choose_thread)( processor_t processor, int priority, ast_t reason); /* True if scheduler supports stealing threads for this pset */ bool (*steal_thread_enabled)(processor_set_t pset); /* * Steal a thread from another processor in the pset so that it can run * immediately */ thread_t (*steal_thread)( processor_set_t pset); /* * Compute priority for a timeshare thread based on base priority. */ int (*compute_timeshare_priority)(thread_t thread); /* * Pick the best node for a thread to run on. */ pset_node_t (*choose_node)( thread_t thread); /* * Pick the best processor for a thread (any kind of thread) to run on. */ processor_t (*choose_processor)( processor_set_t pset, processor_t processor, thread_t thread); /* * Enqueue a timeshare or fixed priority thread onto the per-processor * runqueue */ boolean_t (*processor_enqueue)( processor_t processor, thread_t thread, sched_options_t options); /* Migrate threads away in preparation for processor shutdown */ void (*processor_queue_shutdown)( processor_t processor); /* Remove the specific thread from the per-processor runqueue */ boolean_t (*processor_queue_remove)( processor_t processor, thread_t thread); /* * Does the per-processor runqueue have any timeshare or fixed priority * threads on it? Called without pset lock held, so should * not assume immutability while executing. */ boolean_t (*processor_queue_empty)(processor_t processor); /* * Would this priority trigger an urgent preemption if it's sitting * on the per-processor runqueue? */ boolean_t (*priority_is_urgent)(int priority); /* * Does the per-processor runqueue contain runnable threads that * should cause the currently-running thread to be preempted? */ ast_t (*processor_csw_check)(processor_t processor); /* * Does the per-processor runqueue contain a runnable thread * of > or >= priority, as a preflight for choose_thread() or other * thread selection */ boolean_t (*processor_queue_has_priority)(processor_t processor, int priority, boolean_t gte); /* Quantum size for the specified non-realtime thread. */ uint32_t (*initial_quantum_size)(thread_t thread); /* Scheduler mode for a new thread */ sched_mode_t (*initial_thread_sched_mode)(task_t parent_task); /* * Is it safe to call update_priority, which may change a thread's * runqueue or other state. This can be used to throttle changes * to dynamic priority. */ boolean_t (*can_update_priority)(thread_t thread); /* * Update both scheduled priority and other persistent state. * Side effects may including migration to another processor's runqueue. */ void (*update_priority)(thread_t thread); /* Lower overhead update to scheduled priority and state. */ void (*lightweight_update_priority)(thread_t thread); /* Callback for non-realtime threads when the quantum timer fires */ void (*quantum_expire)(thread_t thread); /* * Runnable threads on per-processor runqueue. Should only * be used for relative comparisons of load between processors. */ int (*processor_runq_count)(processor_t processor); /* Aggregate runcount statistics for per-processor runqueue */ uint64_t (*processor_runq_stats_count_sum)(processor_t processor); boolean_t (*processor_bound_count)(processor_t processor); void (*thread_update_scan)(sched_update_scan_context_t scan_context); /* Supports more than one pset */ boolean_t multiple_psets_enabled; /* Supports scheduler groups */ boolean_t sched_groups_enabled; /* Supports avoid-processor */ boolean_t avoid_processor_enabled; /* Returns true if this processor should avoid running this thread. */ bool (*thread_avoid_processor)(processor_t processor, thread_t thread); /* * Invoked when a processor is about to choose the idle thread * Used to send IPIs to a processor which would be preferred to be idle instead. * Called with pset lock held, returns pset lock unlocked. */ void (*processor_balance)(processor_t processor, processor_set_t pset); rt_queue_t (*rt_runq)(processor_set_t pset); void (*rt_init)(processor_set_t pset); void (*rt_queue_shutdown)(processor_t processor); void (*rt_runq_scan)(sched_update_scan_context_t scan_context); int64_t (*rt_runq_count_sum)(void); thread_t (*rt_steal_thread)(processor_set_t pset, uint64_t earliest_deadline); uint32_t (*qos_max_parallelism)(int qos, uint64_t options); void (*check_spill)(processor_set_t pset, thread_t thread); sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event); bool (*thread_should_yield)(processor_t processor, thread_t thread); /* Routine to update run counts */ uint32_t (*run_count_incr)(thread_t thread); uint32_t (*run_count_decr)(thread_t thread); /* Routine to update scheduling bucket for a thread */ void (*update_thread_bucket)(thread_t thread); /* Routine to inform the scheduler when a new pset becomes schedulable */ void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock); #if CONFIG_THREAD_GROUPS /* Routine to inform the scheduler when CLPC changes a thread group recommendation */ void (*thread_group_recommendation_change)(struct thread_group *tg, cluster_type_t new_recommendation); #endif /* Routine to inform the scheduler when all CPUs have finished initializing */ void (*cpu_init_completed)(void); /* Routine to check if a thread is eligible to execute on a specific pset */ bool (*thread_eligible_for_pset)(thread_t thread, processor_set_t pset); }; #if defined(CONFIG_SCHED_TRADITIONAL) extern const struct sched_dispatch_table sched_traditional_dispatch; extern const struct sched_dispatch_table sched_traditional_with_pset_runqueue_dispatch; #endif #if defined(CONFIG_SCHED_MULTIQ) extern const struct sched_dispatch_table sched_multiq_dispatch; extern const struct sched_dispatch_table sched_dualq_dispatch; #if __AMP__ extern const struct sched_dispatch_table sched_amp_dispatch; #endif #endif #if defined(CONFIG_SCHED_PROTO) extern const struct sched_dispatch_table sched_proto_dispatch; #endif #if defined(CONFIG_SCHED_GRRR) extern const struct sched_dispatch_table sched_grrr_dispatch; #endif #if defined(CONFIG_SCHED_CLUTCH) extern const struct sched_dispatch_table sched_clutch_dispatch; #endif #if defined(CONFIG_SCHED_EDGE) extern const struct sched_dispatch_table sched_edge_dispatch; #endif extern void sched_set_max_unsafe_rt_quanta(int max); extern void sched_set_max_unsafe_fixed_quanta(int max); #endif /* MACH_KERNEL_PRIVATE */ __END_DECLS #endif /* _KERN_SCHED_PRIM_H_ */