/* * Copyright (c) 2019-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include #include #include #include #include #include /* for TASK_CHUNK */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * BSD interface functions */ size_t coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz); boolean_t coalition_is_leader(task_t task, coalition_t coal); task_t coalition_get_leader(coalition_t coal); int coalition_get_task_count(coalition_t coal); uint64_t coalition_get_page_count(coalition_t coal, int *ntasks); int coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order, int *pid_list, int list_sz); /* defined in task.c */ extern ledger_template_t task_ledger_template; /* * Templates; task template is copied due to potential allocation limits on * task ledgers. */ ledger_template_t coalition_task_ledger_template = NULL; ledger_template_t coalition_ledger_template = NULL; extern int proc_selfpid(void); /* * Coalition zone needs limits. We expect there will be as many coalitions as * tasks (same order of magnitude), so use the task zone's limits. * */ #define CONFIG_COALITION_MAX CONFIG_TASK_MAX #define COALITION_CHUNK TASK_CHUNK #if DEBUG || DEVELOPMENT TUNABLE_WRITEABLE(int, unrestrict_coalition_syscalls, "unrestrict_coalition_syscalls", 0); #else #define unrestrict_coalition_syscalls false #endif static LCK_GRP_DECLARE(coalitions_lck_grp, "coalition"); /* coalitions_list_lock protects coalition_hash, next_coalition_id. */ static LCK_MTX_DECLARE(coalitions_list_lock, &coalitions_lck_grp); static uint64_t coalition_next_id = 1; static struct smr_hash coalition_hash; coalition_t init_coalition[COALITION_NUM_TYPES]; coalition_t corpse_coalition[COALITION_NUM_TYPES]; static const char * coal_type_str(int type) { switch (type) { case COALITION_TYPE_RESOURCE: return "RESOURCE"; case COALITION_TYPE_JETSAM: return "JETSAM"; default: return ""; } } struct coalition_type { int type; int has_default; /* * init * pre-condition: coalition just allocated (unlocked), unreferenced, * type field set */ kern_return_t (*init)(coalition_t coal, boolean_t privileged, boolean_t efficient); /* * dealloc * pre-condition: coalition unlocked * pre-condition: coalition refcount=0, active_count=0, * termrequested=1, terminated=1, reaped=1 */ void (*dealloc)(coalition_t coal); /* * adopt_task * pre-condition: coalition locked * pre-condition: coalition !repead and !terminated */ kern_return_t (*adopt_task)(coalition_t coal, task_t task); /* * remove_task * pre-condition: coalition locked * pre-condition: task has been removed from coalition's task list */ kern_return_t (*remove_task)(coalition_t coal, task_t task); /* * set_taskrole * pre-condition: coalition locked * pre-condition: task added to coalition's task list, * active_count >= 1 (at least the given task is active) */ kern_return_t (*set_taskrole)(coalition_t coal, task_t task, int role); /* * get_taskrole * pre-condition: coalition locked * pre-condition: task added to coalition's task list, * active_count >= 1 (at least the given task is active) */ int (*get_taskrole)(coalition_t coal, task_t task); /* * iterate_tasks * pre-condition: coalition locked * if the callback returns true, stop iteration and return the task */ task_t (*iterate_tasks)(coalition_t coal, coalition_for_each_task_block_t); }; /* * COALITION_TYPE_RESOURCE */ static kern_return_t i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient); static void i_coal_resource_dealloc(coalition_t coal); static kern_return_t i_coal_resource_adopt_task(coalition_t coal, task_t task); static kern_return_t i_coal_resource_remove_task(coalition_t coal, task_t task); static kern_return_t i_coal_resource_set_taskrole(coalition_t coal, task_t task, int role); static int i_coal_resource_get_taskrole(coalition_t coal, task_t task); static task_t i_coal_resource_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block); /* * Ensure COALITION_NUM_THREAD_QOS_TYPES defined in mach/coalition.h still * matches THREAD_QOS_LAST defined in mach/thread_policy.h */ static_assert(COALITION_NUM_THREAD_QOS_TYPES == THREAD_QOS_LAST); struct i_resource_coalition { /* * This keeps track of resource utilization of tasks that are no longer active * in the coalition and is updated when a task is removed from the coalition. */ ledger_t ledger; uint64_t bytesread; uint64_t byteswritten; uint64_t energy; uint64_t gpu_time; uint64_t logical_immediate_writes; uint64_t logical_deferred_writes; uint64_t logical_invalidated_writes; uint64_t logical_metadata_writes; uint64_t logical_immediate_writes_to_external; uint64_t logical_deferred_writes_to_external; uint64_t logical_invalidated_writes_to_external; uint64_t logical_metadata_writes_to_external; uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per effective QoS class */ uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per requested QoS class */ uint64_t cpu_instructions; uint64_t cpu_cycles; uint64_t ane_mach_time; uint64_t ane_energy_nj; _Atomic uint64_t gpu_energy_nj; _Atomic uint64_t gpu_energy_nj_billed_to_me; _Atomic uint64_t gpu_energy_nj_billed_to_others; struct recount_coalition co_recount; uint64_t task_count; /* tasks that have started in this coalition */ uint64_t dead_task_count; /* tasks that have exited in this coalition; * subtract from task_count to get count * of "active" tasks */ /* * Count the length of time this coalition had at least one active task. * This can be a 'denominator' to turn e.g. cpu_time to %cpu. * */ uint64_t last_became_nonempty_time; uint64_t time_nonempty; queue_head_t tasks; /* List of active tasks in the coalition */ /* * This ledger is used for triggering resource exception. For the tracked resources, this is updated * when the member tasks' resource usage changes. */ ledger_t resource_monitor_ledger; #if CONFIG_PHYS_WRITE_ACCT uint64_t fs_metadata_writes; #endif /* CONFIG_PHYS_WRITE_ACCT */ }; /* * COALITION_TYPE_JETSAM */ static kern_return_t i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient); static void i_coal_jetsam_dealloc(coalition_t coal); static kern_return_t i_coal_jetsam_adopt_task(coalition_t coal, task_t task); static kern_return_t i_coal_jetsam_remove_task(coalition_t coal, task_t task); static kern_return_t i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role); int i_coal_jetsam_get_taskrole(coalition_t coal, task_t task); static task_t i_coal_jetsam_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block); struct i_jetsam_coalition { task_t leader; queue_head_t extensions; queue_head_t services; queue_head_t other; struct thread_group *thread_group; bool swap_enabled; struct coalition_requested_policy { uint64_t crp_darwinbg :1, /* marked as darwinbg via setpriority */ crp_reserved :63; } c_requested_policy; struct coalition_effective_policy { uint64_t cep_darwinbg :1, /* marked as darwinbg via setpriority */ cep_reserved :63; } c_effective_policy; }; /* * main coalition structure */ struct coalition { uint64_t id; /* monotonically increasing */ uint32_t type; uint32_t role; /* default task role (background, adaptive, interactive, etc) */ os_ref_atomic_t ref_count; /* Number of references to the memory containing this struct */ uint32_t active_count; /* Number of members of (tasks in) the * coalition, plus vouchers referring * to the coalition */ uint32_t focal_task_count; /* Number of TASK_FOREGROUND_APPLICATION tasks in the coalition */ uint32_t nonfocal_task_count; /* Number of TASK_BACKGROUND_APPLICATION tasks in the coalition */ uint32_t game_task_count; /* Number of GAME_MODE tasks in the coalition */ uint32_t carplay_task_count; /* Number of CARPLAY_MODE tasks in the coalition */ /* coalition flags */ uint32_t privileged : 1; /* Members of this coalition may create * and manage coalitions and may posix_spawn * processes into selected coalitions */ /* ast? */ /* voucher */ uint32_t termrequested : 1; /* launchd has requested termination when coalition becomes empty */ uint32_t terminated : 1; /* coalition became empty and spawns are now forbidden */ uint32_t reaped : 1; /* reaped, invisible to userspace, but waiting for ref_count to go to zero */ uint32_t notified : 1; /* no-more-processes notification was sent via special port */ uint32_t efficient : 1; /* launchd has marked the coalition as efficient */ #if DEVELOPMENT || DEBUG uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */ #endif struct smrq_slink link; /* global list of coalitions */ union { lck_mtx_t lock; /* Coalition lock. */ struct smr_node smr_node; }; /* put coalition type-specific structures here */ union { struct i_resource_coalition r; struct i_jetsam_coalition j; }; }; os_refgrp_decl(static, coal_ref_grp, "coalitions", NULL); #define coal_ref_init(coal, c) os_ref_init_count_raw(&(coal)->ref_count, &coal_ref_grp, c) #define coal_ref_count(coal) os_ref_get_count_raw(&(coal)->ref_count) #define coal_ref_try_retain(coal) os_ref_retain_try_raw(&(coal)->ref_count, &coal_ref_grp) #define coal_ref_retain(coal) os_ref_retain_raw(&(coal)->ref_count, &coal_ref_grp) #define coal_ref_release(coal) os_ref_release_raw(&(coal)->ref_count, &coal_ref_grp) #define coal_ref_release_live(coal) os_ref_release_live_raw(&(coal)->ref_count, &coal_ref_grp) #define COALITION_HASH_SIZE_MIN 16 static bool coal_hash_obj_try_get(void *coal) { return coal_ref_try_retain((struct coalition *)coal); } SMRH_TRAITS_DEFINE_SCALAR(coal_hash_traits, struct coalition, id, link, .domain = &smr_proc_task, .obj_try_get = coal_hash_obj_try_get); /* * register different coalition types: * these must be kept in the order specified in coalition.h */ static const struct coalition_type s_coalition_types[COALITION_NUM_TYPES] = { { .type = COALITION_TYPE_RESOURCE, .has_default = 1, .init = i_coal_resource_init, .dealloc = i_coal_resource_dealloc, .adopt_task = i_coal_resource_adopt_task, .remove_task = i_coal_resource_remove_task, .set_taskrole = i_coal_resource_set_taskrole, .get_taskrole = i_coal_resource_get_taskrole, .iterate_tasks = i_coal_resource_iterate_tasks, }, { .type = COALITION_TYPE_JETSAM, .has_default = 1, .init = i_coal_jetsam_init, .dealloc = i_coal_jetsam_dealloc, .adopt_task = i_coal_jetsam_adopt_task, .remove_task = i_coal_jetsam_remove_task, .set_taskrole = i_coal_jetsam_set_taskrole, .get_taskrole = i_coal_jetsam_get_taskrole, .iterate_tasks = i_coal_jetsam_iterate_tasks, }, }; static KALLOC_TYPE_DEFINE(coalition_zone, struct coalition, KT_PRIV_ACCT); #define coal_call(coal, func, ...) \ (s_coalition_types[(coal)->type].func)(coal, ## __VA_ARGS__) #define coalition_lock(c) lck_mtx_lock(&(c)->lock) #define coalition_unlock(c) lck_mtx_unlock(&(c)->lock) /* * Define the coalition type to track focal tasks. * On embedded, track them using jetsam coalitions since they have associated thread * groups which reflect this property as a flag (and pass it down to CLPC). * On non-embedded platforms, since not all coalitions have jetsam coalitions * track focal counts on the resource coalition. */ #if !XNU_TARGET_OS_OSX #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_JETSAM #else /* !XNU_TARGET_OS_OSX */ #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_RESOURCE #endif /* !XNU_TARGET_OS_OSX */ /* * * Coalition ledger implementation * */ struct coalition_ledger_indices coalition_ledgers = {.logical_writes = -1, }; void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor); ledger_t coalition_ledger_get_from_task(task_t task) { ledger_t ledger = LEDGER_NULL; coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE]; if (coal != NULL && (!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]))) { ledger = coal->r.resource_monitor_ledger; ledger_reference(ledger); } return ledger; } enum { COALITION_IO_LEDGER_ENABLE, COALITION_IO_LEDGER_DISABLE }; void coalition_io_monitor_ctl(struct coalition *coalition, uint32_t flags, int64_t limit) { ledger_t ledger = coalition->r.resource_monitor_ledger; if (flags == COALITION_IO_LEDGER_ENABLE) { /* Configure the logical I/O ledger */ ledger_set_limit(ledger, coalition_ledgers.logical_writes, (limit * 1024 * 1024), 0); ledger_set_period(ledger, coalition_ledgers.logical_writes, (COALITION_LEDGER_MONITOR_INTERVAL_SECS * NSEC_PER_SEC)); } else if (flags == COALITION_IO_LEDGER_DISABLE) { ledger_disable_refill(ledger, coalition_ledgers.logical_writes); ledger_disable_callback(ledger, coalition_ledgers.logical_writes); } } int coalition_ledger_set_logical_writes_limit(struct coalition *coalition, int64_t limit) { int error = 0; /* limit = -1 will be used to disable the limit and the callback */ if (limit > COALITION_MAX_LOGICAL_WRITES_LIMIT || limit == 0 || limit < -1) { error = EINVAL; goto out; } coalition_lock(coalition); if (limit == -1) { coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_DISABLE, limit); } else { coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_ENABLE, limit); } coalition_unlock(coalition); out: return error; } void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor) { int pid = proc_selfpid(); ledger_amount_t new_limit; task_t task = current_task(); struct ledger_entry_info lei; kern_return_t kr; ledger_t ledger; struct coalition *coalition = task->coalition[COALITION_TYPE_RESOURCE]; assert(coalition != NULL); ledger = coalition->r.resource_monitor_ledger; switch (flavor) { case FLAVOR_IO_LOGICAL_WRITES: ledger_get_entry_info(ledger, coalition_ledgers.logical_writes, &lei); trace_resource_violation(RMON_LOGWRITES_VIOLATED, &lei); break; default: goto Exit; } os_log(OS_LOG_DEFAULT, "Coalition [%lld] caught causing excessive I/O (flavor: %d). Task I/O: %lld MB. [Limit : %lld MB per %lld secs]. Triggered by process [%d]\n", coalition->id, flavor, (lei.lei_balance / (1024 * 1024)), (lei.lei_limit / (1024 * 1024)), (lei.lei_refill_period / NSEC_PER_SEC), pid); kr = send_resource_violation(send_disk_writes_violation, task, &lei, kRNFlagsNone); if (kr) { os_log(OS_LOG_DEFAULT, "ERROR %#x returned from send_resource_violation(disk_writes, ...)\n", kr); } /* * Continue to monitor the coalition after it hits the initital limit, but increase * the limit exponentially so that we don't spam the listener. */ new_limit = (lei.lei_limit / 1024 / 1024) * 4; coalition_lock(coalition); if (new_limit > COALITION_MAX_LOGICAL_WRITES_LIMIT) { coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_DISABLE, -1); } else { coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_ENABLE, new_limit); } coalition_unlock(coalition); Exit: return; } void coalition_io_rate_exceeded(int warning, const void *param0, __unused const void *param1) { if (warning == 0) { SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO((int)param0); } } void init_coalition_ledgers(void) { ledger_template_t t; assert(coalition_ledger_template == NULL); if ((t = ledger_template_create("Per-coalition ledgers")) == NULL) { panic("couldn't create coalition ledger template"); } coalition_ledgers.logical_writes = ledger_entry_add(t, "logical_writes", "res", "bytes"); if (coalition_ledgers.logical_writes < 0) { panic("couldn't create entries for coaliton ledger template"); } ledger_set_callback(t, coalition_ledgers.logical_writes, coalition_io_rate_exceeded, (void *)FLAVOR_IO_LOGICAL_WRITES, NULL); ledger_template_complete(t); coalition_task_ledger_template = ledger_template_copy(task_ledger_template, "Coalition task ledgers"); if (coalition_task_ledger_template == NULL) { panic("couldn't create coalition task ledger template"); } ledger_template_complete(coalition_task_ledger_template); coalition_ledger_template = t; } void coalition_io_ledger_update(task_t task, int32_t flavor, boolean_t is_credit, uint32_t io_size) { ledger_t ledger; coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE]; assert(coal != NULL); ledger = coal->r.resource_monitor_ledger; if (LEDGER_VALID(ledger)) { if (flavor == FLAVOR_IO_LOGICAL_WRITES) { if (is_credit) { ledger_credit(ledger, coalition_ledgers.logical_writes, io_size); } else { ledger_debit(ledger, coalition_ledgers.logical_writes, io_size); } } } } static void coalition_notify_user(uint64_t id, uint32_t flags) { mach_port_t user_port; kern_return_t kr; kr = host_get_coalition_port(host_priv_self(), &user_port); if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { return; } coalition_notification(user_port, id, flags); ipc_port_release_send(user_port); } /* * * COALITION_TYPE_RESOURCE * */ static kern_return_t i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient) { #pragma unused(privileged, efficient) assert(coal && coal->type == COALITION_TYPE_RESOURCE); recount_coalition_init(&coal->r.co_recount); coal->r.ledger = ledger_instantiate(coalition_task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES); if (coal->r.ledger == NULL) { return KERN_RESOURCE_SHORTAGE; } coal->r.resource_monitor_ledger = ledger_instantiate(coalition_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES); if (coal->r.resource_monitor_ledger == NULL) { return KERN_RESOURCE_SHORTAGE; } queue_init(&coal->r.tasks); return KERN_SUCCESS; } static void i_coal_resource_dealloc(coalition_t coal) { assert(coal && coal->type == COALITION_TYPE_RESOURCE); recount_coalition_deinit(&coal->r.co_recount); ledger_dereference(coal->r.ledger); ledger_dereference(coal->r.resource_monitor_ledger); } static kern_return_t i_coal_resource_adopt_task(coalition_t coal, task_t task) { struct i_resource_coalition *cr; assert(coal && coal->type == COALITION_TYPE_RESOURCE); assert(queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE])); cr = &coal->r; cr->task_count++; if (cr->task_count < cr->dead_task_count) { panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)", __func__, coal, coal->id, coal_type_str(coal->type), cr->task_count, cr->dead_task_count); } /* If moving from 0->1 active tasks */ if (cr->task_count - cr->dead_task_count == 1) { cr->last_became_nonempty_time = mach_absolute_time(); } /* put the task on the coalition's list of tasks */ enqueue_tail(&cr->tasks, &task->task_coalition[COALITION_TYPE_RESOURCE]); coal_dbg("Added PID:%d to id:%llu, task_count:%llu, dead_count:%llu, nonempty_time:%llu", task_pid(task), coal->id, cr->task_count, cr->dead_task_count, cr->last_became_nonempty_time); return KERN_SUCCESS; } static kern_return_t i_coal_resource_remove_task(coalition_t coal, task_t task) { struct i_resource_coalition *cr; assert(coal && coal->type == COALITION_TYPE_RESOURCE); assert(task->coalition[COALITION_TYPE_RESOURCE] == coal); assert(!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE])); /* * handle resource coalition accounting rollup for dead tasks */ cr = &coal->r; cr->dead_task_count++; if (cr->task_count < cr->dead_task_count) { panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)", __func__, coal, coal->id, coal_type_str(coal->type), cr->task_count, cr->dead_task_count); } /* If moving from 1->0 active tasks */ if (cr->task_count - cr->dead_task_count == 0) { uint64_t last_time_nonempty = mach_absolute_time() - cr->last_became_nonempty_time; cr->last_became_nonempty_time = 0; cr->time_nonempty += last_time_nonempty; } /* Do not roll up for exec'd task or exec copy task */ if (!task_is_exec_copy(task) && !task_did_exec(task)) { ledger_rollup(cr->ledger, task->ledger); cr->bytesread += task->task_io_stats->disk_reads.size; cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size; #if defined(__x86_64__) cr->gpu_time += task_gpu_utilisation(task); #endif /* defined(__x86_64__) */ cr->logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes; cr->logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes; cr->logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes; cr->logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes; cr->logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes; cr->logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes; cr->logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes; cr->logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes; #if CONFIG_PHYS_WRITE_ACCT cr->fs_metadata_writes += task->task_fs_metadata_writes; #endif /* CONFIG_PHYS_WRITE_ACCT */ task_update_cpu_time_qos_stats(task, cr->cpu_time_eqos, cr->cpu_time_rqos); recount_coalition_rollup_task(&cr->co_recount, &task->tk_recount); } /* remove the task from the coalition's list */ remqueue(&task->task_coalition[COALITION_TYPE_RESOURCE]); queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]); coal_dbg("removed PID:%d from id:%llu, task_count:%llu, dead_count:%llu", task_pid(task), coal->id, cr->task_count, cr->dead_task_count); return KERN_SUCCESS; } static kern_return_t i_coal_resource_set_taskrole(__unused coalition_t coal, __unused task_t task, __unused int role) { return KERN_SUCCESS; } static int i_coal_resource_get_taskrole(__unused coalition_t coal, __unused task_t task) { task_t t; assert(coal && coal->type == COALITION_TYPE_RESOURCE); qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) { if (t == task) { return COALITION_TASKROLE_UNDEF; } } return -1; } static task_t i_coal_resource_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block) { task_t t; assert(coal && coal->type == COALITION_TYPE_RESOURCE); qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) { bool should_return = block(t); if (should_return) { return t; } } return TASK_NULL; } #if CONFIG_PHYS_WRITE_ACCT extern uint64_t kernel_pm_writes; #endif /* CONFIG_PHYS_WRITE_ACCT */ kern_return_t coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_usage *cru_out) { kern_return_t kr; ledger_amount_t credit, debit; int i; if (coal->type != COALITION_TYPE_RESOURCE) { return KERN_INVALID_ARGUMENT; } /* Return KERN_INVALID_ARGUMENT for Corpse coalition */ for (i = 0; i < COALITION_NUM_TYPES; i++) { if (coal == corpse_coalition[i]) { return KERN_INVALID_ARGUMENT; } } ledger_t sum_ledger = ledger_instantiate(coalition_task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES); if (sum_ledger == LEDGER_NULL) { return KERN_RESOURCE_SHORTAGE; } coalition_lock(coal); /* * Start with the coalition's ledger, which holds the totals from all * the dead tasks. */ ledger_rollup(sum_ledger, coal->r.ledger); uint64_t bytesread = coal->r.bytesread; uint64_t byteswritten = coal->r.byteswritten; uint64_t gpu_time = coal->r.gpu_time; uint64_t logical_immediate_writes = coal->r.logical_immediate_writes; uint64_t logical_deferred_writes = coal->r.logical_deferred_writes; uint64_t logical_invalidated_writes = coal->r.logical_invalidated_writes; uint64_t logical_metadata_writes = coal->r.logical_metadata_writes; uint64_t logical_immediate_writes_to_external = coal->r.logical_immediate_writes_to_external; uint64_t logical_deferred_writes_to_external = coal->r.logical_deferred_writes_to_external; uint64_t logical_invalidated_writes_to_external = coal->r.logical_invalidated_writes_to_external; uint64_t logical_metadata_writes_to_external = coal->r.logical_metadata_writes_to_external; uint64_t ane_mach_time = os_atomic_load(&coal->r.ane_mach_time, relaxed); uint64_t ane_energy_nj = os_atomic_load(&coal->r.ane_energy_nj, relaxed); #if CONFIG_PHYS_WRITE_ACCT uint64_t fs_metadata_writes = coal->r.fs_metadata_writes; #endif /* CONFIG_PHYS_WRITE_ACCT */ int64_t conclave_mem = 0; int64_t cpu_time_billed_to_me = 0; int64_t cpu_time_billed_to_others = 0; int64_t energy_billed_to_me = 0; int64_t energy_billed_to_others = 0; int64_t phys_footprint = 0; struct recount_usage stats_sum = { 0 }; struct recount_usage stats_perf_only = { 0 }; recount_coalition_usage_perf_only(&coal->r.co_recount, &stats_sum, &stats_perf_only); uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 }; uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 }; /* * Add to that all the active tasks' ledgers. Tasks cannot deallocate * out from under us, since we hold the coalition lock. */ task_t task; qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) { /* * Rolling up stats for exec copy task or exec'd task will lead to double accounting. * Cannot take task lock after taking coaliton lock */ if (task_is_exec_copy(task) || task_did_exec(task)) { continue; } ledger_rollup(sum_ledger, task->ledger); bytesread += task->task_io_stats->disk_reads.size; byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size; #if defined(__x86_64__) gpu_time += task_gpu_utilisation(task); #endif /* defined(__x86_64__) */ logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes; logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes; logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes; logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes; logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes; logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes; logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes; logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes; #if CONFIG_PHYS_WRITE_ACCT fs_metadata_writes += task->task_fs_metadata_writes; #endif /* CONFIG_PHYS_WRITE_ACCT */ task_update_cpu_time_qos_stats(task, cpu_time_eqos, cpu_time_rqos); recount_task_usage_perf_only(task, &stats_sum, &stats_perf_only); } kr = ledger_get_balance(sum_ledger, task_ledgers.conclave_mem, (int64_t *)&conclave_mem); if (kr != KERN_SUCCESS || conclave_mem < 0) { conclave_mem = 0; } kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me); if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) { cpu_time_billed_to_me = 0; } kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others); if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) { cpu_time_billed_to_others = 0; } kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_me, (int64_t *)&energy_billed_to_me); if (kr != KERN_SUCCESS || energy_billed_to_me < 0) { energy_billed_to_me = 0; } kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_others, (int64_t *)&energy_billed_to_others); if (kr != KERN_SUCCESS || energy_billed_to_others < 0) { energy_billed_to_others = 0; } kr = ledger_get_balance(sum_ledger, task_ledgers.phys_footprint, (int64_t *)&phys_footprint); if (kr != KERN_SUCCESS || phys_footprint < 0) { phys_footprint = 0; } /* collect information from the coalition itself */ cru_out->tasks_started = coal->r.task_count; cru_out->tasks_exited = coal->r.dead_task_count; uint64_t time_nonempty = coal->r.time_nonempty; uint64_t last_became_nonempty_time = coal->r.last_became_nonempty_time; cru_out->gpu_energy_nj = os_atomic_load(&coal->r.gpu_energy_nj, relaxed); cru_out->gpu_energy_nj_billed_to_me = os_atomic_load(&coal->r.gpu_energy_nj_billed_to_me, relaxed); cru_out->gpu_energy_nj_billed_to_others = os_atomic_load(&coal->r.gpu_energy_nj_billed_to_others, relaxed); coalition_unlock(coal); /* Copy the totals out of sum_ledger */ kr = ledger_get_entries(sum_ledger, task_ledgers.cpu_time, &credit, &debit); if (kr != KERN_SUCCESS) { credit = 0; } cru_out->conclave_mem = conclave_mem; cru_out->cpu_time = credit; cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me; cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others; cru_out->energy_billed_to_me = (uint64_t)energy_billed_to_me; cru_out->energy_billed_to_others = (uint64_t)energy_billed_to_others; cru_out->phys_footprint = phys_footprint; kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups, &credit, &debit); if (kr != KERN_SUCCESS) { credit = 0; } cru_out->interrupt_wakeups = credit; kr = ledger_get_entries(sum_ledger, task_ledgers.platform_idle_wakeups, &credit, &debit); if (kr != KERN_SUCCESS) { credit = 0; } cru_out->platform_idle_wakeups = credit; cru_out->bytesread = bytesread; cru_out->byteswritten = byteswritten; cru_out->gpu_time = gpu_time; cru_out->ane_mach_time = ane_mach_time; cru_out->ane_energy_nj = ane_energy_nj; cru_out->logical_immediate_writes = logical_immediate_writes; cru_out->logical_deferred_writes = logical_deferred_writes; cru_out->logical_invalidated_writes = logical_invalidated_writes; cru_out->logical_metadata_writes = logical_metadata_writes; cru_out->logical_immediate_writes_to_external = logical_immediate_writes_to_external; cru_out->logical_deferred_writes_to_external = logical_deferred_writes_to_external; cru_out->logical_invalidated_writes_to_external = logical_invalidated_writes_to_external; cru_out->logical_metadata_writes_to_external = logical_metadata_writes_to_external; #if CONFIG_PHYS_WRITE_ACCT cru_out->fs_metadata_writes = fs_metadata_writes; #else cru_out->fs_metadata_writes = 0; #endif /* CONFIG_PHYS_WRITE_ACCT */ cru_out->cpu_time_eqos_len = COALITION_NUM_THREAD_QOS_TYPES; memcpy(cru_out->cpu_time_eqos, cpu_time_eqos, sizeof(cru_out->cpu_time_eqos)); cru_out->cpu_ptime = recount_usage_time_mach(&stats_perf_only); #if CONFIG_PERVASIVE_CPI cru_out->cpu_instructions = recount_usage_instructions(&stats_sum); cru_out->cpu_cycles = recount_usage_cycles(&stats_sum); cru_out->cpu_pinstructions = recount_usage_instructions(&stats_perf_only); cru_out->cpu_pcycles = recount_usage_cycles(&stats_perf_only); #endif // CONFIG_PERVASIVE_CPI ledger_dereference(sum_ledger); sum_ledger = LEDGER_NULL; #if CONFIG_PERVASIVE_ENERGY cru_out->energy = stats_sum.ru_energy_nj; #endif /* CONFIG_PERVASIVE_ENERGY */ #if CONFIG_PHYS_WRITE_ACCT // kernel_pm_writes are only recorded under kernel_task coalition if (coalition_id(coal) == COALITION_ID_KERNEL) { cru_out->pm_writes = kernel_pm_writes; } else { cru_out->pm_writes = 0; } #else cru_out->pm_writes = 0; #endif /* CONFIG_PHYS_WRITE_ACCT */ if (last_became_nonempty_time) { time_nonempty += mach_absolute_time() - last_became_nonempty_time; } absolutetime_to_nanoseconds(time_nonempty, &cru_out->time_nonempty); return KERN_SUCCESS; } bool coalition_add_to_gpu_energy(uint64_t coalition_id, coalition_gpu_energy_t which, uint64_t energy) { assert(coalition_id != 0); bool exists = false; smrh_key_t key = SMRH_SCALAR_KEY(coalition_id); smrht_enter(&coal_hash_traits); coalition_t coal = smr_hash_entered_find(&coalition_hash, key, &coal_hash_traits); /* Preemption is disabled, and until we leave, coalition_free won't be called. */ if (coal != COALITION_NULL && coal->type == COALITION_TYPE_RESOURCE && !coal->terminated && !coal->reaped) { /* * After termination, there are no more member tasks and * launchd is about to capture the final snapshot, so * a terminated energy id considered invalid. * * Note that we don't lock against the termination transition * here, so it's possible for a race to occur where new energy * is accounted past the final snapshot. */ exists = true; if (which & CGE_SELF) { os_atomic_add(&coal->r.gpu_energy_nj, energy, relaxed); } if (which & CGE_BILLED) { os_atomic_add(&coal->r.gpu_energy_nj_billed_to_me, energy, relaxed); } if (which & CGE_OTHERS) { os_atomic_add(&coal->r.gpu_energy_nj_billed_to_others, energy, relaxed); } } smrht_leave(&coal_hash_traits); return exists; } kern_return_t coalition_debug_info_internal(coalition_t coal, struct coalinfo_debuginfo *c_debuginfo) { /* Return KERN_INVALID_ARGUMENT for Corpse coalition */ for (int i = 0; i < COALITION_NUM_TYPES; i++) { if (coal == corpse_coalition[i]) { return KERN_INVALID_ARGUMENT; } } if (coal->type == COALITION_FOCAL_TASKS_ACCOUNTING) { c_debuginfo->focal_task_count = coal->focal_task_count; c_debuginfo->nonfocal_task_count = coal->nonfocal_task_count; c_debuginfo->game_task_count = coal->game_task_count; c_debuginfo->carplay_task_count = coal->carplay_task_count; } #if CONFIG_THREAD_GROUPS struct thread_group * group = coalition_get_thread_group(coal); if (group != NULL) { c_debuginfo->thread_group_id = thread_group_id(group); c_debuginfo->thread_group_flags = thread_group_get_flags(group); c_debuginfo->thread_group_recommendation = thread_group_recommendation(group); } #endif /* CONFIG_THREAD_GROUPS */ return KERN_SUCCESS; } /* * * COALITION_TYPE_JETSAM * */ static kern_return_t i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient) { assert(coal && coal->type == COALITION_TYPE_JETSAM); (void)privileged; (void)efficient; coal->j.leader = TASK_NULL; queue_head_init(coal->j.extensions); queue_head_init(coal->j.services); queue_head_init(coal->j.other); #if CONFIG_THREAD_GROUPS switch (coal->role) { case COALITION_ROLE_SYSTEM: coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_SYSTEM); break; case COALITION_ROLE_BACKGROUND: coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_BACKGROUND); break; default: coal->j.thread_group = thread_group_create_and_retain(efficient ? THREAD_GROUP_FLAGS_EFFICIENT : THREAD_GROUP_FLAGS_DEFAULT); } assert(coal->j.thread_group != NULL); #endif return KERN_SUCCESS; } static void i_coal_jetsam_dealloc(__unused coalition_t coal) { assert(coal && coal->type == COALITION_TYPE_JETSAM); /* the coalition should be completely clear at this point */ assert(queue_empty(&coal->j.extensions)); assert(queue_empty(&coal->j.services)); assert(queue_empty(&coal->j.other)); assert(coal->j.leader == TASK_NULL); #if CONFIG_THREAD_GROUPS /* disassociate from the thread group */ assert(coal->j.thread_group != NULL); thread_group_release(coal->j.thread_group); coal->j.thread_group = NULL; #endif } static kern_return_t i_coal_jetsam_adopt_task(coalition_t coal, task_t task) { struct i_jetsam_coalition *cj; assert(coal && coal->type == COALITION_TYPE_JETSAM); cj = &coal->j; assert(queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM])); /* put each task initially in the "other" list */ enqueue_tail(&cj->other, &task->task_coalition[COALITION_TYPE_JETSAM]); coal_dbg("coalition %lld adopted PID:%d as UNDEF", coal->id, task_pid(task)); return KERN_SUCCESS; } static kern_return_t i_coal_jetsam_remove_task(coalition_t coal, task_t task) { assert(coal && coal->type == COALITION_TYPE_JETSAM); assert(task->coalition[COALITION_TYPE_JETSAM] == coal); coal_dbg("removing PID:%d from coalition id:%lld", task_pid(task), coal->id); if (task == coal->j.leader) { coal->j.leader = NULL; coal_dbg(" PID:%d was the leader!", task_pid(task)); } else { assert(!queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM])); } /* remove the task from the specific coalition role queue */ remqueue(&task->task_coalition[COALITION_TYPE_JETSAM]); queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]); return KERN_SUCCESS; } static kern_return_t i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role) { struct i_jetsam_coalition *cj; queue_t q = NULL; assert(coal && coal->type == COALITION_TYPE_JETSAM); assert(task->coalition[COALITION_TYPE_JETSAM] == coal); cj = &coal->j; switch (role) { case COALITION_TASKROLE_LEADER: coal_dbg("setting PID:%d as LEADER of %lld", task_pid(task), coal->id); if (cj->leader != TASK_NULL) { /* re-queue the exiting leader onto the "other" list */ coal_dbg(" re-queue existing leader (%d) as OTHER", task_pid(cj->leader)); re_queue_tail(&cj->other, &cj->leader->task_coalition[COALITION_TYPE_JETSAM]); } /* * remove the task from the "other" list * (where it was put by default) */ remqueue(&task->task_coalition[COALITION_TYPE_JETSAM]); queue_chain_init(task->task_coalition[COALITION_TYPE_JETSAM]); /* set the coalition leader */ cj->leader = task; break; case COALITION_TASKROLE_XPC: coal_dbg("setting PID:%d as XPC in %lld", task_pid(task), coal->id); q = (queue_t)&cj->services; break; case COALITION_TASKROLE_EXT: coal_dbg("setting PID:%d as EXT in %lld", task_pid(task), coal->id); q = (queue_t)&cj->extensions; break; case COALITION_TASKROLE_NONE: /* * Tasks with a role of "none" should fall through to an * undefined role so long as the task is currently a member * of the coalition. This scenario can happen if a task is * killed (usually via jetsam) during exec. */ if (task->coalition[COALITION_TYPE_JETSAM] != coal) { panic("%s: task %p attempting to set role %d " "in coalition %p to which it does not belong!", __func__, task, role, coal); } OS_FALLTHROUGH; case COALITION_TASKROLE_UNDEF: coal_dbg("setting PID:%d as UNDEF in %lld", task_pid(task), coal->id); q = (queue_t)&cj->other; break; default: panic("%s: invalid role(%d) for task", __func__, role); return KERN_INVALID_ARGUMENT; } if (q != NULL) { re_queue_tail(q, &task->task_coalition[COALITION_TYPE_JETSAM]); } return KERN_SUCCESS; } int i_coal_jetsam_get_taskrole(coalition_t coal, task_t task) { struct i_jetsam_coalition *cj; task_t t; assert(coal && coal->type == COALITION_TYPE_JETSAM); assert(task->coalition[COALITION_TYPE_JETSAM] == coal); cj = &coal->j; if (task == cj->leader) { return COALITION_TASKROLE_LEADER; } qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) { if (t == task) { return COALITION_TASKROLE_XPC; } } qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) { if (t == task) { return COALITION_TASKROLE_EXT; } } qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) { if (t == task) { return COALITION_TASKROLE_UNDEF; } } /* task not in the coalition?! */ return COALITION_TASKROLE_NONE; } static task_t i_coal_jetsam_iterate_tasks( coalition_t coal, coalition_for_each_task_block_t block) { struct i_jetsam_coalition *cj; task_t t; assert(coal && coal->type == COALITION_TYPE_JETSAM); cj = &coal->j; if (cj->leader) { t = cj->leader; bool should_return = block( t); if (should_return) { return t; } } qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) { bool should_return = block(t); if (should_return) { return t; } } qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) { bool should_return = block(t); if (should_return) { return t; } } qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) { bool should_return = block(t); if (should_return) { return t; } } return TASK_NULL; } /* * * Main Coalition implementation * */ /* * coalition_create_internal * Returns: New coalition object, referenced for the caller and unlocked. * Condition: coalitions_list_lock must be UNLOCKED. */ kern_return_t coalition_create_internal(int type, int role, boolean_t privileged, boolean_t efficient, coalition_t *out, uint64_t *coalition_id) { kern_return_t kr; struct coalition *new_coal; uint64_t cid; if (type < 0 || type > COALITION_TYPE_MAX) { return KERN_INVALID_ARGUMENT; } new_coal = zalloc_flags(coalition_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL); new_coal->type = type; new_coal->role = role; /* initialize type-specific resources */ kr = coal_call(new_coal, init, privileged, efficient); if (kr != KERN_SUCCESS) { zfree(coalition_zone, new_coal); return kr; } /* One for caller, one for coalitions list */ coal_ref_init(new_coal, 2); new_coal->privileged = privileged ? TRUE : FALSE; new_coal->efficient = efficient ? TRUE : FALSE; #if DEVELOPMENT || DEBUG new_coal->should_notify = 1; #endif lck_mtx_init(&new_coal->lock, &coalitions_lck_grp, LCK_ATTR_NULL); lck_mtx_lock(&coalitions_list_lock); new_coal->id = cid = coalition_next_id++; smr_hash_serialized_insert(&coalition_hash, &new_coal->link, &coal_hash_traits); #if CONFIG_THREAD_GROUPS KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW), new_coal->id, new_coal->type, (new_coal->type == COALITION_TYPE_JETSAM && new_coal->j.thread_group) ? thread_group_get_id(new_coal->j.thread_group) : 0); #else KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW), new_coal->id, new_coal->type); #endif if (smr_hash_serialized_should_grow(&coalition_hash, 1, 4)) { /* grow if more more than 4 elements per 1 bucket */ smr_hash_grow_and_unlock(&coalition_hash, &coalitions_list_lock, &coal_hash_traits); } else { lck_mtx_unlock(&coalitions_list_lock); } coal_dbg("id:%llu, type:%s", cid, coal_type_str(type)); if (coalition_id != NULL) { *coalition_id = cid; } *out = new_coal; return KERN_SUCCESS; } static void coalition_free(struct smr_node *node) { struct coalition *coal; coal = __container_of(node, struct coalition, smr_node); zfree(coalition_zone, coal); } static __attribute__((noinline)) void coalition_retire(coalition_t coal) { coalition_lock(coal); coal_dbg("id:%llu type:%s active_count:%u%s", coal->id, coal_type_str(coal->type), coal->active_count, rc <= 0 ? ", will deallocate now" : ""); assert(coal->termrequested); assert(coal->terminated); assert(coal->active_count == 0); assert(coal->reaped); assert(coal->focal_task_count == 0); assert(coal->nonfocal_task_count == 0); assert(coal->game_task_count == 0); assert(coal->carplay_task_count == 0); #if CONFIG_THREAD_GROUPS KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE), coal->id, coal->type, coal->type == COALITION_TYPE_JETSAM ? coal->j.thread_group : 0); #else KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE), coal->id, coal->type); #endif coal_call(coal, dealloc); coalition_unlock(coal); lck_mtx_destroy(&coal->lock, &coalitions_lck_grp); smr_proc_task_call(&coal->smr_node, sizeof(*coal), coalition_free); } /* * coalition_retain * */ void coalition_retain(coalition_t coal) { coal_ref_retain(coal); } /* * coalition_release * Condition: coalition must be UNLOCKED. * */ void coalition_release(coalition_t coal) { if (coal_ref_release(coal) > 0) { return; } coalition_retire(coal); } /* * coalition_find_by_id * Returns: Coalition object with specified id, referenced. */ coalition_t coalition_find_by_id(uint64_t cid) { smrh_key_t key = SMRH_SCALAR_KEY(cid); if (cid == 0) { return COALITION_NULL; } return smr_hash_get(&coalition_hash, key, &coal_hash_traits); } /* * coalition_find_and_activate_by_id * Returns: Coalition object with specified id, referenced, and activated. * This is the function to use when putting a 'new' thing into a coalition, * like posix_spawn of an XPC service by launchd. * See also coalition_extend_active. */ coalition_t coalition_find_and_activate_by_id(uint64_t cid) { coalition_t coal = coalition_find_by_id(cid); if (coal == COALITION_NULL) { return COALITION_NULL; } coalition_lock(coal); if (coal->reaped || coal->terminated) { /* Too late to put something new into this coalition, it's * already on its way out the door */ coalition_unlock(coal); coalition_release(coal); return COALITION_NULL; } coal->active_count++; #if COALITION_DEBUG uint32_t rc = coal_ref_count(coal); uint32_t ac = coal->active_count; #endif coalition_unlock(coal); coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u", coal->id, coal_type_str(coal->type), rc, ac); return coal; } uint64_t coalition_id(coalition_t coal) { assert(coal != COALITION_NULL); return coal->id; } void task_coalition_ids(task_t task, uint64_t ids[COALITION_NUM_TYPES]) { int i; for (i = 0; i < COALITION_NUM_TYPES; i++) { if (task->coalition[i]) { ids[i] = task->coalition[i]->id; } else { ids[i] = 0; } } } void task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES]) { int i; memset(roles, 0, COALITION_NUM_TYPES * sizeof(roles[0])); for (i = 0; i < COALITION_NUM_TYPES; i++) { if (task->coalition[i]) { coalition_lock(task->coalition[i]); roles[i] = coal_call(task->coalition[i], get_taskrole, task); coalition_unlock(task->coalition[i]); } else { roles[i] = COALITION_TASKROLE_NONE; } } } int task_coalition_role_for_type(task_t task, int coalition_type) { coalition_t coal; int role; if (coalition_type >= COALITION_NUM_TYPES) { panic("Attempt to call task_coalition_role_for_type with invalid coalition_type: %d\n", coalition_type); } coal = task->coalition[coalition_type]; if (coal == NULL) { return COALITION_TASKROLE_NONE; } coalition_lock(coal); role = coal_call(coal, get_taskrole, task); coalition_unlock(coal); return role; } int coalition_type(coalition_t coal) { return coal->type; } boolean_t coalition_term_requested(coalition_t coal) { return coal->termrequested; } boolean_t coalition_is_terminated(coalition_t coal) { return coal->terminated; } boolean_t coalition_is_reaped(coalition_t coal) { return coal->reaped; } boolean_t coalition_is_privileged(coalition_t coal) { return coal->privileged || unrestrict_coalition_syscalls; } boolean_t task_is_in_privileged_coalition(task_t task, int type) { if (type < 0 || type > COALITION_TYPE_MAX) { return FALSE; } if (unrestrict_coalition_syscalls) { return TRUE; } if (!task->coalition[type]) { return FALSE; } return task->coalition[type]->privileged; } void task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta) { coalition_t coal; assert(task != TASK_NULL); if (gpu_ns_delta == 0) { return; } coal = task->coalition[COALITION_TYPE_RESOURCE]; assert(coal != COALITION_NULL); coalition_lock(coal); coal->r.gpu_time += gpu_ns_delta; coalition_unlock(coal); } void coalition_update_ane_stats(coalition_t coalition, uint64_t ane_mach_time, uint64_t ane_energy_nj) { assert(coalition != COALITION_NULL); os_atomic_add(&coalition->r.ane_mach_time, ane_mach_time, relaxed); os_atomic_add(&coalition->r.ane_energy_nj, ane_energy_nj, relaxed); } boolean_t task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return FALSE; } *new_count = os_atomic_add(&coal->focal_task_count, count, relaxed); assert(*new_count != UINT32_MAX); return TRUE; } uint32_t task_coalition_focal_count(task_t task) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return 0; } return coal->focal_task_count; } uint32_t task_coalition_game_mode_count(task_t task) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return 0; } return coal->game_task_count; } uint32_t task_coalition_carplay_mode_count(task_t task) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return 0; } return coal->carplay_task_count; } boolean_t task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return FALSE; } *new_count = os_atomic_add(&coal->nonfocal_task_count, count, relaxed); assert(*new_count != UINT32_MAX); return TRUE; } uint32_t task_coalition_nonfocal_count(task_t task) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return 0; } return coal->nonfocal_task_count; } bool task_coalition_adjust_game_mode_count(task_t task, int count, uint32_t *new_count) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return false; } *new_count = os_atomic_add(&coal->game_task_count, count, relaxed); assert(*new_count != UINT32_MAX); return true; } bool task_coalition_adjust_carplay_mode_count(task_t task, int count, uint32_t *new_count) { coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; if (coal == COALITION_NULL) { return false; } *new_count = os_atomic_add(&coal->carplay_task_count, count, relaxed); assert(*new_count != UINT32_MAX); return true; } #if CONFIG_THREAD_GROUPS /* Thread group lives as long as the task is holding the coalition reference */ struct thread_group * task_coalition_get_thread_group(task_t task) { coalition_t coal = task->coalition[COALITION_TYPE_JETSAM]; /* return system thread group for non-jetsam coalitions */ if (coal == COALITION_NULL) { return init_coalition[COALITION_TYPE_JETSAM]->j.thread_group; } return coal->j.thread_group; } struct thread_group * kdp_coalition_get_thread_group(coalition_t coal) { if (coal->type != COALITION_TYPE_JETSAM) { return NULL; } assert(coal->j.thread_group != NULL); return coal->j.thread_group; } /* Thread group lives as long as the coalition reference is held */ struct thread_group * coalition_get_thread_group(coalition_t coal) { if (coal->type != COALITION_TYPE_JETSAM) { return NULL; } assert(coal->j.thread_group != NULL); return coal->j.thread_group; } /* Donates the thread group reference to the coalition */ void coalition_set_thread_group(coalition_t coal, struct thread_group *tg) { assert(coal != COALITION_NULL); assert(tg != NULL); if (coal->type != COALITION_TYPE_JETSAM) { return; } struct thread_group *old_tg = coal->j.thread_group; assert(old_tg != NULL); coal->j.thread_group = tg; KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_THREAD_GROUP_SET), coal->id, coal->type, thread_group_get_id(tg)); thread_group_release(old_tg); } void task_coalition_thread_group_focal_update(task_t task) { assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL); thread_group_flags_update_lock(); uint32_t focal_count = task_coalition_focal_count(task); if (focal_count) { thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP); } else { thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP); } thread_group_flags_update_unlock(); } void task_coalition_thread_group_game_mode_update(task_t task) { assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL); thread_group_flags_update_lock(); if (task_coalition_game_mode_count(task)) { thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE); } else { thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE); } thread_group_flags_update_unlock(); } void task_coalition_thread_group_carplay_mode_update(task_t task) { assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL); thread_group_flags_update_lock(); if (task_coalition_carplay_mode_count(task)) { thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_CARPLAY_MODE); } else { thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_CARPLAY_MODE); } thread_group_flags_update_unlock(); } void task_coalition_thread_group_application_set(task_t task) { /* * Setting the "Application" flag on the thread group is a one way transition. * Once a coalition has a single task with an application apptype, the * thread group associated with the coalition is tagged as Application. */ thread_group_flags_update_lock(); thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_APPLICATION); thread_group_flags_update_unlock(); } #endif /* CONFIG_THREAD_GROUPS */ static void coalition_for_each_task_locked(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block) { assert(coal != COALITION_NULL); coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u", coal, coal->id, coal_type_str(coal->type), coal->active_count); __assert_only task_t selected_task; selected_task = coal_call(coal, iterate_tasks, block); assert(selected_task == TASK_NULL); } void coalition_for_each_task(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block) { assert(coal != COALITION_NULL); coalition_lock(coal); coalition_for_each_task_locked(coal, block); coalition_unlock(coal); } static task_t coalition_select_task(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block) { assert(coal != COALITION_NULL); coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u", coal, coal->id, coal_type_str(coal->type), coal->active_count); coalition_lock(coal); task_t selected_task = coal_call(coal, iterate_tasks, block); if (selected_task != TASK_NULL) { task_reference(selected_task); } coalition_unlock(coal); return selected_task; } void coalition_remove_active(coalition_t coal) { coalition_lock(coal); assert(!coalition_is_reaped(coal)); assert(coal->active_count > 0); coal->active_count--; boolean_t do_notify = FALSE; uint64_t notify_id = 0; uint32_t notify_flags = 0; if (coal->termrequested && coal->active_count == 0) { /* We only notify once, when active_count reaches zero. * We just decremented, so if it reached zero, we mustn't have * notified already. */ assert(!coal->terminated); coal->terminated = TRUE; assert(!coal->notified); coal->notified = TRUE; #if DEVELOPMENT || DEBUG do_notify = coal->should_notify; #else do_notify = TRUE; #endif notify_id = coal->id; notify_flags = 0; } #if COALITION_DEBUG uint64_t cid = coal->id; uint32_t rc = coal_ref_count(coal); int ac = coal->active_count; int ct = coal->type; #endif coalition_unlock(coal); coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u,%s", cid, coal_type_str(ct), rc, ac, do_notify ? " NOTIFY" : " "); if (do_notify) { coalition_notify_user(notify_id, notify_flags); } } /* Used for kernel_task, launchd, launchd's early boot tasks... */ kern_return_t coalitions_adopt_init_task(task_t task) { kern_return_t kr; kr = coalitions_adopt_task(init_coalition, task); if (kr != KERN_SUCCESS) { panic("failed to adopt task %p into default coalition: %d", task, kr); } return kr; } /* Used for forked corpses. */ kern_return_t coalitions_adopt_corpse_task(task_t task) { kern_return_t kr; kr = coalitions_adopt_task(corpse_coalition, task); if (kr != KERN_SUCCESS) { panic("failed to adopt task %p into corpse coalition: %d", task, kr); } return kr; } /* * coalition_adopt_task_internal * Condition: Coalition must be referenced and unlocked. Will fail if coalition * is already terminated. */ static kern_return_t coalition_adopt_task_internal(coalition_t coal, task_t task) { kern_return_t kr; if (task->coalition[coal->type]) { return KERN_ALREADY_IN_SET; } coalition_lock(coal); if (coal->reaped || coal->terminated) { coalition_unlock(coal); return KERN_TERMINATED; } kr = coal_call(coal, adopt_task, task); if (kr != KERN_SUCCESS) { goto out_unlock; } coal->active_count++; coal_ref_retain(coal); task->coalition[coal->type] = coal; out_unlock: #if COALITION_DEBUG (void)coal; /* need expression after label */ uint64_t cid = coal->id; uint32_t rc = coal_ref_count(coal); uint32_t ct = coal->type; #endif if (get_task_uniqueid(task) != UINT64_MAX) { /* On 32-bit targets, uniqueid will get truncated to 32 bits */ KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_ADOPT), coal->id, get_task_uniqueid(task)); } coalition_unlock(coal); coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d", task_pid(task), cid, coal_type_str(ct), rc, kr); return kr; } static kern_return_t coalition_remove_task_internal(task_t task, int type) { kern_return_t kr; coalition_t coal = task->coalition[type]; if (!coal) { return KERN_SUCCESS; } assert(coal->type == (uint32_t)type); coalition_lock(coal); kr = coal_call(coal, remove_task, task); #if COALITION_DEBUG uint64_t cid = coal->id; uint32_t rc = coal_ref_count(coal); int ac = coal->active_count; int ct = coal->type; #endif KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_REMOVE), coal->id, get_task_uniqueid(task)); coalition_unlock(coal); coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d", cid, coal_type_str(ct), rc, ac, kr); coalition_remove_active(coal); return kr; } /* * coalitions_adopt_task * Condition: All coalitions must be referenced and unlocked. * Will fail if any coalition is already terminated. */ kern_return_t coalitions_adopt_task(coalition_t *coals, task_t task) { int i; kern_return_t kr; if (!coals || coals[COALITION_TYPE_RESOURCE] == COALITION_NULL) { return KERN_INVALID_ARGUMENT; } /* verify that the incoming coalitions are what they say they are */ for (i = 0; i < COALITION_NUM_TYPES; i++) { if (coals[i] && coals[i]->type != (uint32_t)i) { return KERN_INVALID_ARGUMENT; } } for (i = 0; i < COALITION_NUM_TYPES; i++) { kr = KERN_SUCCESS; if (coals[i]) { kr = coalition_adopt_task_internal(coals[i], task); } if (kr != KERN_SUCCESS) { /* dis-associate any coalitions that just adopted this task */ while (--i >= 0) { if (task->coalition[i]) { coalition_remove_task_internal(task, i); } } break; } } return kr; } /* * coalitions_remove_task * Condition: task must be referenced and UNLOCKED; all task's coalitions must be UNLOCKED */ kern_return_t coalitions_remove_task(task_t task) { kern_return_t kr; int i; task_lock(task); if (!task_is_coalition_member(task)) { task_unlock(task); return KERN_SUCCESS; } task_clear_coalition_member(task); task_unlock(task); for (i = 0; i < COALITION_NUM_TYPES; i++) { kr = coalition_remove_task_internal(task, i); assert(kr == KERN_SUCCESS); } return kr; } /* * task_release_coalitions * helper function to release references to all coalitions in which * 'task' is a member. */ void task_release_coalitions(task_t task) { int i; for (i = 0; i < COALITION_NUM_TYPES; i++) { if (task->coalition[i]) { coalition_release(task->coalition[i]); } else if (i == COALITION_TYPE_RESOURCE) { panic("deallocating task %p was not a member of a resource coalition", task); } } } /* * coalitions_set_roles * for each type of coalition, if the task is a member of a coalition of * that type (given in the coalitions parameter) then set the role of * the task within that that coalition. */ kern_return_t coalitions_set_roles(coalition_t coalitions[COALITION_NUM_TYPES], task_t task, int roles[COALITION_NUM_TYPES]) { kern_return_t kr = KERN_SUCCESS; int i; for (i = 0; i < COALITION_NUM_TYPES; i++) { if (!coalitions[i]) { continue; } coalition_lock(coalitions[i]); kr = coal_call(coalitions[i], set_taskrole, task, roles[i]); coalition_unlock(coalitions[i]); assert(kr == KERN_SUCCESS); } return kr; } /* * coalition_terminate_internal * Condition: Coalition must be referenced and UNLOCKED. */ kern_return_t coalition_request_terminate_internal(coalition_t coal) { assert(coal->type >= 0 && coal->type <= COALITION_TYPE_MAX); if (coal == init_coalition[coal->type]) { return KERN_DEFAULT_SET; } coalition_lock(coal); if (coal->reaped) { coalition_unlock(coal); return KERN_INVALID_NAME; } if (coal->terminated || coal->termrequested) { coalition_unlock(coal); return KERN_TERMINATED; } coal->termrequested = TRUE; boolean_t do_notify = FALSE; uint64_t note_id = 0; uint32_t note_flags = 0; if (coal->active_count == 0) { /* * We only notify once, when active_count reaches zero. * We just set termrequested to zero. If the active count * was already at zero (tasks died before we could request * a termination notification), we should notify. */ assert(!coal->terminated); coal->terminated = TRUE; assert(!coal->notified); coal->notified = TRUE; #if DEVELOPMENT || DEBUG do_notify = coal->should_notify; #else do_notify = TRUE; #endif note_id = coal->id; note_flags = 0; } coalition_unlock(coal); if (do_notify) { coalition_notify_user(note_id, note_flags); } return KERN_SUCCESS; } /* * coalition_reap_internal * Condition: Coalition must be referenced and UNLOCKED. */ kern_return_t coalition_reap_internal(coalition_t coal) { assert(coal->type <= COALITION_TYPE_MAX); if (coal == init_coalition[coal->type]) { return KERN_DEFAULT_SET; } coalition_lock(coal); if (coal->reaped) { coalition_unlock(coal); return KERN_TERMINATED; } if (!coal->terminated) { coalition_unlock(coal); return KERN_FAILURE; } assert(coal->termrequested); if (coal->active_count > 0) { coalition_unlock(coal); return KERN_FAILURE; } coal->reaped = TRUE; coalition_unlock(coal); lck_mtx_lock(&coalitions_list_lock); smr_hash_serialized_remove(&coalition_hash, &coal->link, &coal_hash_traits); if (smr_hash_serialized_should_shrink(&coalition_hash, COALITION_HASH_SIZE_MIN, 2, 1)) { /* shrink if more than 2 buckets per 1 element */ smr_hash_shrink_and_unlock(&coalition_hash, &coalitions_list_lock, &coal_hash_traits); } else { lck_mtx_unlock(&coalitions_list_lock); } /* Release the list's reference and launchd's reference. */ coal_ref_release_live(coal); coalition_release(coal); return KERN_SUCCESS; } #if DEVELOPMENT || DEBUG int coalition_should_notify(coalition_t coal) { int should; if (!coal) { return -1; } coalition_lock(coal); should = coal->should_notify; coalition_unlock(coal); return should; } void coalition_set_notify(coalition_t coal, int notify) { if (!coal) { return; } coalition_lock(coal); coal->should_notify = !!notify; coalition_unlock(coal); } #endif void coalitions_init(void) { kern_return_t kr; int i; const struct coalition_type *ctype; smr_hash_init(&coalition_hash, COALITION_HASH_SIZE_MIN); init_task_ledgers(); init_coalition_ledgers(); for (i = 0, ctype = &s_coalition_types[0]; i < COALITION_NUM_TYPES; ctype++, i++) { /* verify the entry in the global coalition types array */ if (ctype->type != i || !ctype->init || !ctype->dealloc || !ctype->adopt_task || !ctype->remove_task) { panic("%s: Malformed coalition type %s(%d) in slot for type:%s(%d)", __func__, coal_type_str(ctype->type), ctype->type, coal_type_str(i), i); } if (!ctype->has_default) { continue; } kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, TRUE, FALSE, &init_coalition[ctype->type], NULL); if (kr != KERN_SUCCESS) { panic("%s: could not create init %s coalition: kr:%d", __func__, coal_type_str(i), kr); } if (i == COALITION_TYPE_RESOURCE) { assert(COALITION_ID_KERNEL == init_coalition[ctype->type]->id); } kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, FALSE, FALSE, &corpse_coalition[ctype->type], NULL); if (kr != KERN_SUCCESS) { panic("%s: could not create corpse %s coalition: kr:%d", __func__, coal_type_str(i), kr); } } /* "Leak" our reference to the global object */ } /* * BSD Kernel interface functions * */ static void coalition_fill_procinfo(struct coalition *coal, struct procinfo_coalinfo *coalinfo) { coalinfo->coalition_id = coal->id; coalinfo->coalition_type = coal->type; coalinfo->coalition_tasks = coalition_get_task_count(coal); } size_t coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz) { size_t ncoals = 0; struct coalition *coal; lck_mtx_lock(&coalitions_list_lock); smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) { if (!coal->reaped && (type < 0 || type == (int)coal->type)) { if (coal_list && ncoals < list_sz) { coalition_fill_procinfo(coal, &coal_list[ncoals]); } ++ncoals; } } lck_mtx_unlock(&coalitions_list_lock); return ncoals; } /* * Return the coaltion of the given type to which the task belongs. */ coalition_t task_get_coalition(task_t task, int coal_type) { coalition_t c; if (task == NULL || coal_type > COALITION_TYPE_MAX) { return COALITION_NULL; } c = task->coalition[coal_type]; assert(c == COALITION_NULL || (int)c->type == coal_type); return c; } /* * Report if the given task is the leader of the given jetsam coalition. */ boolean_t coalition_is_leader(task_t task, coalition_t coal) { boolean_t ret = FALSE; if (coal != COALITION_NULL) { coalition_lock(coal); ret = (coal->type == COALITION_TYPE_JETSAM && coal->j.leader == task); coalition_unlock(coal); } return ret; } kern_return_t coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type) { coalition_t coal; int i = 0; smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) { if (coal == NULL || !ml_validate_nofault((vm_offset_t)coal, sizeof(struct coalition))) { return KERN_FAILURE; } if (coalition_type == coal->type) { callout(arg, i++, coal); } } return KERN_SUCCESS; } task_t kdp_coalition_get_leader(coalition_t coal) { if (!coal) { return TASK_NULL; } if (coal->type == COALITION_TYPE_JETSAM) { return coal->j.leader; } return TASK_NULL; } task_t coalition_get_leader(coalition_t coal) { task_t leader = TASK_NULL; if (!coal) { return TASK_NULL; } coalition_lock(coal); if (coal->type != COALITION_TYPE_JETSAM) { goto out_unlock; } leader = coal->j.leader; if (leader != TASK_NULL) { task_reference(leader); } out_unlock: coalition_unlock(coal); return leader; } int coalition_get_task_count(coalition_t coal) { int ntasks = 0; struct queue_entry *qe; if (!coal) { return 0; } coalition_lock(coal); switch (coal->type) { case COALITION_TYPE_RESOURCE: qe_foreach(qe, &coal->r.tasks) ntasks++; break; case COALITION_TYPE_JETSAM: if (coal->j.leader) { ntasks++; } qe_foreach(qe, &coal->j.other) ntasks++; qe_foreach(qe, &coal->j.extensions) ntasks++; qe_foreach(qe, &coal->j.services) ntasks++; break; default: break; } coalition_unlock(coal); return ntasks; } static uint64_t i_get_list_footprint(queue_t list, int type, int *ntasks) { task_t task; uint64_t bytes = 0; qe_foreach_element(task, list, task_coalition[type]) { bytes += get_task_phys_footprint(task); coal_dbg(" [%d] task_pid:%d, type:%d, footprint:%lld", *ntasks, task_pid(task), type, bytes); *ntasks += 1; } return bytes; } uint64_t coalition_get_page_count(coalition_t coal, int *ntasks) { uint64_t bytes = 0; int num_tasks = 0; if (ntasks) { *ntasks = 0; } if (!coal) { return bytes; } coalition_lock(coal); switch (coal->type) { case COALITION_TYPE_RESOURCE: bytes += i_get_list_footprint(&coal->r.tasks, COALITION_TYPE_RESOURCE, &num_tasks); break; case COALITION_TYPE_JETSAM: if (coal->j.leader) { bytes += get_task_phys_footprint(coal->j.leader); num_tasks = 1; } bytes += i_get_list_footprint(&coal->j.extensions, COALITION_TYPE_JETSAM, &num_tasks); bytes += i_get_list_footprint(&coal->j.services, COALITION_TYPE_JETSAM, &num_tasks); bytes += i_get_list_footprint(&coal->j.other, COALITION_TYPE_JETSAM, &num_tasks); break; default: break; } coalition_unlock(coal); if (ntasks) { *ntasks = num_tasks; } return bytes / PAGE_SIZE_64; } struct coal_sort_s { int pid; int usr_order; uint64_t bytes; }; /* * return < 0 for a < b * 0 for a == b * > 0 for a > b */ typedef int (*cmpfunc_t)(const void *a, const void *b); extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp); static int dflt_cmp(const void *a, const void *b) { const struct coal_sort_s *csA = (const struct coal_sort_s *)a; const struct coal_sort_s *csB = (const struct coal_sort_s *)b; /* * if both A and B are equal, use a memory descending sort */ if (csA->usr_order == csB->usr_order) { return (int)((int64_t)csB->bytes - (int64_t)csA->bytes); } /* otherwise, return the relationship between user specified orders */ return csA->usr_order - csB->usr_order; } static int mem_asc_cmp(const void *a, const void *b) { const struct coal_sort_s *csA = (const struct coal_sort_s *)a; const struct coal_sort_s *csB = (const struct coal_sort_s *)b; return (int)((int64_t)csA->bytes - (int64_t)csB->bytes); } static int mem_dec_cmp(const void *a, const void *b) { const struct coal_sort_s *csA = (const struct coal_sort_s *)a; const struct coal_sort_s *csB = (const struct coal_sort_s *)b; return (int)((int64_t)csB->bytes - (int64_t)csA->bytes); } static int usr_asc_cmp(const void *a, const void *b) { const struct coal_sort_s *csA = (const struct coal_sort_s *)a; const struct coal_sort_s *csB = (const struct coal_sort_s *)b; return csA->usr_order - csB->usr_order; } static int usr_dec_cmp(const void *a, const void *b) { const struct coal_sort_s *csA = (const struct coal_sort_s *)a; const struct coal_sort_s *csB = (const struct coal_sort_s *)b; return csB->usr_order - csA->usr_order; } /* avoid dynamic allocation in this path */ #define MAX_SORTED_PIDS 80 static int coalition_get_sort_list(coalition_t coal, int sort_order, queue_t list, struct coal_sort_s *sort_array, int array_sz) { int ntasks = 0; task_t task; assert(sort_array != NULL); if (array_sz <= 0) { return 0; } if (!list) { /* * this function will only be called with a NULL * list for JETSAM-type coalitions, and is intended * to investigate the leader process */ if (coal->type != COALITION_TYPE_JETSAM || coal->j.leader == TASK_NULL) { return 0; } sort_array[0].pid = task_pid(coal->j.leader); switch (sort_order) { case COALITION_SORT_DEFAULT: sort_array[0].usr_order = 0; OS_FALLTHROUGH; case COALITION_SORT_MEM_ASC: case COALITION_SORT_MEM_DEC: sort_array[0].bytes = get_task_phys_footprint(coal->j.leader); break; case COALITION_SORT_USER_ASC: case COALITION_SORT_USER_DEC: sort_array[0].usr_order = 0; break; default: break; } return 1; } qe_foreach_element(task, list, task_coalition[coal->type]) { if (ntasks >= array_sz) { printf("WARNING: more than %d pids in coalition %llu\n", MAX_SORTED_PIDS, coal->id); break; } sort_array[ntasks].pid = task_pid(task); switch (sort_order) { case COALITION_SORT_DEFAULT: sort_array[ntasks].usr_order = 0; OS_FALLTHROUGH; case COALITION_SORT_MEM_ASC: case COALITION_SORT_MEM_DEC: sort_array[ntasks].bytes = get_task_phys_footprint(task); break; case COALITION_SORT_USER_ASC: case COALITION_SORT_USER_DEC: sort_array[ntasks].usr_order = 0; break; default: break; } ntasks++; } return ntasks; } int coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order, int *pid_list, int list_sz) { struct i_jetsam_coalition *cj; int ntasks = 0; cmpfunc_t cmp_func = NULL; struct coal_sort_s sort_array[MAX_SORTED_PIDS] = { {0, 0, 0} }; /* keep to < 2k */ if (!coal || !(rolemask & COALITION_ROLEMASK_ALLROLES) || !pid_list || list_sz < 1) { coal_dbg("Invalid parameters: coal:%p, type:%d, rolemask:0x%x, " "pid_list:%p, list_sz:%d", coal, coal ? coal->type : -1, rolemask, pid_list, list_sz); return -EINVAL; } switch (sort_order) { case COALITION_SORT_NOSORT: cmp_func = NULL; break; case COALITION_SORT_DEFAULT: cmp_func = dflt_cmp; break; case COALITION_SORT_MEM_ASC: cmp_func = mem_asc_cmp; break; case COALITION_SORT_MEM_DEC: cmp_func = mem_dec_cmp; break; case COALITION_SORT_USER_ASC: cmp_func = usr_asc_cmp; break; case COALITION_SORT_USER_DEC: cmp_func = usr_dec_cmp; break; default: return -ENOTSUP; } coalition_lock(coal); if (coal->type == COALITION_TYPE_RESOURCE) { ntasks += coalition_get_sort_list(coal, sort_order, &coal->r.tasks, sort_array, MAX_SORTED_PIDS); goto unlock_coal; } cj = &coal->j; if (rolemask & COALITION_ROLEMASK_UNDEF) { ntasks += coalition_get_sort_list(coal, sort_order, &cj->other, sort_array + ntasks, MAX_SORTED_PIDS - ntasks); } if (rolemask & COALITION_ROLEMASK_XPC) { ntasks += coalition_get_sort_list(coal, sort_order, &cj->services, sort_array + ntasks, MAX_SORTED_PIDS - ntasks); } if (rolemask & COALITION_ROLEMASK_EXT) { ntasks += coalition_get_sort_list(coal, sort_order, &cj->extensions, sort_array + ntasks, MAX_SORTED_PIDS - ntasks); } if (rolemask & COALITION_ROLEMASK_LEADER) { ntasks += coalition_get_sort_list(coal, sort_order, NULL, sort_array + ntasks, MAX_SORTED_PIDS - ntasks); } unlock_coal: coalition_unlock(coal); /* sort based on the chosen criterion (no sense sorting 1 item) */ if (cmp_func && ntasks > 1) { qsort(sort_array, ntasks, sizeof(struct coal_sort_s), cmp_func); } for (int i = 0; i < ntasks; i++) { if (i >= list_sz) { break; } coal_dbg(" [%d] PID:%d, footprint:%lld, usr_order:%d", i, sort_array[i].pid, sort_array[i].bytes, sort_array[i].usr_order); pid_list[i] = sort_array[i].pid; } return ntasks; } void coalition_mark_swappable(coalition_t coal) { struct i_jetsam_coalition *cj = NULL; coalition_lock(coal); assert(coal && coal->type == COALITION_TYPE_JETSAM); cj = &coal->j; cj->swap_enabled = true; i_coal_jetsam_iterate_tasks(coal, ^bool (task_t task) { vm_task_set_selfdonate_pages(task, true); return false; }); coalition_unlock(coal); } bool coalition_is_swappable(coalition_t coal) { struct i_jetsam_coalition *cj = NULL; coalition_lock(coal); assert(coal && coal->type == COALITION_TYPE_JETSAM); cj = &coal->j; bool enabled = cj->swap_enabled; coalition_unlock(coal); return enabled; } /* * Coalition policy functions */ static uintptr_t crequested(coalition_t coal) { static_assert(sizeof(struct coalition_requested_policy) == sizeof(uint64_t), "size invariant violated"); uintptr_t* raw = (uintptr_t*)&coal->j.c_requested_policy; return raw[0]; } static uintptr_t ceffective(coalition_t coal) { static_assert(sizeof(struct coalition_requested_policy) == sizeof(uint64_t), "size invariant violated"); uintptr_t* raw = (uintptr_t*)&coal->j.c_effective_policy; return raw[0]; } static uint32_t cpending(coalition_pend_token_t pend_token) { static_assert(sizeof(struct coalition_pend_token) == sizeof(uint32_t), "size invariant violated"); return *(uint32_t*)(void*)(pend_token); } static void jetsam_coalition_policy_update_locked(coalition_t coal, coalition_pend_token_t pend_token) { KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_UPDATE, TASK_POLICY_COALITION) | DBG_FUNC_START), coalition_id(coal), ceffective(coal), 0, 0, 0); struct coalition_requested_policy requested = coal->j.c_requested_policy; struct coalition_effective_policy next = {}; /* Update darwinbg state */ next.cep_darwinbg = requested.crp_darwinbg; /* TODO: should termrequested cause darwinbg to be unset */ struct coalition_effective_policy prev = coal->j.c_effective_policy; coal->j.c_effective_policy = next; /* */ if (prev.cep_darwinbg != next.cep_darwinbg) { pend_token->cpt_update_j_coal_tasks = 1; } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_UPDATE, TASK_POLICY_COALITION)) | DBG_FUNC_END, coalition_id(coal), ceffective(coal), 0, 0, 0); } /* * Returns task reference or TASK_NULL if none left. * Called without coalition lock held. * Called with global coalition_policy_lock held. */ static task_t coalition_find_next_task_with_pending_work(coalition_t coal) { return coalition_select_task(coal, ^bool (task_t task){ if (task->pended_coalition_changes.tpt_value != 0) { return true; } return false; }); } /* * Called with coalition unlocked to do things that can't be done * while holding the coalition lock. * Called with global coalition_policy_lock held. */ static void coalition_policy_update_complete_unlocked( coalition_t coal, coalition_pend_token_t coalition_pend_token) { /* * Note that because we dropped the coalition lock, a task may have * exited since we set the pending bit, so we can't assert that * no task exits with a change pending. */ if (coalition_pend_token->cpt_update_j_coal_tasks) { task_t task; while ((task = coalition_find_next_task_with_pending_work(coal)) != TASK_NULL) { task_policy_update_complete_unlocked(task, &task->pended_coalition_changes); task->pended_coalition_changes.tpt_value = 0; task_deallocate(task); } } if (coalition_pend_token->cpt_update_timers) { ml_timer_evaluate(); } } /* * This lock covers the coalition_pend_token fields * in all tasks, so that we don't have to have stack storage * for all of them. We could also use a gate or busy bit * on each coalition, but this is less complex. */ LCK_MTX_DECLARE(coalition_policy_lock, &coalitions_lck_grp); kern_return_t jetsam_coalition_set_policy(coalition_t coal, int flavor, int value) { struct coalition_pend_token pend_token = {}; coalition_pend_token_t pend_token_ref = &pend_token; assert(coalition_type(coal) == COALITION_TYPE_JETSAM); lck_mtx_lock(&coalition_policy_lock); coalition_lock(coal); struct coalition_requested_policy requested = coal->j.c_requested_policy; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(flavor, TASK_POLICY_COALITION)) | DBG_FUNC_START, coalition_id(coal), crequested(coal), 0, value, 0); switch (flavor) { case TASK_POLICY_DARWIN_BG: requested.crp_darwinbg = value; break; default: panic("unknown coalition policy: %d %d", flavor, value); break; } coal->j.c_requested_policy = requested; jetsam_coalition_policy_update_locked(coal, &pend_token); if (pend_token.cpt_update_j_coal_tasks) { coalition_for_each_task_locked(coal, ^bool (task_t task) { coalition_policy_update_task(task, pend_token_ref); return false; }); } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(flavor, TASK_POLICY_COALITION)) | DBG_FUNC_END, coalition_id(coal), crequested(coal), 0, cpending(&pend_token), 0); coalition_unlock(coal); coalition_policy_update_complete_unlocked(coal, &pend_token); lck_mtx_unlock(&coalition_policy_lock); return KERN_SUCCESS; } kern_return_t jetsam_coalition_get_policy(coalition_t coal, int flavor, int *value) { lck_mtx_lock(&coalition_policy_lock); coalition_lock(coal); struct coalition_requested_policy requested = coal->j.c_requested_policy; switch (flavor) { case TASK_POLICY_DARWIN_BG: *value = requested.crp_darwinbg; break; default: panic("unknown coalition policy: %d", flavor); break; } coalition_unlock(coal); lck_mtx_unlock(&coalition_policy_lock); return KERN_SUCCESS; } bool task_get_effective_jetsam_coalition_policy(task_t task, int flavor) { coalition_t coal = task->coalition[COALITION_TYPE_JETSAM]; if (coal == NULL) { return false; } struct coalition_effective_policy effective = coal->j.c_effective_policy; switch (flavor) { case TASK_POLICY_DARWIN_BG: return effective.cep_darwinbg; default: panic("unknown coalition policy: %d", flavor); break; } return false; }