1 /*
2 * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/kern_types.h>
30 #include <mach/mach_types.h>
31 #include <mach/boolean.h>
32
33 #include <kern/coalition.h>
34 #include <kern/exc_resource.h>
35 #include <kern/host.h>
36 #include <kern/ledger.h>
37 #include <kern/mach_param.h> /* for TASK_CHUNK */
38 #include <kern/monotonic.h>
39 #include <kern/policy_internal.h>
40 #include <kern/task.h>
41 #include <kern/smr_hash.h>
42 #include <kern/thread_group.h>
43 #include <kern/zalloc.h>
44 #include <kern/policy_internal.h>
45
46 #include <vm/vm_pageout.h>
47
48 #include <libkern/OSAtomic.h>
49
50 #include <mach/coalition_notification_server.h>
51 #include <mach/host_priv.h>
52 #include <mach/host_special_ports.h>
53
54 #include <os/log.h>
55
56 #include <sys/errno.h>
57
58 /*
59 * BSD interface functions
60 */
61 size_t coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz);
62 boolean_t coalition_is_leader(task_t task, coalition_t coal);
63 task_t coalition_get_leader(coalition_t coal);
64 int coalition_get_task_count(coalition_t coal);
65 uint64_t coalition_get_page_count(coalition_t coal, int *ntasks);
66 int coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order,
67 int *pid_list, int list_sz);
68
69 /* defined in task.c */
70 extern ledger_template_t task_ledger_template;
71
72 /*
73 * Templates; task template is copied due to potential allocation limits on
74 * task ledgers.
75 */
76 ledger_template_t coalition_task_ledger_template = NULL;
77 ledger_template_t coalition_ledger_template = NULL;
78
79 extern int proc_selfpid(void);
80 /*
81 * Coalition zone needs limits. We expect there will be as many coalitions as
82 * tasks (same order of magnitude), so use the task zone's limits.
83 * */
84 #define CONFIG_COALITION_MAX CONFIG_TASK_MAX
85 #define COALITION_CHUNK TASK_CHUNK
86
87 #if DEBUG || DEVELOPMENT
88 TUNABLE_WRITEABLE(int, unrestrict_coalition_syscalls, "unrestrict_coalition_syscalls", 0);
89 #else
90 #define unrestrict_coalition_syscalls false
91 #endif
92
93 static LCK_GRP_DECLARE(coalitions_lck_grp, "coalition");
94
95 /* coalitions_list_lock protects coalition_hash, next_coalition_id. */
96 static LCK_MTX_DECLARE(coalitions_list_lock, &coalitions_lck_grp);
97 static uint64_t coalition_next_id = 1;
98 static struct smr_hash coalition_hash;
99
100 coalition_t init_coalition[COALITION_NUM_TYPES];
101 coalition_t corpse_coalition[COALITION_NUM_TYPES];
102
103 static const char *
coal_type_str(int type)104 coal_type_str(int type)
105 {
106 switch (type) {
107 case COALITION_TYPE_RESOURCE:
108 return "RESOURCE";
109 case COALITION_TYPE_JETSAM:
110 return "JETSAM";
111 default:
112 return "<unknown>";
113 }
114 }
115
116 struct coalition_type {
117 int type;
118 int has_default;
119 /*
120 * init
121 * pre-condition: coalition just allocated (unlocked), unreferenced,
122 * type field set
123 */
124 kern_return_t (*init)(coalition_t coal, boolean_t privileged, boolean_t efficient);
125
126 /*
127 * dealloc
128 * pre-condition: coalition unlocked
129 * pre-condition: coalition refcount=0, active_count=0,
130 * termrequested=1, terminated=1, reaped=1
131 */
132 void (*dealloc)(coalition_t coal);
133
134 /*
135 * adopt_task
136 * pre-condition: coalition locked
137 * pre-condition: coalition !repead and !terminated
138 */
139 kern_return_t (*adopt_task)(coalition_t coal, task_t task);
140
141 /*
142 * remove_task
143 * pre-condition: coalition locked
144 * pre-condition: task has been removed from coalition's task list
145 */
146 kern_return_t (*remove_task)(coalition_t coal, task_t task);
147
148 /*
149 * set_taskrole
150 * pre-condition: coalition locked
151 * pre-condition: task added to coalition's task list,
152 * active_count >= 1 (at least the given task is active)
153 */
154 kern_return_t (*set_taskrole)(coalition_t coal, task_t task, int role);
155
156 /*
157 * get_taskrole
158 * pre-condition: coalition locked
159 * pre-condition: task added to coalition's task list,
160 * active_count >= 1 (at least the given task is active)
161 */
162 int (*get_taskrole)(coalition_t coal, task_t task);
163
164 /*
165 * iterate_tasks
166 * pre-condition: coalition locked
167 * if the callback returns true, stop iteration and return the task
168 */
169 task_t (*iterate_tasks)(coalition_t coal, coalition_for_each_task_block_t);
170 };
171
172 /*
173 * COALITION_TYPE_RESOURCE
174 */
175
176 static kern_return_t i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient);
177 static void i_coal_resource_dealloc(coalition_t coal);
178 static kern_return_t i_coal_resource_adopt_task(coalition_t coal, task_t task);
179 static kern_return_t i_coal_resource_remove_task(coalition_t coal, task_t task);
180 static kern_return_t i_coal_resource_set_taskrole(coalition_t coal,
181 task_t task, int role);
182 static int i_coal_resource_get_taskrole(coalition_t coal, task_t task);
183 static task_t i_coal_resource_iterate_tasks(coalition_t coal,
184 coalition_for_each_task_block_t block);
185
186 /*
187 * Ensure COALITION_NUM_THREAD_QOS_TYPES defined in mach/coalition.h still
188 * matches THREAD_QOS_LAST defined in mach/thread_policy.h
189 */
190 static_assert(COALITION_NUM_THREAD_QOS_TYPES == THREAD_QOS_LAST);
191
192 struct i_resource_coalition {
193 /*
194 * This keeps track of resource utilization of tasks that are no longer active
195 * in the coalition and is updated when a task is removed from the coalition.
196 */
197 ledger_t ledger;
198 uint64_t bytesread;
199 uint64_t byteswritten;
200 uint64_t energy;
201 uint64_t gpu_time;
202 uint64_t logical_immediate_writes;
203 uint64_t logical_deferred_writes;
204 uint64_t logical_invalidated_writes;
205 uint64_t logical_metadata_writes;
206 uint64_t logical_immediate_writes_to_external;
207 uint64_t logical_deferred_writes_to_external;
208 uint64_t logical_invalidated_writes_to_external;
209 uint64_t logical_metadata_writes_to_external;
210 uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per effective QoS class */
211 uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per requested QoS class */
212 uint64_t cpu_instructions;
213 uint64_t cpu_cycles;
214 uint64_t ane_mach_time;
215 uint64_t ane_energy_nj;
216
217 _Atomic uint64_t gpu_energy_nj;
218 _Atomic uint64_t gpu_energy_nj_billed_to_me;
219 _Atomic uint64_t gpu_energy_nj_billed_to_others;
220
221 struct recount_coalition co_recount;
222
223 uint64_t task_count; /* tasks that have started in this coalition */
224 uint64_t dead_task_count; /* tasks that have exited in this coalition;
225 * subtract from task_count to get count
226 * of "active" tasks */
227 /*
228 * Count the length of time this coalition had at least one active task.
229 * This can be a 'denominator' to turn e.g. cpu_time to %cpu.
230 * */
231 uint64_t last_became_nonempty_time;
232 uint64_t time_nonempty;
233
234 queue_head_t tasks; /* List of active tasks in the coalition */
235 /*
236 * This ledger is used for triggering resource exception. For the tracked resources, this is updated
237 * when the member tasks' resource usage changes.
238 */
239 ledger_t resource_monitor_ledger;
240 #if CONFIG_PHYS_WRITE_ACCT
241 uint64_t fs_metadata_writes;
242 #endif /* CONFIG_PHYS_WRITE_ACCT */
243 };
244
245 /*
246 * COALITION_TYPE_JETSAM
247 */
248
249 static kern_return_t i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient);
250 static void i_coal_jetsam_dealloc(coalition_t coal);
251 static kern_return_t i_coal_jetsam_adopt_task(coalition_t coal, task_t task);
252 static kern_return_t i_coal_jetsam_remove_task(coalition_t coal, task_t task);
253 static kern_return_t i_coal_jetsam_set_taskrole(coalition_t coal,
254 task_t task, int role);
255 int i_coal_jetsam_get_taskrole(coalition_t coal, task_t task);
256 static task_t i_coal_jetsam_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block);
257
258 struct i_jetsam_coalition {
259 task_t leader;
260 queue_head_t extensions;
261 queue_head_t services;
262 queue_head_t other;
263 struct thread_group *thread_group;
264 bool swap_enabled;
265
266 struct coalition_requested_policy {
267 uint64_t crp_darwinbg :1, /* marked as darwinbg via setpriority */
268 crp_reserved :63;
269 } c_requested_policy;
270 struct coalition_effective_policy {
271 uint64_t cep_darwinbg :1, /* marked as darwinbg via setpriority */
272 cep_reserved :63;
273 } c_effective_policy;
274 };
275
276
277 /*
278 * main coalition structure
279 */
280 struct coalition {
281 uint64_t id; /* monotonically increasing */
282 uint32_t type;
283 uint32_t role; /* default task role (background, adaptive, interactive, etc) */
284 os_ref_atomic_t ref_count; /* Number of references to the memory containing this struct */
285 uint32_t active_count; /* Number of members of (tasks in) the
286 * coalition, plus vouchers referring
287 * to the coalition */
288 uint32_t focal_task_count; /* Number of TASK_FOREGROUND_APPLICATION tasks in the coalition */
289 uint32_t nonfocal_task_count; /* Number of TASK_BACKGROUND_APPLICATION tasks in the coalition */
290 uint32_t game_task_count; /* Number of GAME_MODE tasks in the coalition */
291 uint32_t carplay_task_count; /* Number of CARPLAY_MODE tasks in the coalition */
292
293 /* coalition flags */
294 uint32_t privileged : 1; /* Members of this coalition may create
295 * and manage coalitions and may posix_spawn
296 * processes into selected coalitions */
297 /* ast? */
298 /* voucher */
299 uint32_t termrequested : 1; /* launchd has requested termination when coalition becomes empty */
300 uint32_t terminated : 1; /* coalition became empty and spawns are now forbidden */
301 uint32_t reaped : 1; /* reaped, invisible to userspace, but waiting for ref_count to go to zero */
302 uint32_t notified : 1; /* no-more-processes notification was sent via special port */
303 uint32_t efficient : 1; /* launchd has marked the coalition as efficient */
304 #if DEVELOPMENT || DEBUG
305 uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */
306 #endif
307
308 struct smrq_slink link; /* global list of coalitions */
309
310 union {
311 lck_mtx_t lock; /* Coalition lock. */
312 struct smr_node smr_node;
313 };
314
315 /* put coalition type-specific structures here */
316 union {
317 struct i_resource_coalition r;
318 struct i_jetsam_coalition j;
319 };
320 };
321
322 os_refgrp_decl(static, coal_ref_grp, "coalitions", NULL);
323 #define coal_ref_init(coal, c) os_ref_init_count_raw(&(coal)->ref_count, &coal_ref_grp, c)
324 #define coal_ref_count(coal) os_ref_get_count_raw(&(coal)->ref_count)
325 #define coal_ref_try_retain(coal) os_ref_retain_try_raw(&(coal)->ref_count, &coal_ref_grp)
326 #define coal_ref_retain(coal) os_ref_retain_raw(&(coal)->ref_count, &coal_ref_grp)
327 #define coal_ref_release(coal) os_ref_release_raw(&(coal)->ref_count, &coal_ref_grp)
328 #define coal_ref_release_live(coal) os_ref_release_live_raw(&(coal)->ref_count, &coal_ref_grp)
329
330 #define COALITION_HASH_SIZE_MIN 16
331
332 static bool
coal_hash_obj_try_get(void * coal)333 coal_hash_obj_try_get(void *coal)
334 {
335 return coal_ref_try_retain((struct coalition *)coal);
336 }
337
338 SMRH_TRAITS_DEFINE_SCALAR(coal_hash_traits, struct coalition, id, link,
339 .domain = &smr_proc_task,
340 .obj_try_get = coal_hash_obj_try_get);
341
342 /*
343 * register different coalition types:
344 * these must be kept in the order specified in coalition.h
345 */
346 static const struct coalition_type s_coalition_types[COALITION_NUM_TYPES] = {
347 {
348 .type = COALITION_TYPE_RESOURCE,
349 .has_default = 1,
350 .init = i_coal_resource_init,
351 .dealloc = i_coal_resource_dealloc,
352 .adopt_task = i_coal_resource_adopt_task,
353 .remove_task = i_coal_resource_remove_task,
354 .set_taskrole = i_coal_resource_set_taskrole,
355 .get_taskrole = i_coal_resource_get_taskrole,
356 .iterate_tasks = i_coal_resource_iterate_tasks,
357 },
358 {
359 .type = COALITION_TYPE_JETSAM,
360 .has_default = 1,
361 .init = i_coal_jetsam_init,
362 .dealloc = i_coal_jetsam_dealloc,
363 .adopt_task = i_coal_jetsam_adopt_task,
364 .remove_task = i_coal_jetsam_remove_task,
365 .set_taskrole = i_coal_jetsam_set_taskrole,
366 .get_taskrole = i_coal_jetsam_get_taskrole,
367 .iterate_tasks = i_coal_jetsam_iterate_tasks,
368 },
369 };
370
371 static KALLOC_TYPE_DEFINE(coalition_zone, struct coalition, KT_PRIV_ACCT);
372
373 #define coal_call(coal, func, ...) \
374 (s_coalition_types[(coal)->type].func)(coal, ## __VA_ARGS__)
375
376
377 #define coalition_lock(c) lck_mtx_lock(&(c)->lock)
378 #define coalition_unlock(c) lck_mtx_unlock(&(c)->lock)
379
380 /*
381 * Define the coalition type to track focal tasks.
382 * On embedded, track them using jetsam coalitions since they have associated thread
383 * groups which reflect this property as a flag (and pass it down to CLPC).
384 * On non-embedded platforms, since not all coalitions have jetsam coalitions
385 * track focal counts on the resource coalition.
386 */
387 #if !XNU_TARGET_OS_OSX
388 #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_JETSAM
389 #else /* !XNU_TARGET_OS_OSX */
390 #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_RESOURCE
391 #endif /* !XNU_TARGET_OS_OSX */
392
393
394 /*
395 *
396 * Coalition ledger implementation
397 *
398 */
399
400 struct coalition_ledger_indices coalition_ledgers =
401 {.logical_writes = -1, };
402 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor);
403
404 ledger_t
coalition_ledger_get_from_task(task_t task)405 coalition_ledger_get_from_task(task_t task)
406 {
407 ledger_t ledger = LEDGER_NULL;
408 coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE];
409
410 if (coal != NULL && (!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]))) {
411 ledger = coal->r.resource_monitor_ledger;
412 ledger_reference(ledger);
413 }
414 return ledger;
415 }
416
417
418 enum {
419 COALITION_IO_LEDGER_ENABLE,
420 COALITION_IO_LEDGER_DISABLE
421 };
422
423 void
coalition_io_monitor_ctl(struct coalition * coalition,uint32_t flags,int64_t limit)424 coalition_io_monitor_ctl(struct coalition *coalition, uint32_t flags, int64_t limit)
425 {
426 ledger_t ledger = coalition->r.resource_monitor_ledger;
427
428 if (flags == COALITION_IO_LEDGER_ENABLE) {
429 /* Configure the logical I/O ledger */
430 ledger_set_limit(ledger, coalition_ledgers.logical_writes, (limit * 1024 * 1024), 0);
431 ledger_set_period(ledger, coalition_ledgers.logical_writes, (COALITION_LEDGER_MONITOR_INTERVAL_SECS * NSEC_PER_SEC));
432 } else if (flags == COALITION_IO_LEDGER_DISABLE) {
433 ledger_disable_refill(ledger, coalition_ledgers.logical_writes);
434 ledger_disable_callback(ledger, coalition_ledgers.logical_writes);
435 }
436 }
437
438 int
coalition_ledger_set_logical_writes_limit(struct coalition * coalition,int64_t limit)439 coalition_ledger_set_logical_writes_limit(struct coalition *coalition, int64_t limit)
440 {
441 int error = 0;
442
443 /* limit = -1 will be used to disable the limit and the callback */
444 if (limit > COALITION_MAX_LOGICAL_WRITES_LIMIT || limit == 0 || limit < -1) {
445 error = EINVAL;
446 goto out;
447 }
448
449 coalition_lock(coalition);
450 if (limit == -1) {
451 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_DISABLE, limit);
452 } else {
453 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_ENABLE, limit);
454 }
455 coalition_unlock(coalition);
456 out:
457 return error;
458 }
459
460 void __attribute__((noinline))
SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor)461 SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor)
462 {
463 int pid = proc_selfpid();
464 ledger_amount_t new_limit;
465 task_t task = current_task();
466 struct ledger_entry_info lei;
467 kern_return_t kr;
468 ledger_t ledger;
469 struct coalition *coalition = task->coalition[COALITION_TYPE_RESOURCE];
470
471 assert(coalition != NULL);
472 ledger = coalition->r.resource_monitor_ledger;
473
474 switch (flavor) {
475 case FLAVOR_IO_LOGICAL_WRITES:
476 ledger_get_entry_info(ledger, coalition_ledgers.logical_writes, &lei);
477 trace_resource_violation(RMON_LOGWRITES_VIOLATED, &lei);
478 break;
479 default:
480 goto Exit;
481 }
482
483 os_log(OS_LOG_DEFAULT, "Coalition [%lld] caught causing excessive I/O (flavor: %d). Task I/O: %lld MB. [Limit : %lld MB per %lld secs]. Triggered by process [%d]\n",
484 coalition->id, flavor, (lei.lei_balance / (1024 * 1024)), (lei.lei_limit / (1024 * 1024)),
485 (lei.lei_refill_period / NSEC_PER_SEC), pid);
486
487 kr = send_resource_violation(send_disk_writes_violation, task, &lei, kRNFlagsNone);
488 if (kr) {
489 os_log(OS_LOG_DEFAULT, "ERROR %#x returned from send_resource_violation(disk_writes, ...)\n", kr);
490 }
491
492 /*
493 * Continue to monitor the coalition after it hits the initital limit, but increase
494 * the limit exponentially so that we don't spam the listener.
495 */
496 new_limit = (lei.lei_limit / 1024 / 1024) * 4;
497 coalition_lock(coalition);
498 if (new_limit > COALITION_MAX_LOGICAL_WRITES_LIMIT) {
499 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_DISABLE, -1);
500 } else {
501 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_ENABLE, new_limit);
502 }
503 coalition_unlock(coalition);
504
505 Exit:
506 return;
507 }
508
509 void
coalition_io_rate_exceeded(int warning,const void * param0,__unused const void * param1)510 coalition_io_rate_exceeded(int warning, const void *param0, __unused const void *param1)
511 {
512 if (warning == 0) {
513 SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO((int)param0);
514 }
515 }
516
517 void
init_coalition_ledgers(void)518 init_coalition_ledgers(void)
519 {
520 ledger_template_t t;
521 assert(coalition_ledger_template == NULL);
522
523 if ((t = ledger_template_create("Per-coalition ledgers")) == NULL) {
524 panic("couldn't create coalition ledger template");
525 }
526
527 coalition_ledgers.logical_writes = ledger_entry_add(t, "logical_writes", "res", "bytes");
528
529 if (coalition_ledgers.logical_writes < 0) {
530 panic("couldn't create entries for coaliton ledger template");
531 }
532
533 ledger_set_callback(t, coalition_ledgers.logical_writes, coalition_io_rate_exceeded, (void *)FLAVOR_IO_LOGICAL_WRITES, NULL);
534 ledger_template_complete(t);
535
536 coalition_task_ledger_template = ledger_template_copy(task_ledger_template, "Coalition task ledgers");
537
538 if (coalition_task_ledger_template == NULL) {
539 panic("couldn't create coalition task ledger template");
540 }
541
542 ledger_template_complete(coalition_task_ledger_template);
543
544 coalition_ledger_template = t;
545 }
546
547 void
coalition_io_ledger_update(task_t task,int32_t flavor,boolean_t is_credit,uint32_t io_size)548 coalition_io_ledger_update(task_t task, int32_t flavor, boolean_t is_credit, uint32_t io_size)
549 {
550 ledger_t ledger;
551 coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE];
552
553 assert(coal != NULL);
554 ledger = coal->r.resource_monitor_ledger;
555 if (LEDGER_VALID(ledger)) {
556 if (flavor == FLAVOR_IO_LOGICAL_WRITES) {
557 if (is_credit) {
558 ledger_credit(ledger, coalition_ledgers.logical_writes, io_size);
559 } else {
560 ledger_debit(ledger, coalition_ledgers.logical_writes, io_size);
561 }
562 }
563 }
564 }
565
566 static void
coalition_notify_user(uint64_t id,uint32_t flags)567 coalition_notify_user(uint64_t id, uint32_t flags)
568 {
569 mach_port_t user_port;
570 kern_return_t kr;
571
572 kr = host_get_coalition_port(host_priv_self(), &user_port);
573 if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
574 return;
575 }
576
577 coalition_notification(user_port, id, flags);
578 ipc_port_release_send(user_port);
579 }
580
581 /*
582 *
583 * COALITION_TYPE_RESOURCE
584 *
585 */
586 static kern_return_t
i_coal_resource_init(coalition_t coal,boolean_t privileged,boolean_t efficient)587 i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient)
588 {
589 #pragma unused(privileged, efficient)
590
591 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
592
593 recount_coalition_init(&coal->r.co_recount);
594 coal->r.ledger = ledger_instantiate(coalition_task_ledger_template,
595 LEDGER_CREATE_ACTIVE_ENTRIES);
596 if (coal->r.ledger == NULL) {
597 return KERN_RESOURCE_SHORTAGE;
598 }
599
600 coal->r.resource_monitor_ledger = ledger_instantiate(coalition_ledger_template,
601 LEDGER_CREATE_ACTIVE_ENTRIES);
602 if (coal->r.resource_monitor_ledger == NULL) {
603 return KERN_RESOURCE_SHORTAGE;
604 }
605
606 queue_init(&coal->r.tasks);
607
608 return KERN_SUCCESS;
609 }
610
611 static void
i_coal_resource_dealloc(coalition_t coal)612 i_coal_resource_dealloc(coalition_t coal)
613 {
614 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
615
616 recount_coalition_deinit(&coal->r.co_recount);
617 ledger_dereference(coal->r.ledger);
618 ledger_dereference(coal->r.resource_monitor_ledger);
619 }
620
621 static kern_return_t
i_coal_resource_adopt_task(coalition_t coal,task_t task)622 i_coal_resource_adopt_task(coalition_t coal, task_t task)
623 {
624 struct i_resource_coalition *cr;
625
626 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
627 assert(queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]));
628
629 cr = &coal->r;
630 cr->task_count++;
631
632 if (cr->task_count < cr->dead_task_count) {
633 panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)",
634 __func__, coal, coal->id, coal_type_str(coal->type),
635 cr->task_count, cr->dead_task_count);
636 }
637
638 /* If moving from 0->1 active tasks */
639 if (cr->task_count - cr->dead_task_count == 1) {
640 cr->last_became_nonempty_time = mach_absolute_time();
641 }
642
643 /* put the task on the coalition's list of tasks */
644 enqueue_tail(&cr->tasks, &task->task_coalition[COALITION_TYPE_RESOURCE]);
645
646 coal_dbg("Added PID:%d to id:%llu, task_count:%llu, dead_count:%llu, nonempty_time:%llu",
647 task_pid(task), coal->id, cr->task_count, cr->dead_task_count,
648 cr->last_became_nonempty_time);
649
650 return KERN_SUCCESS;
651 }
652
653 static kern_return_t
i_coal_resource_remove_task(coalition_t coal,task_t task)654 i_coal_resource_remove_task(coalition_t coal, task_t task)
655 {
656 struct i_resource_coalition *cr;
657
658 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
659 assert(task->coalition[COALITION_TYPE_RESOURCE] == coal);
660 assert(!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]));
661
662 /*
663 * handle resource coalition accounting rollup for dead tasks
664 */
665 cr = &coal->r;
666
667 cr->dead_task_count++;
668
669 if (cr->task_count < cr->dead_task_count) {
670 panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)",
671 __func__, coal, coal->id, coal_type_str(coal->type), cr->task_count, cr->dead_task_count);
672 }
673
674 /* If moving from 1->0 active tasks */
675 if (cr->task_count - cr->dead_task_count == 0) {
676 uint64_t last_time_nonempty = mach_absolute_time() - cr->last_became_nonempty_time;
677 cr->last_became_nonempty_time = 0;
678 cr->time_nonempty += last_time_nonempty;
679 }
680
681 /* Do not roll up for exec'd task or exec copy task */
682 if (!task_is_exec_copy(task) && !task_did_exec(task)) {
683 ledger_rollup(cr->ledger, task->ledger);
684 cr->bytesread += task->task_io_stats->disk_reads.size;
685 cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
686 #if defined(__x86_64__)
687 cr->gpu_time += task_gpu_utilisation(task);
688 #endif /* defined(__x86_64__) */
689
690 cr->logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes;
691 cr->logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes;
692 cr->logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes;
693 cr->logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes;
694 cr->logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes;
695 cr->logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes;
696 cr->logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes;
697 cr->logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes;
698 #if CONFIG_PHYS_WRITE_ACCT
699 cr->fs_metadata_writes += task->task_fs_metadata_writes;
700 #endif /* CONFIG_PHYS_WRITE_ACCT */
701 task_update_cpu_time_qos_stats(task, cr->cpu_time_eqos, cr->cpu_time_rqos);
702 recount_coalition_rollup_task(&cr->co_recount, &task->tk_recount);
703 }
704
705 /* remove the task from the coalition's list */
706 remqueue(&task->task_coalition[COALITION_TYPE_RESOURCE]);
707 queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]);
708
709 coal_dbg("removed PID:%d from id:%llu, task_count:%llu, dead_count:%llu",
710 task_pid(task), coal->id, cr->task_count, cr->dead_task_count);
711
712 return KERN_SUCCESS;
713 }
714
715 static kern_return_t
i_coal_resource_set_taskrole(__unused coalition_t coal,__unused task_t task,__unused int role)716 i_coal_resource_set_taskrole(__unused coalition_t coal,
717 __unused task_t task, __unused int role)
718 {
719 return KERN_SUCCESS;
720 }
721
722 static int
i_coal_resource_get_taskrole(__unused coalition_t coal,__unused task_t task)723 i_coal_resource_get_taskrole(__unused coalition_t coal, __unused task_t task)
724 {
725 task_t t;
726
727 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
728
729 qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
730 if (t == task) {
731 return COALITION_TASKROLE_UNDEF;
732 }
733 }
734
735 return -1;
736 }
737
738 static task_t
i_coal_resource_iterate_tasks(coalition_t coal,coalition_for_each_task_block_t block)739 i_coal_resource_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block)
740 {
741 task_t t;
742 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
743
744 qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
745 bool should_return = block(t);
746 if (should_return) {
747 return t;
748 }
749 }
750
751 return TASK_NULL;
752 }
753
754 #if CONFIG_PHYS_WRITE_ACCT
755 extern uint64_t kernel_pm_writes;
756 #endif /* CONFIG_PHYS_WRITE_ACCT */
757
758 kern_return_t
coalition_resource_usage_internal(coalition_t coal,struct coalition_resource_usage * cru_out)759 coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_usage *cru_out)
760 {
761 kern_return_t kr;
762 ledger_amount_t credit, debit;
763 int i;
764
765 if (coal->type != COALITION_TYPE_RESOURCE) {
766 return KERN_INVALID_ARGUMENT;
767 }
768
769 /* Return KERN_INVALID_ARGUMENT for Corpse coalition */
770 for (i = 0; i < COALITION_NUM_TYPES; i++) {
771 if (coal == corpse_coalition[i]) {
772 return KERN_INVALID_ARGUMENT;
773 }
774 }
775
776 ledger_t sum_ledger = ledger_instantiate(coalition_task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
777 if (sum_ledger == LEDGER_NULL) {
778 return KERN_RESOURCE_SHORTAGE;
779 }
780
781 coalition_lock(coal);
782
783 /*
784 * Start with the coalition's ledger, which holds the totals from all
785 * the dead tasks.
786 */
787 ledger_rollup(sum_ledger, coal->r.ledger);
788 uint64_t bytesread = coal->r.bytesread;
789 uint64_t byteswritten = coal->r.byteswritten;
790 uint64_t gpu_time = coal->r.gpu_time;
791 uint64_t logical_immediate_writes = coal->r.logical_immediate_writes;
792 uint64_t logical_deferred_writes = coal->r.logical_deferred_writes;
793 uint64_t logical_invalidated_writes = coal->r.logical_invalidated_writes;
794 uint64_t logical_metadata_writes = coal->r.logical_metadata_writes;
795 uint64_t logical_immediate_writes_to_external = coal->r.logical_immediate_writes_to_external;
796 uint64_t logical_deferred_writes_to_external = coal->r.logical_deferred_writes_to_external;
797 uint64_t logical_invalidated_writes_to_external = coal->r.logical_invalidated_writes_to_external;
798 uint64_t logical_metadata_writes_to_external = coal->r.logical_metadata_writes_to_external;
799 uint64_t ane_mach_time = os_atomic_load(&coal->r.ane_mach_time, relaxed);
800 uint64_t ane_energy_nj = os_atomic_load(&coal->r.ane_energy_nj, relaxed);
801 #if CONFIG_PHYS_WRITE_ACCT
802 uint64_t fs_metadata_writes = coal->r.fs_metadata_writes;
803 #endif /* CONFIG_PHYS_WRITE_ACCT */
804 int64_t conclave_mem = 0;
805 int64_t cpu_time_billed_to_me = 0;
806 int64_t cpu_time_billed_to_others = 0;
807 int64_t energy_billed_to_me = 0;
808 int64_t energy_billed_to_others = 0;
809 int64_t phys_footprint = 0;
810 int64_t swapins = 0;
811 struct recount_usage stats_sum = { 0 };
812 struct recount_usage stats_perf_only = { 0 };
813 recount_coalition_usage_perf_only(&coal->r.co_recount, &stats_sum,
814 &stats_perf_only);
815 uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 };
816 uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 };
817 memcpy(cpu_time_eqos, &coal->r.cpu_time_eqos, sizeof(cpu_time_eqos));
818 memcpy(cpu_time_rqos, &coal->r.cpu_time_rqos, sizeof(cpu_time_rqos));
819 /*
820 * Add to that all the active tasks' ledgers. Tasks cannot deallocate
821 * out from under us, since we hold the coalition lock.
822 */
823 task_t task;
824 qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
825 int64_t task_phys_footprint = 0;
826
827 /*
828 * Rolling up stats for exec copy task or exec'd task will lead to double accounting.
829 * Cannot take task lock after taking coaliton lock
830 */
831 if (task_is_exec_copy(task) || task_did_exec(task)) {
832 continue;
833 }
834
835 ledger_rollup(sum_ledger, task->ledger);
836 bytesread += task->task_io_stats->disk_reads.size;
837 byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
838 #if defined(__x86_64__)
839 gpu_time += task_gpu_utilisation(task);
840 #endif /* defined(__x86_64__) */
841
842 logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes;
843 logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes;
844 logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes;
845 logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes;
846 logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes;
847 logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes;
848 logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes;
849 logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes;
850 #if CONFIG_PHYS_WRITE_ACCT
851 fs_metadata_writes += task->task_fs_metadata_writes;
852 #endif /* CONFIG_PHYS_WRITE_ACCT */
853
854 /* The exited process ledger can have a phys_footprint balance, which should be ignored. */
855 kr = ledger_get_balance(task->ledger, task_ledgers.phys_footprint, (int64_t *)&task_phys_footprint);
856 if (kr != KERN_SUCCESS || task_phys_footprint < 0) {
857 task_phys_footprint = 0;
858 }
859
860 phys_footprint += task_phys_footprint;
861
862 task_update_cpu_time_qos_stats(task, cpu_time_eqos, cpu_time_rqos);
863 recount_task_usage_perf_only(task, &stats_sum, &stats_perf_only);
864 }
865
866 kr = ledger_get_balance(sum_ledger, task_ledgers.conclave_mem, (int64_t *)&conclave_mem);
867 if (kr != KERN_SUCCESS || conclave_mem < 0) {
868 conclave_mem = 0;
869 }
870
871 kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me);
872 if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) {
873 cpu_time_billed_to_me = 0;
874 }
875
876 kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others);
877 if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) {
878 cpu_time_billed_to_others = 0;
879 }
880
881 kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_me, (int64_t *)&energy_billed_to_me);
882 if (kr != KERN_SUCCESS || energy_billed_to_me < 0) {
883 energy_billed_to_me = 0;
884 }
885
886 kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_others, (int64_t *)&energy_billed_to_others);
887 if (kr != KERN_SUCCESS || energy_billed_to_others < 0) {
888 energy_billed_to_others = 0;
889 }
890
891 kr = ledger_get_balance(sum_ledger, task_ledgers.swapins, (int64_t *)&swapins);
892 if (kr != KERN_SUCCESS || swapins < 0) {
893 swapins = 0;
894 }
895
896 /* collect information from the coalition itself */
897 cru_out->tasks_started = coal->r.task_count;
898 cru_out->tasks_exited = coal->r.dead_task_count;
899
900 uint64_t time_nonempty = coal->r.time_nonempty;
901 uint64_t last_became_nonempty_time = coal->r.last_became_nonempty_time;
902
903 cru_out->gpu_energy_nj = os_atomic_load(&coal->r.gpu_energy_nj, relaxed);
904 cru_out->gpu_energy_nj_billed_to_me = os_atomic_load(&coal->r.gpu_energy_nj_billed_to_me, relaxed);
905 cru_out->gpu_energy_nj_billed_to_others = os_atomic_load(&coal->r.gpu_energy_nj_billed_to_others, relaxed);
906
907 coalition_unlock(coal);
908
909 /* Copy the totals out of sum_ledger */
910 kr = ledger_get_entries(sum_ledger, task_ledgers.cpu_time,
911 &credit, &debit);
912 if (kr != KERN_SUCCESS) {
913 credit = 0;
914 }
915 cru_out->conclave_mem = conclave_mem;
916 cru_out->cpu_time = credit;
917 cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me;
918 cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others;
919 cru_out->energy_billed_to_me = (uint64_t)energy_billed_to_me;
920 cru_out->energy_billed_to_others = (uint64_t)energy_billed_to_others;
921 cru_out->phys_footprint = phys_footprint;
922 cru_out->swapins = swapins;
923
924 kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups,
925 &credit, &debit);
926 if (kr != KERN_SUCCESS) {
927 credit = 0;
928 }
929 cru_out->interrupt_wakeups = credit;
930
931 kr = ledger_get_entries(sum_ledger, task_ledgers.platform_idle_wakeups,
932 &credit, &debit);
933 if (kr != KERN_SUCCESS) {
934 credit = 0;
935 }
936 cru_out->platform_idle_wakeups = credit;
937
938 cru_out->bytesread = bytesread;
939 cru_out->byteswritten = byteswritten;
940 cru_out->gpu_time = gpu_time;
941 cru_out->ane_mach_time = ane_mach_time;
942 cru_out->ane_energy_nj = ane_energy_nj;
943 cru_out->logical_immediate_writes = logical_immediate_writes;
944 cru_out->logical_deferred_writes = logical_deferred_writes;
945 cru_out->logical_invalidated_writes = logical_invalidated_writes;
946 cru_out->logical_metadata_writes = logical_metadata_writes;
947 cru_out->logical_immediate_writes_to_external = logical_immediate_writes_to_external;
948 cru_out->logical_deferred_writes_to_external = logical_deferred_writes_to_external;
949 cru_out->logical_invalidated_writes_to_external = logical_invalidated_writes_to_external;
950 cru_out->logical_metadata_writes_to_external = logical_metadata_writes_to_external;
951 #if CONFIG_PHYS_WRITE_ACCT
952 cru_out->fs_metadata_writes = fs_metadata_writes;
953 #else
954 cru_out->fs_metadata_writes = 0;
955 #endif /* CONFIG_PHYS_WRITE_ACCT */
956 cru_out->cpu_time_eqos_len = COALITION_NUM_THREAD_QOS_TYPES;
957 memcpy(cru_out->cpu_time_eqos, cpu_time_eqos, sizeof(cru_out->cpu_time_eqos));
958
959 cru_out->cpu_ptime = recount_usage_time_mach(&stats_perf_only);
960 #if CONFIG_PERVASIVE_CPI
961 cru_out->cpu_instructions = recount_usage_instructions(&stats_sum);
962 cru_out->cpu_cycles = recount_usage_cycles(&stats_sum);
963 cru_out->cpu_pinstructions = recount_usage_instructions(&stats_perf_only);
964 cru_out->cpu_pcycles = recount_usage_cycles(&stats_perf_only);
965 #endif // CONFIG_PERVASIVE_CPI
966
967 ledger_dereference(sum_ledger);
968 sum_ledger = LEDGER_NULL;
969
970 #if CONFIG_PERVASIVE_ENERGY
971 cru_out->energy = stats_sum.ru_energy_nj;
972 #endif /* CONFIG_PERVASIVE_ENERGY */
973
974 #if CONFIG_PHYS_WRITE_ACCT
975 // kernel_pm_writes are only recorded under kernel_task coalition
976 if (coalition_id(coal) == COALITION_ID_KERNEL) {
977 cru_out->pm_writes = kernel_pm_writes;
978 } else {
979 cru_out->pm_writes = 0;
980 }
981 #else
982 cru_out->pm_writes = 0;
983 #endif /* CONFIG_PHYS_WRITE_ACCT */
984
985 if (last_became_nonempty_time) {
986 time_nonempty += mach_absolute_time() - last_became_nonempty_time;
987 }
988 absolutetime_to_nanoseconds(time_nonempty, &cru_out->time_nonempty);
989
990 return KERN_SUCCESS;
991 }
992
993 bool
coalition_add_to_gpu_energy(uint64_t coalition_id,coalition_gpu_energy_t which,uint64_t energy)994 coalition_add_to_gpu_energy(uint64_t coalition_id,
995 coalition_gpu_energy_t which, uint64_t energy)
996 {
997 assert(coalition_id != 0);
998 bool exists = false;
999
1000 smrh_key_t key = SMRH_SCALAR_KEY(coalition_id);
1001
1002 smrht_enter(&coal_hash_traits);
1003
1004 coalition_t coal = smr_hash_entered_find(&coalition_hash, key, &coal_hash_traits);
1005 /* Preemption is disabled, and until we leave, coalition_free won't be called. */
1006
1007 if (coal != COALITION_NULL &&
1008 coal->type == COALITION_TYPE_RESOURCE &&
1009 !coal->terminated && !coal->reaped) {
1010 /*
1011 * After termination, there are no more member tasks and
1012 * launchd is about to capture the final snapshot, so
1013 * a terminated energy id considered invalid.
1014 *
1015 * Note that we don't lock against the termination transition
1016 * here, so it's possible for a race to occur where new energy
1017 * is accounted past the final snapshot.
1018 */
1019 exists = true;
1020
1021 if (which & CGE_SELF) {
1022 os_atomic_add(&coal->r.gpu_energy_nj, energy, relaxed);
1023 }
1024
1025 if (which & CGE_BILLED) {
1026 os_atomic_add(&coal->r.gpu_energy_nj_billed_to_me,
1027 energy, relaxed);
1028 }
1029
1030 if (which & CGE_OTHERS) {
1031 os_atomic_add(&coal->r.gpu_energy_nj_billed_to_others,
1032 energy, relaxed);
1033 }
1034 }
1035
1036 smrht_leave(&coal_hash_traits);
1037
1038 return exists;
1039 }
1040
1041
1042 kern_return_t
coalition_debug_info_internal(coalition_t coal,struct coalinfo_debuginfo * c_debuginfo)1043 coalition_debug_info_internal(coalition_t coal,
1044 struct coalinfo_debuginfo *c_debuginfo)
1045 {
1046 /* Return KERN_INVALID_ARGUMENT for Corpse coalition */
1047 for (int i = 0; i < COALITION_NUM_TYPES; i++) {
1048 if (coal == corpse_coalition[i]) {
1049 return KERN_INVALID_ARGUMENT;
1050 }
1051 }
1052
1053 if (coal->type == COALITION_FOCAL_TASKS_ACCOUNTING) {
1054 c_debuginfo->focal_task_count = coal->focal_task_count;
1055 c_debuginfo->nonfocal_task_count = coal->nonfocal_task_count;
1056 c_debuginfo->game_task_count = coal->game_task_count;
1057 c_debuginfo->carplay_task_count = coal->carplay_task_count;
1058 }
1059
1060 #if CONFIG_THREAD_GROUPS
1061 struct thread_group * group = coalition_get_thread_group(coal);
1062
1063 if (group != NULL) {
1064 c_debuginfo->thread_group_id = thread_group_id(group);
1065 c_debuginfo->thread_group_flags = thread_group_get_flags(group);
1066 c_debuginfo->thread_group_recommendation = thread_group_recommendation(group);
1067 }
1068 #endif /* CONFIG_THREAD_GROUPS */
1069
1070 return KERN_SUCCESS;
1071 }
1072
1073 /*
1074 *
1075 * COALITION_TYPE_JETSAM
1076 *
1077 */
1078 static kern_return_t
i_coal_jetsam_init(coalition_t coal,boolean_t privileged,boolean_t efficient)1079 i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient)
1080 {
1081 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1082 (void)privileged;
1083 (void)efficient;
1084
1085 coal->j.leader = TASK_NULL;
1086 queue_head_init(coal->j.extensions);
1087 queue_head_init(coal->j.services);
1088 queue_head_init(coal->j.other);
1089
1090 #if CONFIG_THREAD_GROUPS
1091 switch (coal->role) {
1092 case COALITION_ROLE_SYSTEM:
1093 coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_SYSTEM);
1094 break;
1095 case COALITION_ROLE_BACKGROUND:
1096 coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_BACKGROUND);
1097 break;
1098 default:
1099 coal->j.thread_group = thread_group_create_and_retain(efficient ? THREAD_GROUP_FLAGS_EFFICIENT : THREAD_GROUP_FLAGS_DEFAULT);
1100 }
1101 assert(coal->j.thread_group != NULL);
1102 #endif
1103 return KERN_SUCCESS;
1104 }
1105
1106 static void
i_coal_jetsam_dealloc(__unused coalition_t coal)1107 i_coal_jetsam_dealloc(__unused coalition_t coal)
1108 {
1109 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1110
1111 /* the coalition should be completely clear at this point */
1112 assert(queue_empty(&coal->j.extensions));
1113 assert(queue_empty(&coal->j.services));
1114 assert(queue_empty(&coal->j.other));
1115 assert(coal->j.leader == TASK_NULL);
1116
1117 #if CONFIG_THREAD_GROUPS
1118 /* disassociate from the thread group */
1119 assert(coal->j.thread_group != NULL);
1120 thread_group_release(coal->j.thread_group);
1121 coal->j.thread_group = NULL;
1122 #endif
1123 }
1124
1125 static kern_return_t
i_coal_jetsam_adopt_task(coalition_t coal,task_t task)1126 i_coal_jetsam_adopt_task(coalition_t coal, task_t task)
1127 {
1128 struct i_jetsam_coalition *cj;
1129 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1130
1131 cj = &coal->j;
1132
1133 assert(queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM]));
1134
1135 /* put each task initially in the "other" list */
1136 enqueue_tail(&cj->other, &task->task_coalition[COALITION_TYPE_JETSAM]);
1137 coal_dbg("coalition %lld adopted PID:%d as UNDEF",
1138 coal->id, task_pid(task));
1139
1140 return KERN_SUCCESS;
1141 }
1142
1143 static kern_return_t
i_coal_jetsam_remove_task(coalition_t coal,task_t task)1144 i_coal_jetsam_remove_task(coalition_t coal, task_t task)
1145 {
1146 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1147 assert(task->coalition[COALITION_TYPE_JETSAM] == coal);
1148
1149 coal_dbg("removing PID:%d from coalition id:%lld",
1150 task_pid(task), coal->id);
1151
1152 if (task == coal->j.leader) {
1153 coal->j.leader = NULL;
1154 coal_dbg(" PID:%d was the leader!", task_pid(task));
1155 } else {
1156 assert(!queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM]));
1157 }
1158
1159 /* remove the task from the specific coalition role queue */
1160 remqueue(&task->task_coalition[COALITION_TYPE_JETSAM]);
1161 queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]);
1162
1163 return KERN_SUCCESS;
1164 }
1165
1166 static kern_return_t
i_coal_jetsam_set_taskrole(coalition_t coal,task_t task,int role)1167 i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role)
1168 {
1169 struct i_jetsam_coalition *cj;
1170 queue_t q = NULL;
1171 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1172 assert(task->coalition[COALITION_TYPE_JETSAM] == coal);
1173
1174 cj = &coal->j;
1175
1176 switch (role) {
1177 case COALITION_TASKROLE_LEADER:
1178 coal_dbg("setting PID:%d as LEADER of %lld",
1179 task_pid(task), coal->id);
1180 if (cj->leader != TASK_NULL) {
1181 /* re-queue the exiting leader onto the "other" list */
1182 coal_dbg(" re-queue existing leader (%d) as OTHER",
1183 task_pid(cj->leader));
1184 re_queue_tail(&cj->other, &cj->leader->task_coalition[COALITION_TYPE_JETSAM]);
1185 }
1186 /*
1187 * remove the task from the "other" list
1188 * (where it was put by default)
1189 */
1190 remqueue(&task->task_coalition[COALITION_TYPE_JETSAM]);
1191 queue_chain_init(task->task_coalition[COALITION_TYPE_JETSAM]);
1192
1193 /* set the coalition leader */
1194 cj->leader = task;
1195 break;
1196 case COALITION_TASKROLE_XPC:
1197 coal_dbg("setting PID:%d as XPC in %lld",
1198 task_pid(task), coal->id);
1199 q = (queue_t)&cj->services;
1200 break;
1201 case COALITION_TASKROLE_EXT:
1202 coal_dbg("setting PID:%d as EXT in %lld",
1203 task_pid(task), coal->id);
1204 q = (queue_t)&cj->extensions;
1205 break;
1206 case COALITION_TASKROLE_NONE:
1207 /*
1208 * Tasks with a role of "none" should fall through to an
1209 * undefined role so long as the task is currently a member
1210 * of the coalition. This scenario can happen if a task is
1211 * killed (usually via jetsam) during exec.
1212 */
1213 if (task->coalition[COALITION_TYPE_JETSAM] != coal) {
1214 panic("%s: task %p attempting to set role %d "
1215 "in coalition %p to which it does not belong!", __func__, task, role, coal);
1216 }
1217 OS_FALLTHROUGH;
1218 case COALITION_TASKROLE_UNDEF:
1219 coal_dbg("setting PID:%d as UNDEF in %lld",
1220 task_pid(task), coal->id);
1221 q = (queue_t)&cj->other;
1222 break;
1223 default:
1224 panic("%s: invalid role(%d) for task", __func__, role);
1225 return KERN_INVALID_ARGUMENT;
1226 }
1227
1228 if (q != NULL) {
1229 re_queue_tail(q, &task->task_coalition[COALITION_TYPE_JETSAM]);
1230 }
1231
1232 return KERN_SUCCESS;
1233 }
1234
1235 int
i_coal_jetsam_get_taskrole(coalition_t coal,task_t task)1236 i_coal_jetsam_get_taskrole(coalition_t coal, task_t task)
1237 {
1238 struct i_jetsam_coalition *cj;
1239 task_t t;
1240
1241 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1242 assert(task->coalition[COALITION_TYPE_JETSAM] == coal);
1243
1244 cj = &coal->j;
1245
1246 if (task == cj->leader) {
1247 return COALITION_TASKROLE_LEADER;
1248 }
1249
1250 qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) {
1251 if (t == task) {
1252 return COALITION_TASKROLE_XPC;
1253 }
1254 }
1255
1256 qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) {
1257 if (t == task) {
1258 return COALITION_TASKROLE_EXT;
1259 }
1260 }
1261
1262 qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) {
1263 if (t == task) {
1264 return COALITION_TASKROLE_UNDEF;
1265 }
1266 }
1267
1268 /* task not in the coalition?! */
1269 return COALITION_TASKROLE_NONE;
1270 }
1271
1272 static task_t
i_coal_jetsam_iterate_tasks(coalition_t coal,coalition_for_each_task_block_t block)1273 i_coal_jetsam_iterate_tasks(
1274 coalition_t coal,
1275 coalition_for_each_task_block_t block)
1276 {
1277 struct i_jetsam_coalition *cj;
1278 task_t t;
1279
1280 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1281
1282 cj = &coal->j;
1283
1284 if (cj->leader) {
1285 t = cj->leader;
1286 bool should_return = block( t);
1287 if (should_return) {
1288 return t;
1289 }
1290 }
1291
1292 qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) {
1293 bool should_return = block(t);
1294 if (should_return) {
1295 return t;
1296 }
1297 }
1298
1299 qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) {
1300 bool should_return = block(t);
1301 if (should_return) {
1302 return t;
1303 }
1304 }
1305
1306 qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) {
1307 bool should_return = block(t);
1308 if (should_return) {
1309 return t;
1310 }
1311 }
1312
1313 return TASK_NULL;
1314 }
1315
1316
1317 /*
1318 *
1319 * Main Coalition implementation
1320 *
1321 */
1322
1323 /*
1324 * coalition_create_internal
1325 * Returns: New coalition object, referenced for the caller and unlocked.
1326 * Condition: coalitions_list_lock must be UNLOCKED.
1327 */
1328 kern_return_t
coalition_create_internal(int type,int role,boolean_t privileged,boolean_t efficient,coalition_t * out,uint64_t * coalition_id)1329 coalition_create_internal(int type, int role, boolean_t privileged, boolean_t efficient, coalition_t *out, uint64_t *coalition_id)
1330 {
1331 kern_return_t kr;
1332 struct coalition *new_coal;
1333 uint64_t cid;
1334
1335 if (type < 0 || type > COALITION_TYPE_MAX) {
1336 return KERN_INVALID_ARGUMENT;
1337 }
1338
1339 new_coal = zalloc_flags(coalition_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1340
1341 new_coal->type = type;
1342 new_coal->role = role;
1343
1344 /* initialize type-specific resources */
1345 kr = coal_call(new_coal, init, privileged, efficient);
1346 if (kr != KERN_SUCCESS) {
1347 zfree(coalition_zone, new_coal);
1348 return kr;
1349 }
1350
1351 /* One for caller, one for coalitions list */
1352 coal_ref_init(new_coal, 2);
1353
1354 new_coal->privileged = privileged ? TRUE : FALSE;
1355 new_coal->efficient = efficient ? TRUE : FALSE;
1356 #if DEVELOPMENT || DEBUG
1357 new_coal->should_notify = 1;
1358 #endif
1359
1360 lck_mtx_init(&new_coal->lock, &coalitions_lck_grp, LCK_ATTR_NULL);
1361
1362 lck_mtx_lock(&coalitions_list_lock);
1363 new_coal->id = cid = coalition_next_id++;
1364
1365 smr_hash_serialized_insert(&coalition_hash, &new_coal->link,
1366 &coal_hash_traits);
1367
1368 #if CONFIG_THREAD_GROUPS
1369 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW),
1370 new_coal->id, new_coal->type,
1371 (new_coal->type == COALITION_TYPE_JETSAM && new_coal->j.thread_group) ?
1372 thread_group_get_id(new_coal->j.thread_group) : 0);
1373
1374 #else
1375 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW),
1376 new_coal->id, new_coal->type);
1377 #endif
1378
1379 if (smr_hash_serialized_should_grow(&coalition_hash, 1, 4)) {
1380 /* grow if more more than 4 elements per 1 bucket */
1381 smr_hash_grow_and_unlock(&coalition_hash,
1382 &coalitions_list_lock, &coal_hash_traits);
1383 } else {
1384 lck_mtx_unlock(&coalitions_list_lock);
1385 }
1386
1387 coal_dbg("id:%llu, type:%s", cid, coal_type_str(type));
1388
1389 if (coalition_id != NULL) {
1390 *coalition_id = cid;
1391 }
1392
1393 *out = new_coal;
1394 return KERN_SUCCESS;
1395 }
1396
1397 static void
coalition_free(struct smr_node * node)1398 coalition_free(struct smr_node *node)
1399 {
1400 struct coalition *coal;
1401
1402 coal = __container_of(node, struct coalition, smr_node);
1403 zfree(coalition_zone, coal);
1404 }
1405
1406 static __attribute__((noinline)) void
coalition_retire(coalition_t coal)1407 coalition_retire(coalition_t coal)
1408 {
1409 coalition_lock(coal);
1410
1411 coal_dbg("id:%llu type:%s active_count:%u%s",
1412 coal->id, coal_type_str(coal->type), coal->active_count,
1413 rc <= 0 ? ", will deallocate now" : "");
1414
1415 assert(coal->termrequested);
1416 assert(coal->terminated);
1417 assert(coal->active_count == 0);
1418 assert(coal->reaped);
1419 assert(coal->focal_task_count == 0);
1420 assert(coal->nonfocal_task_count == 0);
1421 assert(coal->game_task_count == 0);
1422 assert(coal->carplay_task_count == 0);
1423 #if CONFIG_THREAD_GROUPS
1424 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE),
1425 coal->id, coal->type,
1426 coal->type == COALITION_TYPE_JETSAM ?
1427 coal->j.thread_group : 0);
1428 #else
1429 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE),
1430 coal->id, coal->type);
1431 #endif
1432
1433 coal_call(coal, dealloc);
1434
1435 coalition_unlock(coal);
1436
1437 lck_mtx_destroy(&coal->lock, &coalitions_lck_grp);
1438
1439 smr_proc_task_call(&coal->smr_node, sizeof(*coal), coalition_free);
1440 }
1441
1442 /*
1443 * coalition_retain
1444 * */
1445 void
coalition_retain(coalition_t coal)1446 coalition_retain(coalition_t coal)
1447 {
1448 coal_ref_retain(coal);
1449 }
1450
1451 /*
1452 * coalition_release
1453 * Condition: coalition must be UNLOCKED.
1454 * */
1455 void
coalition_release(coalition_t coal)1456 coalition_release(coalition_t coal)
1457 {
1458 if (coal_ref_release(coal) > 0) {
1459 return;
1460 }
1461
1462 coalition_retire(coal);
1463 }
1464
1465 /*
1466 * coalition_find_by_id
1467 * Returns: Coalition object with specified id, referenced.
1468 */
1469 coalition_t
coalition_find_by_id(uint64_t cid)1470 coalition_find_by_id(uint64_t cid)
1471 {
1472 smrh_key_t key = SMRH_SCALAR_KEY(cid);
1473
1474 if (cid == 0) {
1475 return COALITION_NULL;
1476 }
1477
1478 return smr_hash_get(&coalition_hash, key, &coal_hash_traits);
1479 }
1480
1481 /*
1482 * coalition_find_and_activate_by_id
1483 * Returns: Coalition object with specified id, referenced, and activated.
1484 * This is the function to use when putting a 'new' thing into a coalition,
1485 * like posix_spawn of an XPC service by launchd.
1486 * See also coalition_extend_active.
1487 */
1488 coalition_t
coalition_find_and_activate_by_id(uint64_t cid)1489 coalition_find_and_activate_by_id(uint64_t cid)
1490 {
1491 coalition_t coal = coalition_find_by_id(cid);
1492
1493 if (coal == COALITION_NULL) {
1494 return COALITION_NULL;
1495 }
1496
1497 coalition_lock(coal);
1498
1499 if (coal->reaped || coal->terminated) {
1500 /* Too late to put something new into this coalition, it's
1501 * already on its way out the door */
1502 coalition_unlock(coal);
1503 coalition_release(coal);
1504 return COALITION_NULL;
1505 }
1506
1507 coal->active_count++;
1508
1509 #if COALITION_DEBUG
1510 uint32_t rc = coal_ref_count(coal);
1511 uint32_t ac = coal->active_count;
1512 #endif
1513 coalition_unlock(coal);
1514
1515 coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u",
1516 coal->id, coal_type_str(coal->type), rc, ac);
1517
1518 return coal;
1519 }
1520
1521 uint64_t
coalition_id(coalition_t coal)1522 coalition_id(coalition_t coal)
1523 {
1524 assert(coal != COALITION_NULL);
1525 return coal->id;
1526 }
1527
1528 void
task_coalition_ids(task_t task,uint64_t ids[COALITION_NUM_TYPES])1529 task_coalition_ids(task_t task, uint64_t ids[COALITION_NUM_TYPES])
1530 {
1531 int i;
1532 for (i = 0; i < COALITION_NUM_TYPES; i++) {
1533 if (task->coalition[i]) {
1534 ids[i] = task->coalition[i]->id;
1535 } else {
1536 ids[i] = 0;
1537 }
1538 }
1539 }
1540
1541 void
task_coalition_roles(task_t task,int roles[COALITION_NUM_TYPES])1542 task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES])
1543 {
1544 int i;
1545 memset(roles, 0, COALITION_NUM_TYPES * sizeof(roles[0]));
1546
1547 for (i = 0; i < COALITION_NUM_TYPES; i++) {
1548 if (task->coalition[i]) {
1549 coalition_lock(task->coalition[i]);
1550 roles[i] = coal_call(task->coalition[i],
1551 get_taskrole, task);
1552 coalition_unlock(task->coalition[i]);
1553 } else {
1554 roles[i] = COALITION_TASKROLE_NONE;
1555 }
1556 }
1557 }
1558
1559 int
task_coalition_role_for_type(task_t task,int coalition_type)1560 task_coalition_role_for_type(task_t task, int coalition_type)
1561 {
1562 coalition_t coal;
1563 int role;
1564 if (coalition_type >= COALITION_NUM_TYPES) {
1565 panic("Attempt to call task_coalition_role_for_type with invalid coalition_type: %d\n", coalition_type);
1566 }
1567 coal = task->coalition[coalition_type];
1568 if (coal == NULL) {
1569 return COALITION_TASKROLE_NONE;
1570 }
1571 coalition_lock(coal);
1572 role = coal_call(coal, get_taskrole, task);
1573 coalition_unlock(coal);
1574 return role;
1575 }
1576
1577 int
coalition_type(coalition_t coal)1578 coalition_type(coalition_t coal)
1579 {
1580 return coal->type;
1581 }
1582
1583 boolean_t
coalition_term_requested(coalition_t coal)1584 coalition_term_requested(coalition_t coal)
1585 {
1586 return coal->termrequested;
1587 }
1588
1589 boolean_t
coalition_is_terminated(coalition_t coal)1590 coalition_is_terminated(coalition_t coal)
1591 {
1592 return coal->terminated;
1593 }
1594
1595 boolean_t
coalition_is_reaped(coalition_t coal)1596 coalition_is_reaped(coalition_t coal)
1597 {
1598 return coal->reaped;
1599 }
1600
1601 boolean_t
coalition_is_privileged(coalition_t coal)1602 coalition_is_privileged(coalition_t coal)
1603 {
1604 return coal->privileged || unrestrict_coalition_syscalls;
1605 }
1606
1607 boolean_t
task_is_in_privileged_coalition(task_t task,int type)1608 task_is_in_privileged_coalition(task_t task, int type)
1609 {
1610 if (type < 0 || type > COALITION_TYPE_MAX) {
1611 return FALSE;
1612 }
1613 if (unrestrict_coalition_syscalls) {
1614 return TRUE;
1615 }
1616 if (!task->coalition[type]) {
1617 return FALSE;
1618 }
1619 return task->coalition[type]->privileged;
1620 }
1621
1622 void
task_coalition_update_gpu_stats(task_t task,uint64_t gpu_ns_delta)1623 task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta)
1624 {
1625 coalition_t coal;
1626
1627 assert(task != TASK_NULL);
1628 if (gpu_ns_delta == 0) {
1629 return;
1630 }
1631
1632 coal = task->coalition[COALITION_TYPE_RESOURCE];
1633 assert(coal != COALITION_NULL);
1634
1635 coalition_lock(coal);
1636 coal->r.gpu_time += gpu_ns_delta;
1637 coalition_unlock(coal);
1638 }
1639
1640 void
coalition_update_ane_stats(coalition_t coalition,uint64_t ane_mach_time,uint64_t ane_energy_nj)1641 coalition_update_ane_stats(coalition_t coalition, uint64_t ane_mach_time, uint64_t ane_energy_nj)
1642 {
1643 assert(coalition != COALITION_NULL);
1644
1645 os_atomic_add(&coalition->r.ane_mach_time, ane_mach_time, relaxed);
1646 os_atomic_add(&coalition->r.ane_energy_nj, ane_energy_nj, relaxed);
1647 }
1648
1649 boolean_t
task_coalition_adjust_focal_count(task_t task,int count,uint32_t * new_count)1650 task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count)
1651 {
1652 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1653 if (coal == COALITION_NULL) {
1654 return FALSE;
1655 }
1656
1657 *new_count = os_atomic_add(&coal->focal_task_count, count, relaxed);
1658 assert(*new_count != UINT32_MAX);
1659 return TRUE;
1660 }
1661
1662 uint32_t
task_coalition_focal_count(task_t task)1663 task_coalition_focal_count(task_t task)
1664 {
1665 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1666 if (coal == COALITION_NULL) {
1667 return 0;
1668 }
1669
1670 return coal->focal_task_count;
1671 }
1672
1673 uint32_t
task_coalition_game_mode_count(task_t task)1674 task_coalition_game_mode_count(task_t task)
1675 {
1676 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1677 if (coal == COALITION_NULL) {
1678 return 0;
1679 }
1680
1681 return coal->game_task_count;
1682 }
1683
1684 uint32_t
task_coalition_carplay_mode_count(task_t task)1685 task_coalition_carplay_mode_count(task_t task)
1686 {
1687 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1688 if (coal == COALITION_NULL) {
1689 return 0;
1690 }
1691 return coal->carplay_task_count;
1692 }
1693
1694 boolean_t
task_coalition_adjust_nonfocal_count(task_t task,int count,uint32_t * new_count)1695 task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count)
1696 {
1697 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1698 if (coal == COALITION_NULL) {
1699 return FALSE;
1700 }
1701
1702 *new_count = os_atomic_add(&coal->nonfocal_task_count, count, relaxed);
1703 assert(*new_count != UINT32_MAX);
1704 return TRUE;
1705 }
1706
1707 uint32_t
task_coalition_nonfocal_count(task_t task)1708 task_coalition_nonfocal_count(task_t task)
1709 {
1710 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1711 if (coal == COALITION_NULL) {
1712 return 0;
1713 }
1714
1715 return coal->nonfocal_task_count;
1716 }
1717
1718 bool
task_coalition_adjust_game_mode_count(task_t task,int count,uint32_t * new_count)1719 task_coalition_adjust_game_mode_count(task_t task, int count, uint32_t *new_count)
1720 {
1721 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1722 if (coal == COALITION_NULL) {
1723 return false;
1724 }
1725
1726 *new_count = os_atomic_add(&coal->game_task_count, count, relaxed);
1727 assert(*new_count != UINT32_MAX);
1728 return true;
1729 }
1730
1731 bool
task_coalition_adjust_carplay_mode_count(task_t task,int count,uint32_t * new_count)1732 task_coalition_adjust_carplay_mode_count(task_t task, int count, uint32_t *new_count)
1733 {
1734 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1735 if (coal == COALITION_NULL) {
1736 return false;
1737 }
1738
1739 *new_count = os_atomic_add(&coal->carplay_task_count, count, relaxed);
1740 assert(*new_count != UINT32_MAX);
1741 return true;
1742 }
1743
1744 #if CONFIG_THREAD_GROUPS
1745
1746 /* Thread group lives as long as the task is holding the coalition reference */
1747 struct thread_group *
task_coalition_get_thread_group(task_t task)1748 task_coalition_get_thread_group(task_t task)
1749 {
1750 coalition_t coal = task->coalition[COALITION_TYPE_JETSAM];
1751 /* return system thread group for non-jetsam coalitions */
1752 if (coal == COALITION_NULL) {
1753 return init_coalition[COALITION_TYPE_JETSAM]->j.thread_group;
1754 }
1755 return coal->j.thread_group;
1756 }
1757
1758
1759 struct thread_group *
kdp_coalition_get_thread_group(coalition_t coal)1760 kdp_coalition_get_thread_group(coalition_t coal)
1761 {
1762 if (coal->type != COALITION_TYPE_JETSAM) {
1763 return NULL;
1764 }
1765 assert(coal->j.thread_group != NULL);
1766 return coal->j.thread_group;
1767 }
1768
1769 /* Thread group lives as long as the coalition reference is held */
1770 struct thread_group *
coalition_get_thread_group(coalition_t coal)1771 coalition_get_thread_group(coalition_t coal)
1772 {
1773 if (coal->type != COALITION_TYPE_JETSAM) {
1774 return NULL;
1775 }
1776 assert(coal->j.thread_group != NULL);
1777 return coal->j.thread_group;
1778 }
1779
1780 /* Donates the thread group reference to the coalition */
1781 void
coalition_set_thread_group(coalition_t coal,struct thread_group * tg)1782 coalition_set_thread_group(coalition_t coal, struct thread_group *tg)
1783 {
1784 assert(coal != COALITION_NULL);
1785 assert(tg != NULL);
1786
1787 if (coal->type != COALITION_TYPE_JETSAM) {
1788 return;
1789 }
1790 struct thread_group *old_tg = coal->j.thread_group;
1791 assert(old_tg != NULL);
1792 coal->j.thread_group = tg;
1793
1794 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_THREAD_GROUP_SET),
1795 coal->id, coal->type, thread_group_get_id(tg));
1796
1797 thread_group_release(old_tg);
1798 }
1799
1800 void
task_coalition_thread_group_focal_update(task_t task)1801 task_coalition_thread_group_focal_update(task_t task)
1802 {
1803 assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL);
1804 thread_group_flags_update_lock();
1805 uint32_t focal_count = task_coalition_focal_count(task);
1806 if (focal_count) {
1807 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP);
1808 } else {
1809 thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP);
1810 }
1811 thread_group_flags_update_unlock();
1812 }
1813
1814 void
task_coalition_thread_group_game_mode_update(task_t task)1815 task_coalition_thread_group_game_mode_update(task_t task)
1816 {
1817 assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL);
1818 thread_group_flags_update_lock();
1819 if (task_coalition_game_mode_count(task)) {
1820 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE);
1821 } else {
1822 thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE);
1823 }
1824 thread_group_flags_update_unlock();
1825 }
1826
1827 void
task_coalition_thread_group_carplay_mode_update(task_t task)1828 task_coalition_thread_group_carplay_mode_update(task_t task)
1829 {
1830 assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL);
1831 thread_group_flags_update_lock();
1832 if (task_coalition_carplay_mode_count(task)) {
1833 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_CARPLAY_MODE);
1834 } else {
1835 thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_CARPLAY_MODE);
1836 }
1837 thread_group_flags_update_unlock();
1838 }
1839
1840 void
task_coalition_thread_group_application_set(task_t task)1841 task_coalition_thread_group_application_set(task_t task)
1842 {
1843 /*
1844 * Setting the "Application" flag on the thread group is a one way transition.
1845 * Once a coalition has a single task with an application apptype, the
1846 * thread group associated with the coalition is tagged as Application.
1847 */
1848 thread_group_flags_update_lock();
1849 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_APPLICATION);
1850 thread_group_flags_update_unlock();
1851 }
1852
1853 #endif /* CONFIG_THREAD_GROUPS */
1854
1855 static void
coalition_for_each_task_locked(coalition_t coal,OS_NOESCAPE coalition_for_each_task_block_t block)1856 coalition_for_each_task_locked(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block)
1857 {
1858 assert(coal != COALITION_NULL);
1859
1860 coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u",
1861 coal, coal->id, coal_type_str(coal->type), coal->active_count);
1862
1863 __assert_only task_t selected_task;
1864
1865 selected_task = coal_call(coal, iterate_tasks, block);
1866
1867 assert(selected_task == TASK_NULL);
1868 }
1869
1870 void
coalition_for_each_task(coalition_t coal,OS_NOESCAPE coalition_for_each_task_block_t block)1871 coalition_for_each_task(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block)
1872 {
1873 assert(coal != COALITION_NULL);
1874
1875 coalition_lock(coal);
1876
1877 coalition_for_each_task_locked(coal, block);
1878
1879 coalition_unlock(coal);
1880 }
1881
1882 static task_t
coalition_select_task(coalition_t coal,OS_NOESCAPE coalition_for_each_task_block_t block)1883 coalition_select_task(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block)
1884 {
1885 assert(coal != COALITION_NULL);
1886
1887 coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u",
1888 coal, coal->id, coal_type_str(coal->type), coal->active_count);
1889
1890 coalition_lock(coal);
1891
1892 task_t selected_task = coal_call(coal, iterate_tasks, block);
1893
1894 if (selected_task != TASK_NULL) {
1895 task_reference(selected_task);
1896 }
1897
1898 coalition_unlock(coal);
1899 return selected_task;
1900 }
1901
1902
1903 void
coalition_remove_active(coalition_t coal)1904 coalition_remove_active(coalition_t coal)
1905 {
1906 coalition_lock(coal);
1907
1908 assert(!coalition_is_reaped(coal));
1909 assert(coal->active_count > 0);
1910
1911 coal->active_count--;
1912
1913 boolean_t do_notify = FALSE;
1914 uint64_t notify_id = 0;
1915 uint32_t notify_flags = 0;
1916 if (coal->termrequested && coal->active_count == 0) {
1917 /* We only notify once, when active_count reaches zero.
1918 * We just decremented, so if it reached zero, we mustn't have
1919 * notified already.
1920 */
1921 assert(!coal->terminated);
1922 coal->terminated = TRUE;
1923
1924 assert(!coal->notified);
1925
1926 coal->notified = TRUE;
1927 #if DEVELOPMENT || DEBUG
1928 do_notify = coal->should_notify;
1929 #else
1930 do_notify = TRUE;
1931 #endif
1932 notify_id = coal->id;
1933 notify_flags = 0;
1934 }
1935
1936 #if COALITION_DEBUG
1937 uint64_t cid = coal->id;
1938 uint32_t rc = coal_ref_count(coal);
1939 int ac = coal->active_count;
1940 int ct = coal->type;
1941 #endif
1942 coalition_unlock(coal);
1943
1944 coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u,%s",
1945 cid, coal_type_str(ct), rc, ac, do_notify ? " NOTIFY" : " ");
1946
1947 if (do_notify) {
1948 coalition_notify_user(notify_id, notify_flags);
1949 }
1950 }
1951
1952 /* Used for kernel_task, launchd, launchd's early boot tasks... */
1953 kern_return_t
coalitions_adopt_init_task(task_t task)1954 coalitions_adopt_init_task(task_t task)
1955 {
1956 kern_return_t kr;
1957 kr = coalitions_adopt_task(init_coalition, task);
1958 if (kr != KERN_SUCCESS) {
1959 panic("failed to adopt task %p into default coalition: %d", task, kr);
1960 }
1961 return kr;
1962 }
1963
1964 /* Used for forked corpses. */
1965 kern_return_t
coalitions_adopt_corpse_task(task_t task)1966 coalitions_adopt_corpse_task(task_t task)
1967 {
1968 kern_return_t kr;
1969 kr = coalitions_adopt_task(corpse_coalition, task);
1970 if (kr != KERN_SUCCESS) {
1971 panic("failed to adopt task %p into corpse coalition: %d", task, kr);
1972 }
1973 return kr;
1974 }
1975
1976 /*
1977 * coalition_adopt_task_internal
1978 * Condition: Coalition must be referenced and unlocked. Will fail if coalition
1979 * is already terminated.
1980 */
1981 static kern_return_t
coalition_adopt_task_internal(coalition_t coal,task_t task)1982 coalition_adopt_task_internal(coalition_t coal, task_t task)
1983 {
1984 kern_return_t kr;
1985
1986 if (task->coalition[coal->type]) {
1987 return KERN_ALREADY_IN_SET;
1988 }
1989
1990 coalition_lock(coal);
1991
1992 if (coal->reaped || coal->terminated) {
1993 coalition_unlock(coal);
1994 return KERN_TERMINATED;
1995 }
1996
1997 kr = coal_call(coal, adopt_task, task);
1998 if (kr != KERN_SUCCESS) {
1999 goto out_unlock;
2000 }
2001
2002 coal->active_count++;
2003
2004 coal_ref_retain(coal);
2005
2006 task->coalition[coal->type] = coal;
2007
2008 out_unlock:
2009 #if COALITION_DEBUG
2010 (void)coal; /* need expression after label */
2011 uint64_t cid = coal->id;
2012 uint32_t rc = coal_ref_count(coal);
2013 uint32_t ct = coal->type;
2014 #endif
2015 if (get_task_uniqueid(task) != UINT64_MAX) {
2016 /* On 32-bit targets, uniqueid will get truncated to 32 bits */
2017 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_ADOPT),
2018 coal->id, get_task_uniqueid(task));
2019 }
2020
2021 coalition_unlock(coal);
2022
2023 coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d",
2024 task_pid(task), cid, coal_type_str(ct), rc, kr);
2025 return kr;
2026 }
2027
2028 static kern_return_t
coalition_remove_task_internal(task_t task,int type)2029 coalition_remove_task_internal(task_t task, int type)
2030 {
2031 kern_return_t kr;
2032
2033 coalition_t coal = task->coalition[type];
2034
2035 if (!coal) {
2036 return KERN_SUCCESS;
2037 }
2038
2039 assert(coal->type == (uint32_t)type);
2040
2041 coalition_lock(coal);
2042
2043 kr = coal_call(coal, remove_task, task);
2044
2045 #if COALITION_DEBUG
2046 uint64_t cid = coal->id;
2047 uint32_t rc = coal_ref_count(coal);
2048 int ac = coal->active_count;
2049 int ct = coal->type;
2050 #endif
2051 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_REMOVE),
2052 coal->id, get_task_uniqueid(task));
2053 coalition_unlock(coal);
2054
2055 coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d",
2056 cid, coal_type_str(ct), rc, ac, kr);
2057
2058 coalition_remove_active(coal);
2059
2060 return kr;
2061 }
2062
2063 /*
2064 * coalitions_adopt_task
2065 * Condition: All coalitions must be referenced and unlocked.
2066 * Will fail if any coalition is already terminated.
2067 */
2068 kern_return_t
coalitions_adopt_task(coalition_t * coals,task_t task)2069 coalitions_adopt_task(coalition_t *coals, task_t task)
2070 {
2071 int i;
2072 kern_return_t kr;
2073
2074 if (!coals || coals[COALITION_TYPE_RESOURCE] == COALITION_NULL) {
2075 return KERN_INVALID_ARGUMENT;
2076 }
2077
2078 /* verify that the incoming coalitions are what they say they are */
2079 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2080 if (coals[i] && coals[i]->type != (uint32_t)i) {
2081 return KERN_INVALID_ARGUMENT;
2082 }
2083 }
2084
2085 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2086 kr = KERN_SUCCESS;
2087 if (coals[i]) {
2088 kr = coalition_adopt_task_internal(coals[i], task);
2089 }
2090 if (kr != KERN_SUCCESS) {
2091 /* dis-associate any coalitions that just adopted this task */
2092 while (--i >= 0) {
2093 if (task->coalition[i]) {
2094 coalition_remove_task_internal(task, i);
2095 }
2096 }
2097 break;
2098 }
2099 }
2100 return kr;
2101 }
2102
2103 /*
2104 * coalitions_remove_task
2105 * Condition: task must be referenced and UNLOCKED; all task's coalitions must be UNLOCKED
2106 */
2107 kern_return_t
coalitions_remove_task(task_t task)2108 coalitions_remove_task(task_t task)
2109 {
2110 kern_return_t kr;
2111 int i;
2112
2113 task_lock(task);
2114 if (!task_is_coalition_member(task)) {
2115 task_unlock(task);
2116 return KERN_SUCCESS;
2117 }
2118
2119 task_clear_coalition_member(task);
2120 task_unlock(task);
2121
2122 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2123 kr = coalition_remove_task_internal(task, i);
2124 assert(kr == KERN_SUCCESS);
2125 }
2126
2127 return kr;
2128 }
2129
2130 /*
2131 * task_release_coalitions
2132 * helper function to release references to all coalitions in which
2133 * 'task' is a member.
2134 */
2135 void
task_release_coalitions(task_t task)2136 task_release_coalitions(task_t task)
2137 {
2138 int i;
2139 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2140 if (task->coalition[i]) {
2141 coalition_release(task->coalition[i]);
2142 } else if (i == COALITION_TYPE_RESOURCE) {
2143 panic("deallocating task %p was not a member of a resource coalition", task);
2144 }
2145 }
2146 }
2147
2148 /*
2149 * coalitions_set_roles
2150 * for each type of coalition, if the task is a member of a coalition of
2151 * that type (given in the coalitions parameter) then set the role of
2152 * the task within that that coalition.
2153 */
2154 kern_return_t
coalitions_set_roles(coalition_t coalitions[COALITION_NUM_TYPES],task_t task,int roles[COALITION_NUM_TYPES])2155 coalitions_set_roles(coalition_t coalitions[COALITION_NUM_TYPES],
2156 task_t task, int roles[COALITION_NUM_TYPES])
2157 {
2158 kern_return_t kr = KERN_SUCCESS;
2159 int i;
2160
2161 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2162 if (!coalitions[i]) {
2163 continue;
2164 }
2165 coalition_lock(coalitions[i]);
2166 kr = coal_call(coalitions[i], set_taskrole, task, roles[i]);
2167 coalition_unlock(coalitions[i]);
2168 assert(kr == KERN_SUCCESS);
2169 }
2170
2171 return kr;
2172 }
2173
2174 /*
2175 * coalition_terminate_internal
2176 * Condition: Coalition must be referenced and UNLOCKED.
2177 */
2178 kern_return_t
coalition_request_terminate_internal(coalition_t coal)2179 coalition_request_terminate_internal(coalition_t coal)
2180 {
2181 assert(coal->type >= 0 && coal->type <= COALITION_TYPE_MAX);
2182
2183 if (coal == init_coalition[coal->type]) {
2184 return KERN_DEFAULT_SET;
2185 }
2186
2187 coalition_lock(coal);
2188
2189 if (coal->reaped) {
2190 coalition_unlock(coal);
2191 return KERN_INVALID_NAME;
2192 }
2193
2194 if (coal->terminated || coal->termrequested) {
2195 coalition_unlock(coal);
2196 return KERN_TERMINATED;
2197 }
2198
2199 coal->termrequested = TRUE;
2200
2201 boolean_t do_notify = FALSE;
2202 uint64_t note_id = 0;
2203 uint32_t note_flags = 0;
2204
2205 if (coal->active_count == 0) {
2206 /*
2207 * We only notify once, when active_count reaches zero.
2208 * We just set termrequested to zero. If the active count
2209 * was already at zero (tasks died before we could request
2210 * a termination notification), we should notify.
2211 */
2212 assert(!coal->terminated);
2213 coal->terminated = TRUE;
2214
2215 assert(!coal->notified);
2216
2217 coal->notified = TRUE;
2218 #if DEVELOPMENT || DEBUG
2219 do_notify = coal->should_notify;
2220 #else
2221 do_notify = TRUE;
2222 #endif
2223 note_id = coal->id;
2224 note_flags = 0;
2225 }
2226
2227 coalition_unlock(coal);
2228
2229 if (do_notify) {
2230 coalition_notify_user(note_id, note_flags);
2231 }
2232
2233 return KERN_SUCCESS;
2234 }
2235
2236 /*
2237 * coalition_reap_internal
2238 * Condition: Coalition must be referenced and UNLOCKED.
2239 */
2240 kern_return_t
coalition_reap_internal(coalition_t coal)2241 coalition_reap_internal(coalition_t coal)
2242 {
2243 assert(coal->type <= COALITION_TYPE_MAX);
2244
2245 if (coal == init_coalition[coal->type]) {
2246 return KERN_DEFAULT_SET;
2247 }
2248
2249 coalition_lock(coal);
2250 if (coal->reaped) {
2251 coalition_unlock(coal);
2252 return KERN_TERMINATED;
2253 }
2254 if (!coal->terminated) {
2255 coalition_unlock(coal);
2256 return KERN_FAILURE;
2257 }
2258 assert(coal->termrequested);
2259 if (coal->active_count > 0) {
2260 coalition_unlock(coal);
2261 return KERN_FAILURE;
2262 }
2263
2264 coal->reaped = TRUE;
2265
2266 coalition_unlock(coal);
2267
2268 lck_mtx_lock(&coalitions_list_lock);
2269 smr_hash_serialized_remove(&coalition_hash, &coal->link,
2270 &coal_hash_traits);
2271 if (smr_hash_serialized_should_shrink(&coalition_hash,
2272 COALITION_HASH_SIZE_MIN, 2, 1)) {
2273 /* shrink if more than 2 buckets per 1 element */
2274 smr_hash_shrink_and_unlock(&coalition_hash,
2275 &coalitions_list_lock, &coal_hash_traits);
2276 } else {
2277 lck_mtx_unlock(&coalitions_list_lock);
2278 }
2279
2280 /* Release the list's reference and launchd's reference. */
2281 coal_ref_release_live(coal);
2282 coalition_release(coal);
2283
2284 return KERN_SUCCESS;
2285 }
2286
2287 #if DEVELOPMENT || DEBUG
2288 int
coalition_should_notify(coalition_t coal)2289 coalition_should_notify(coalition_t coal)
2290 {
2291 int should;
2292 if (!coal) {
2293 return -1;
2294 }
2295 coalition_lock(coal);
2296 should = coal->should_notify;
2297 coalition_unlock(coal);
2298
2299 return should;
2300 }
2301
2302 void
coalition_set_notify(coalition_t coal,int notify)2303 coalition_set_notify(coalition_t coal, int notify)
2304 {
2305 if (!coal) {
2306 return;
2307 }
2308 coalition_lock(coal);
2309 coal->should_notify = !!notify;
2310 coalition_unlock(coal);
2311 }
2312 #endif
2313
2314 void
coalitions_init(void)2315 coalitions_init(void)
2316 {
2317 kern_return_t kr;
2318 int i;
2319 const struct coalition_type *ctype;
2320
2321 smr_hash_init(&coalition_hash, COALITION_HASH_SIZE_MIN);
2322
2323 init_task_ledgers();
2324
2325 init_coalition_ledgers();
2326
2327 for (i = 0, ctype = &s_coalition_types[0]; i < COALITION_NUM_TYPES; ctype++, i++) {
2328 /* verify the entry in the global coalition types array */
2329 if (ctype->type != i ||
2330 !ctype->init ||
2331 !ctype->dealloc ||
2332 !ctype->adopt_task ||
2333 !ctype->remove_task) {
2334 panic("%s: Malformed coalition type %s(%d) in slot for type:%s(%d)",
2335 __func__, coal_type_str(ctype->type), ctype->type, coal_type_str(i), i);
2336 }
2337 if (!ctype->has_default) {
2338 continue;
2339 }
2340 kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, TRUE, FALSE, &init_coalition[ctype->type], NULL);
2341 if (kr != KERN_SUCCESS) {
2342 panic("%s: could not create init %s coalition: kr:%d",
2343 __func__, coal_type_str(i), kr);
2344 }
2345 if (i == COALITION_TYPE_RESOURCE) {
2346 assert(COALITION_ID_KERNEL == init_coalition[ctype->type]->id);
2347 }
2348 kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, FALSE, FALSE, &corpse_coalition[ctype->type], NULL);
2349 if (kr != KERN_SUCCESS) {
2350 panic("%s: could not create corpse %s coalition: kr:%d",
2351 __func__, coal_type_str(i), kr);
2352 }
2353 }
2354
2355 /* "Leak" our reference to the global object */
2356 }
2357
2358 /*
2359 * BSD Kernel interface functions
2360 *
2361 */
2362 static void
coalition_fill_procinfo(struct coalition * coal,struct procinfo_coalinfo * coalinfo)2363 coalition_fill_procinfo(struct coalition *coal,
2364 struct procinfo_coalinfo *coalinfo)
2365 {
2366 coalinfo->coalition_id = coal->id;
2367 coalinfo->coalition_type = coal->type;
2368 coalinfo->coalition_tasks = coalition_get_task_count(coal);
2369 }
2370
2371
2372 size_t
coalitions_get_list(int type,struct procinfo_coalinfo * coal_list,size_t list_sz)2373 coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz)
2374 {
2375 size_t ncoals = 0;
2376 struct coalition *coal;
2377
2378 lck_mtx_lock(&coalitions_list_lock);
2379 smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) {
2380 if (!coal->reaped && (type < 0 || type == (int)coal->type)) {
2381 if (coal_list && ncoals < list_sz) {
2382 coalition_fill_procinfo(coal, &coal_list[ncoals]);
2383 }
2384 ++ncoals;
2385 }
2386 }
2387 lck_mtx_unlock(&coalitions_list_lock);
2388
2389 return ncoals;
2390 }
2391
2392 /*
2393 * Return the coaltion of the given type to which the task belongs.
2394 */
2395 coalition_t
task_get_coalition(task_t task,int coal_type)2396 task_get_coalition(task_t task, int coal_type)
2397 {
2398 coalition_t c;
2399
2400 if (task == NULL || coal_type > COALITION_TYPE_MAX) {
2401 return COALITION_NULL;
2402 }
2403
2404 c = task->coalition[coal_type];
2405 assert(c == COALITION_NULL || (int)c->type == coal_type);
2406 return c;
2407 }
2408
2409 /*
2410 * Report if the given task is the leader of the given jetsam coalition.
2411 */
2412 boolean_t
coalition_is_leader(task_t task,coalition_t coal)2413 coalition_is_leader(task_t task, coalition_t coal)
2414 {
2415 boolean_t ret = FALSE;
2416
2417 if (coal != COALITION_NULL) {
2418 coalition_lock(coal);
2419
2420 ret = (coal->type == COALITION_TYPE_JETSAM && coal->j.leader == task);
2421
2422 coalition_unlock(coal);
2423 }
2424
2425 return ret;
2426 }
2427
2428 kern_return_t
coalition_iterate_stackshot(coalition_iterate_fn_t callout,void * arg,uint32_t coalition_type)2429 coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type)
2430 {
2431 coalition_t coal;
2432 int i = 0;
2433
2434 smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) {
2435 if (coal == NULL || !ml_validate_nofault((vm_offset_t)coal, sizeof(struct coalition))) {
2436 return KERN_FAILURE;
2437 }
2438
2439 if (coalition_type == coal->type) {
2440 callout(arg, i++, coal);
2441 }
2442 }
2443
2444 return KERN_SUCCESS;
2445 }
2446
2447 task_t
kdp_coalition_get_leader(coalition_t coal)2448 kdp_coalition_get_leader(coalition_t coal)
2449 {
2450 if (!coal) {
2451 return TASK_NULL;
2452 }
2453
2454 if (coal->type == COALITION_TYPE_JETSAM) {
2455 return coal->j.leader;
2456 }
2457 return TASK_NULL;
2458 }
2459
2460 task_t
coalition_get_leader(coalition_t coal)2461 coalition_get_leader(coalition_t coal)
2462 {
2463 task_t leader = TASK_NULL;
2464
2465 if (!coal) {
2466 return TASK_NULL;
2467 }
2468
2469 coalition_lock(coal);
2470 if (coal->type != COALITION_TYPE_JETSAM) {
2471 goto out_unlock;
2472 }
2473
2474 leader = coal->j.leader;
2475 if (leader != TASK_NULL) {
2476 task_reference(leader);
2477 }
2478
2479 out_unlock:
2480 coalition_unlock(coal);
2481 return leader;
2482 }
2483
2484
2485 int
coalition_get_task_count(coalition_t coal)2486 coalition_get_task_count(coalition_t coal)
2487 {
2488 int ntasks = 0;
2489 struct queue_entry *qe;
2490 if (!coal) {
2491 return 0;
2492 }
2493
2494 coalition_lock(coal);
2495 switch (coal->type) {
2496 case COALITION_TYPE_RESOURCE:
2497 qe_foreach(qe, &coal->r.tasks)
2498 ntasks++;
2499 break;
2500 case COALITION_TYPE_JETSAM:
2501 if (coal->j.leader) {
2502 ntasks++;
2503 }
2504 qe_foreach(qe, &coal->j.other)
2505 ntasks++;
2506 qe_foreach(qe, &coal->j.extensions)
2507 ntasks++;
2508 qe_foreach(qe, &coal->j.services)
2509 ntasks++;
2510 break;
2511 default:
2512 break;
2513 }
2514 coalition_unlock(coal);
2515
2516 return ntasks;
2517 }
2518
2519
2520 static uint64_t
i_get_list_footprint(queue_t list,int type,int * ntasks)2521 i_get_list_footprint(queue_t list, int type, int *ntasks)
2522 {
2523 task_t task;
2524 uint64_t bytes = 0;
2525
2526 qe_foreach_element(task, list, task_coalition[type]) {
2527 bytes += get_task_phys_footprint(task);
2528 coal_dbg(" [%d] task_pid:%d, type:%d, footprint:%lld",
2529 *ntasks, task_pid(task), type, bytes);
2530 *ntasks += 1;
2531 }
2532
2533 return bytes;
2534 }
2535
2536 uint64_t
coalition_get_page_count(coalition_t coal,int * ntasks)2537 coalition_get_page_count(coalition_t coal, int *ntasks)
2538 {
2539 uint64_t bytes = 0;
2540 int num_tasks = 0;
2541
2542 if (ntasks) {
2543 *ntasks = 0;
2544 }
2545 if (!coal) {
2546 return bytes;
2547 }
2548
2549 coalition_lock(coal);
2550
2551 switch (coal->type) {
2552 case COALITION_TYPE_RESOURCE:
2553 bytes += i_get_list_footprint(&coal->r.tasks, COALITION_TYPE_RESOURCE, &num_tasks);
2554 break;
2555 case COALITION_TYPE_JETSAM:
2556 if (coal->j.leader) {
2557 bytes += get_task_phys_footprint(coal->j.leader);
2558 num_tasks = 1;
2559 }
2560 bytes += i_get_list_footprint(&coal->j.extensions, COALITION_TYPE_JETSAM, &num_tasks);
2561 bytes += i_get_list_footprint(&coal->j.services, COALITION_TYPE_JETSAM, &num_tasks);
2562 bytes += i_get_list_footprint(&coal->j.other, COALITION_TYPE_JETSAM, &num_tasks);
2563 break;
2564 default:
2565 break;
2566 }
2567
2568 coalition_unlock(coal);
2569
2570 if (ntasks) {
2571 *ntasks = num_tasks;
2572 }
2573
2574 return bytes / PAGE_SIZE_64;
2575 }
2576
2577 struct coal_sort_s {
2578 int pid;
2579 int usr_order;
2580 uint64_t bytes;
2581 };
2582
2583 /*
2584 * return < 0 for a < b
2585 * 0 for a == b
2586 * > 0 for a > b
2587 */
2588 typedef int (*cmpfunc_t)(const void *a, const void *b);
2589
2590 extern void
2591 qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
2592
2593 static int
dflt_cmp(const void * a,const void * b)2594 dflt_cmp(const void *a, const void *b)
2595 {
2596 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2597 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2598
2599 /*
2600 * if both A and B are equal, use a memory descending sort
2601 */
2602 if (csA->usr_order == csB->usr_order) {
2603 return (int)((int64_t)csB->bytes - (int64_t)csA->bytes);
2604 }
2605
2606 /* otherwise, return the relationship between user specified orders */
2607 return csA->usr_order - csB->usr_order;
2608 }
2609
2610 static int
mem_asc_cmp(const void * a,const void * b)2611 mem_asc_cmp(const void *a, const void *b)
2612 {
2613 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2614 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2615
2616 return (int)((int64_t)csA->bytes - (int64_t)csB->bytes);
2617 }
2618
2619 static int
mem_dec_cmp(const void * a,const void * b)2620 mem_dec_cmp(const void *a, const void *b)
2621 {
2622 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2623 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2624
2625 return (int)((int64_t)csB->bytes - (int64_t)csA->bytes);
2626 }
2627
2628 static int
usr_asc_cmp(const void * a,const void * b)2629 usr_asc_cmp(const void *a, const void *b)
2630 {
2631 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2632 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2633
2634 return csA->usr_order - csB->usr_order;
2635 }
2636
2637 static int
usr_dec_cmp(const void * a,const void * b)2638 usr_dec_cmp(const void *a, const void *b)
2639 {
2640 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2641 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2642
2643 return csB->usr_order - csA->usr_order;
2644 }
2645
2646 /* avoid dynamic allocation in this path */
2647 #define MAX_SORTED_PIDS 80
2648
2649 static int
coalition_get_sort_list(coalition_t coal,int sort_order,queue_t list,struct coal_sort_s * sort_array,int array_sz)2650 coalition_get_sort_list(coalition_t coal, int sort_order, queue_t list,
2651 struct coal_sort_s *sort_array, int array_sz)
2652 {
2653 int ntasks = 0;
2654 task_t task;
2655
2656 assert(sort_array != NULL);
2657
2658 if (array_sz <= 0) {
2659 return 0;
2660 }
2661
2662 if (!list) {
2663 /*
2664 * this function will only be called with a NULL
2665 * list for JETSAM-type coalitions, and is intended
2666 * to investigate the leader process
2667 */
2668 if (coal->type != COALITION_TYPE_JETSAM ||
2669 coal->j.leader == TASK_NULL) {
2670 return 0;
2671 }
2672 sort_array[0].pid = task_pid(coal->j.leader);
2673 switch (sort_order) {
2674 case COALITION_SORT_DEFAULT:
2675 sort_array[0].usr_order = 0;
2676 OS_FALLTHROUGH;
2677 case COALITION_SORT_MEM_ASC:
2678 case COALITION_SORT_MEM_DEC:
2679 sort_array[0].bytes = get_task_phys_footprint(coal->j.leader);
2680 break;
2681 case COALITION_SORT_USER_ASC:
2682 case COALITION_SORT_USER_DEC:
2683 sort_array[0].usr_order = 0;
2684 break;
2685 default:
2686 break;
2687 }
2688 return 1;
2689 }
2690
2691 qe_foreach_element(task, list, task_coalition[coal->type]) {
2692 if (ntasks >= array_sz) {
2693 printf("WARNING: more than %d pids in coalition %llu\n",
2694 MAX_SORTED_PIDS, coal->id);
2695 break;
2696 }
2697
2698 sort_array[ntasks].pid = task_pid(task);
2699
2700 switch (sort_order) {
2701 case COALITION_SORT_DEFAULT:
2702 sort_array[ntasks].usr_order = 0;
2703 OS_FALLTHROUGH;
2704 case COALITION_SORT_MEM_ASC:
2705 case COALITION_SORT_MEM_DEC:
2706 sort_array[ntasks].bytes = get_task_phys_footprint(task);
2707 break;
2708 case COALITION_SORT_USER_ASC:
2709 case COALITION_SORT_USER_DEC:
2710 sort_array[ntasks].usr_order = 0;
2711 break;
2712 default:
2713 break;
2714 }
2715
2716 ntasks++;
2717 }
2718
2719 return ntasks;
2720 }
2721
2722 int
coalition_get_pid_list(coalition_t coal,uint32_t rolemask,int sort_order,int * pid_list,int list_sz)2723 coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order,
2724 int *pid_list, int list_sz)
2725 {
2726 struct i_jetsam_coalition *cj;
2727 int ntasks = 0;
2728 cmpfunc_t cmp_func = NULL;
2729 struct coal_sort_s sort_array[MAX_SORTED_PIDS] = { {0, 0, 0} }; /* keep to < 2k */
2730
2731 if (!coal ||
2732 !(rolemask & COALITION_ROLEMASK_ALLROLES) ||
2733 !pid_list || list_sz < 1) {
2734 coal_dbg("Invalid parameters: coal:%p, type:%d, rolemask:0x%x, "
2735 "pid_list:%p, list_sz:%d", coal, coal ? coal->type : -1,
2736 rolemask, pid_list, list_sz);
2737 return -EINVAL;
2738 }
2739
2740 switch (sort_order) {
2741 case COALITION_SORT_NOSORT:
2742 cmp_func = NULL;
2743 break;
2744 case COALITION_SORT_DEFAULT:
2745 cmp_func = dflt_cmp;
2746 break;
2747 case COALITION_SORT_MEM_ASC:
2748 cmp_func = mem_asc_cmp;
2749 break;
2750 case COALITION_SORT_MEM_DEC:
2751 cmp_func = mem_dec_cmp;
2752 break;
2753 case COALITION_SORT_USER_ASC:
2754 cmp_func = usr_asc_cmp;
2755 break;
2756 case COALITION_SORT_USER_DEC:
2757 cmp_func = usr_dec_cmp;
2758 break;
2759 default:
2760 return -ENOTSUP;
2761 }
2762
2763 coalition_lock(coal);
2764
2765 if (coal->type == COALITION_TYPE_RESOURCE) {
2766 ntasks += coalition_get_sort_list(coal, sort_order, &coal->r.tasks,
2767 sort_array, MAX_SORTED_PIDS);
2768 goto unlock_coal;
2769 }
2770
2771 cj = &coal->j;
2772
2773 if (rolemask & COALITION_ROLEMASK_UNDEF) {
2774 ntasks += coalition_get_sort_list(coal, sort_order, &cj->other,
2775 sort_array + ntasks,
2776 MAX_SORTED_PIDS - ntasks);
2777 }
2778
2779 if (rolemask & COALITION_ROLEMASK_XPC) {
2780 ntasks += coalition_get_sort_list(coal, sort_order, &cj->services,
2781 sort_array + ntasks,
2782 MAX_SORTED_PIDS - ntasks);
2783 }
2784
2785 if (rolemask & COALITION_ROLEMASK_EXT) {
2786 ntasks += coalition_get_sort_list(coal, sort_order, &cj->extensions,
2787 sort_array + ntasks,
2788 MAX_SORTED_PIDS - ntasks);
2789 }
2790
2791 if (rolemask & COALITION_ROLEMASK_LEADER) {
2792 ntasks += coalition_get_sort_list(coal, sort_order, NULL,
2793 sort_array + ntasks,
2794 MAX_SORTED_PIDS - ntasks);
2795 }
2796
2797 unlock_coal:
2798 coalition_unlock(coal);
2799
2800 /* sort based on the chosen criterion (no sense sorting 1 item) */
2801 if (cmp_func && ntasks > 1) {
2802 qsort(sort_array, ntasks, sizeof(struct coal_sort_s), cmp_func);
2803 }
2804
2805 for (int i = 0; i < ntasks; i++) {
2806 if (i >= list_sz) {
2807 break;
2808 }
2809 coal_dbg(" [%d] PID:%d, footprint:%lld, usr_order:%d",
2810 i, sort_array[i].pid, sort_array[i].bytes,
2811 sort_array[i].usr_order);
2812 pid_list[i] = sort_array[i].pid;
2813 }
2814
2815 return ntasks;
2816 }
2817
2818 void
coalition_mark_swappable(coalition_t coal)2819 coalition_mark_swappable(coalition_t coal)
2820 {
2821 struct i_jetsam_coalition *cj = NULL;
2822
2823 coalition_lock(coal);
2824 assert(coal && coal->type == COALITION_TYPE_JETSAM);
2825
2826 cj = &coal->j;
2827 cj->swap_enabled = true;
2828
2829 i_coal_jetsam_iterate_tasks(coal, ^bool (task_t task) {
2830 vm_task_set_selfdonate_pages(task, true);
2831 return false;
2832 });
2833
2834 coalition_unlock(coal);
2835 }
2836
2837 bool
coalition_is_swappable(coalition_t coal)2838 coalition_is_swappable(coalition_t coal)
2839 {
2840 struct i_jetsam_coalition *cj = NULL;
2841
2842 coalition_lock(coal);
2843 assert(coal && coal->type == COALITION_TYPE_JETSAM);
2844
2845 cj = &coal->j;
2846 bool enabled = cj->swap_enabled;
2847
2848 coalition_unlock(coal);
2849
2850 return enabled;
2851 }
2852
2853
2854 /*
2855 * Coalition policy functions
2856 */
2857
2858 static uintptr_t
crequested(coalition_t coal)2859 crequested(coalition_t coal)
2860 {
2861 static_assert(sizeof(struct coalition_requested_policy) == sizeof(uint64_t), "size invariant violated");
2862
2863 uintptr_t* raw = (uintptr_t*)&coal->j.c_requested_policy;
2864
2865 return raw[0];
2866 }
2867
2868 static uintptr_t
ceffective(coalition_t coal)2869 ceffective(coalition_t coal)
2870 {
2871 static_assert(sizeof(struct coalition_requested_policy) == sizeof(uint64_t), "size invariant violated");
2872
2873 uintptr_t* raw = (uintptr_t*)&coal->j.c_effective_policy;
2874
2875 return raw[0];
2876 }
2877
2878 static uint32_t
cpending(coalition_pend_token_t pend_token)2879 cpending(coalition_pend_token_t pend_token)
2880 {
2881 static_assert(sizeof(struct coalition_pend_token) == sizeof(uint32_t), "size invariant violated");
2882
2883 return *(uint32_t*)(void*)(pend_token);
2884 }
2885
2886
2887 static void
jetsam_coalition_policy_update_locked(coalition_t coal,coalition_pend_token_t pend_token)2888 jetsam_coalition_policy_update_locked(coalition_t coal, coalition_pend_token_t pend_token)
2889 {
2890 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2891 (IMPORTANCE_CODE(IMP_UPDATE, TASK_POLICY_COALITION) | DBG_FUNC_START),
2892 coalition_id(coal), ceffective(coal), 0, 0, 0);
2893
2894 struct coalition_requested_policy requested = coal->j.c_requested_policy;
2895 struct coalition_effective_policy next = {};
2896
2897 /* Update darwinbg state */
2898 next.cep_darwinbg = requested.crp_darwinbg;
2899
2900 /* TODO: should termrequested cause darwinbg to be unset */
2901
2902 struct coalition_effective_policy prev = coal->j.c_effective_policy;
2903
2904 coal->j.c_effective_policy = next;
2905
2906 /* */
2907 if (prev.cep_darwinbg != next.cep_darwinbg) {
2908 pend_token->cpt_update_j_coal_tasks = 1;
2909 }
2910
2911 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2912 (IMPORTANCE_CODE(IMP_UPDATE, TASK_POLICY_COALITION)) | DBG_FUNC_END,
2913 coalition_id(coal), ceffective(coal), 0, 0, 0);
2914 }
2915
2916 /*
2917 * Returns task reference or TASK_NULL if none left.
2918 * Called without coalition lock held.
2919 * Called with global coalition_policy_lock held.
2920 */
2921 static task_t
coalition_find_next_task_with_pending_work(coalition_t coal)2922 coalition_find_next_task_with_pending_work(coalition_t coal)
2923 {
2924 return coalition_select_task(coal, ^bool (task_t task){
2925 if (task->pended_coalition_changes.tpt_value != 0) {
2926 return true;
2927 }
2928 return false;
2929 });
2930 }
2931
2932 /*
2933 * Called with coalition unlocked to do things that can't be done
2934 * while holding the coalition lock.
2935 * Called with global coalition_policy_lock held.
2936 */
2937 static void
coalition_policy_update_complete_unlocked(coalition_t coal,coalition_pend_token_t coalition_pend_token)2938 coalition_policy_update_complete_unlocked(
2939 coalition_t coal,
2940 coalition_pend_token_t coalition_pend_token)
2941 {
2942 /*
2943 * Note that because we dropped the coalition lock, a task may have
2944 * exited since we set the pending bit, so we can't assert that
2945 * no task exits with a change pending.
2946 */
2947
2948 if (coalition_pend_token->cpt_update_j_coal_tasks) {
2949 task_t task;
2950
2951 while ((task = coalition_find_next_task_with_pending_work(coal)) != TASK_NULL) {
2952 task_policy_update_complete_unlocked(task, &task->pended_coalition_changes);
2953 task->pended_coalition_changes.tpt_value = 0;
2954 task_deallocate(task);
2955 }
2956 }
2957
2958
2959 if (coalition_pend_token->cpt_update_timers) {
2960 ml_timer_evaluate();
2961 }
2962 }
2963
2964 /*
2965 * This lock covers the coalition_pend_token fields
2966 * in all tasks, so that we don't have to have stack storage
2967 * for all of them. We could also use a gate or busy bit
2968 * on each coalition, but this is less complex.
2969 */
2970 LCK_MTX_DECLARE(coalition_policy_lock, &coalitions_lck_grp);
2971
2972 kern_return_t
jetsam_coalition_set_policy(coalition_t coal,int flavor,int value)2973 jetsam_coalition_set_policy(coalition_t coal,
2974 int flavor,
2975 int value)
2976 {
2977 struct coalition_pend_token pend_token = {};
2978 coalition_pend_token_t pend_token_ref = &pend_token;
2979
2980 assert(coalition_type(coal) == COALITION_TYPE_JETSAM);
2981
2982 lck_mtx_lock(&coalition_policy_lock);
2983
2984 coalition_lock(coal);
2985
2986 struct coalition_requested_policy requested = coal->j.c_requested_policy;
2987
2988 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2989 (IMPORTANCE_CODE(flavor, TASK_POLICY_COALITION)) | DBG_FUNC_START,
2990 coalition_id(coal), crequested(coal), 0, value, 0);
2991
2992 switch (flavor) {
2993 case TASK_POLICY_DARWIN_BG:
2994 requested.crp_darwinbg = value;
2995 break;
2996 default:
2997 panic("unknown coalition policy: %d %d", flavor, value);
2998 break;
2999 }
3000
3001 coal->j.c_requested_policy = requested;
3002
3003 jetsam_coalition_policy_update_locked(coal, &pend_token);
3004
3005 if (pend_token.cpt_update_j_coal_tasks) {
3006 coalition_for_each_task_locked(coal, ^bool (task_t task) {
3007 coalition_policy_update_task(task, pend_token_ref);
3008 return false;
3009 });
3010 }
3011
3012 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3013 (IMPORTANCE_CODE(flavor, TASK_POLICY_COALITION)) | DBG_FUNC_END,
3014 coalition_id(coal), crequested(coal), 0, cpending(&pend_token), 0);
3015
3016 coalition_unlock(coal);
3017
3018 coalition_policy_update_complete_unlocked(coal, &pend_token);
3019
3020 lck_mtx_unlock(&coalition_policy_lock);
3021
3022 return KERN_SUCCESS;
3023 }
3024
3025 kern_return_t
jetsam_coalition_get_policy(coalition_t coal,int flavor,int * value)3026 jetsam_coalition_get_policy(coalition_t coal, int flavor, int *value)
3027 {
3028 lck_mtx_lock(&coalition_policy_lock);
3029
3030 coalition_lock(coal);
3031
3032 struct coalition_requested_policy requested = coal->j.c_requested_policy;
3033
3034 switch (flavor) {
3035 case TASK_POLICY_DARWIN_BG:
3036 *value = requested.crp_darwinbg;
3037 break;
3038 default:
3039 panic("unknown coalition policy: %d", flavor);
3040 break;
3041 }
3042
3043 coalition_unlock(coal);
3044
3045 lck_mtx_unlock(&coalition_policy_lock);
3046
3047 return KERN_SUCCESS;
3048 }
3049
3050 bool
task_get_effective_jetsam_coalition_policy(task_t task,int flavor)3051 task_get_effective_jetsam_coalition_policy(task_t task, int flavor)
3052 {
3053 coalition_t coal = task->coalition[COALITION_TYPE_JETSAM];
3054
3055 if (coal == NULL) {
3056 return false;
3057 }
3058
3059 struct coalition_effective_policy effective = coal->j.c_effective_policy;
3060
3061 switch (flavor) {
3062 case TASK_POLICY_DARWIN_BG:
3063 return effective.cep_darwinbg;
3064 default:
3065 panic("unknown coalition policy: %d", flavor);
3066 break;
3067 }
3068
3069 return false;
3070 }
3071