1 /*
2 * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/kern_types.h>
30 #include <mach/mach_types.h>
31 #include <mach/boolean.h>
32
33 #include <kern/coalition.h>
34 #include <kern/exc_resource.h>
35 #include <kern/host.h>
36 #include <kern/ledger.h>
37 #include <kern/mach_param.h> /* for TASK_CHUNK */
38 #include <kern/monotonic.h>
39 #include <kern/policy_internal.h>
40 #include <kern/task.h>
41 #include <kern/smr_hash.h>
42 #include <kern/thread_group.h>
43 #include <kern/zalloc.h>
44 #include <kern/policy_internal.h>
45
46 #include <vm/vm_pageout.h>
47
48 #include <libkern/OSAtomic.h>
49
50 #include <mach/coalition_notification_server.h>
51 #include <mach/host_priv.h>
52 #include <mach/host_special_ports.h>
53
54 #include <os/log.h>
55
56 #include <sys/errno.h>
57
58 /*
59 * BSD interface functions
60 */
61 size_t coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz);
62 boolean_t coalition_is_leader(task_t task, coalition_t coal);
63 task_t coalition_get_leader(coalition_t coal);
64 int coalition_get_task_count(coalition_t coal);
65 uint64_t coalition_get_page_count(coalition_t coal, int *ntasks);
66 int coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order,
67 int *pid_list, int list_sz);
68
69 /* defined in task.c */
70 extern ledger_template_t task_ledger_template;
71
72 /*
73 * Templates; task template is copied due to potential allocation limits on
74 * task ledgers.
75 */
76 ledger_template_t coalition_task_ledger_template = NULL;
77 ledger_template_t coalition_ledger_template = NULL;
78
79 extern int proc_selfpid(void);
80 /*
81 * Coalition zone needs limits. We expect there will be as many coalitions as
82 * tasks (same order of magnitude), so use the task zone's limits.
83 * */
84 #define CONFIG_COALITION_MAX CONFIG_TASK_MAX
85 #define COALITION_CHUNK TASK_CHUNK
86
87 #if DEBUG || DEVELOPMENT
88 TUNABLE_WRITEABLE(int, unrestrict_coalition_syscalls, "unrestrict_coalition_syscalls", 0);
89 #else
90 #define unrestrict_coalition_syscalls false
91 #endif
92
93 static LCK_GRP_DECLARE(coalitions_lck_grp, "coalition");
94
95 /* coalitions_list_lock protects coalition_hash, next_coalition_id. */
96 static LCK_MTX_DECLARE(coalitions_list_lock, &coalitions_lck_grp);
97 static uint64_t coalition_next_id = 1;
98 static struct smr_hash coalition_hash;
99
100 coalition_t init_coalition[COALITION_NUM_TYPES];
101 coalition_t corpse_coalition[COALITION_NUM_TYPES];
102
103 static const char *
coal_type_str(int type)104 coal_type_str(int type)
105 {
106 switch (type) {
107 case COALITION_TYPE_RESOURCE:
108 return "RESOURCE";
109 case COALITION_TYPE_JETSAM:
110 return "JETSAM";
111 default:
112 return "<unknown>";
113 }
114 }
115
116 struct coalition_type {
117 int type;
118 int has_default;
119 /*
120 * init
121 * pre-condition: coalition just allocated (unlocked), unreferenced,
122 * type field set
123 */
124 kern_return_t (*init)(coalition_t coal, boolean_t privileged, boolean_t efficient);
125
126 /*
127 * dealloc
128 * pre-condition: coalition unlocked
129 * pre-condition: coalition refcount=0, active_count=0,
130 * termrequested=1, terminated=1, reaped=1
131 */
132 void (*dealloc)(coalition_t coal);
133
134 /*
135 * adopt_task
136 * pre-condition: coalition locked
137 * pre-condition: coalition !repead and !terminated
138 */
139 kern_return_t (*adopt_task)(coalition_t coal, task_t task);
140
141 /*
142 * remove_task
143 * pre-condition: coalition locked
144 * pre-condition: task has been removed from coalition's task list
145 */
146 kern_return_t (*remove_task)(coalition_t coal, task_t task);
147
148 /*
149 * set_taskrole
150 * pre-condition: coalition locked
151 * pre-condition: task added to coalition's task list,
152 * active_count >= 1 (at least the given task is active)
153 */
154 kern_return_t (*set_taskrole)(coalition_t coal, task_t task, int role);
155
156 /*
157 * get_taskrole
158 * pre-condition: coalition locked
159 * pre-condition: task added to coalition's task list,
160 * active_count >= 1 (at least the given task is active)
161 */
162 int (*get_taskrole)(coalition_t coal, task_t task);
163
164 /*
165 * iterate_tasks
166 * pre-condition: coalition locked
167 * if the callback returns true, stop iteration and return the task
168 */
169 task_t (*iterate_tasks)(coalition_t coal, coalition_for_each_task_block_t);
170 };
171
172 /*
173 * COALITION_TYPE_RESOURCE
174 */
175
176 static kern_return_t i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient);
177 static void i_coal_resource_dealloc(coalition_t coal);
178 static kern_return_t i_coal_resource_adopt_task(coalition_t coal, task_t task);
179 static kern_return_t i_coal_resource_remove_task(coalition_t coal, task_t task);
180 static kern_return_t i_coal_resource_set_taskrole(coalition_t coal,
181 task_t task, int role);
182 static int i_coal_resource_get_taskrole(coalition_t coal, task_t task);
183 static task_t i_coal_resource_iterate_tasks(coalition_t coal,
184 coalition_for_each_task_block_t block);
185
186 /*
187 * Ensure COALITION_NUM_THREAD_QOS_TYPES defined in mach/coalition.h still
188 * matches THREAD_QOS_LAST defined in mach/thread_policy.h
189 */
190 static_assert(COALITION_NUM_THREAD_QOS_TYPES == THREAD_QOS_LAST);
191
192 struct i_resource_coalition {
193 /*
194 * This keeps track of resource utilization of tasks that are no longer active
195 * in the coalition and is updated when a task is removed from the coalition.
196 */
197 ledger_t ledger;
198 uint64_t bytesread;
199 uint64_t byteswritten;
200 uint64_t energy;
201 uint64_t gpu_time;
202 uint64_t logical_immediate_writes;
203 uint64_t logical_deferred_writes;
204 uint64_t logical_invalidated_writes;
205 uint64_t logical_metadata_writes;
206 uint64_t logical_immediate_writes_to_external;
207 uint64_t logical_deferred_writes_to_external;
208 uint64_t logical_invalidated_writes_to_external;
209 uint64_t logical_metadata_writes_to_external;
210 uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per effective QoS class */
211 uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per requested QoS class */
212 uint64_t cpu_instructions;
213 uint64_t cpu_cycles;
214 uint64_t ane_mach_time;
215 uint64_t ane_energy_nj;
216
217 _Atomic uint64_t gpu_energy_nj;
218 _Atomic uint64_t gpu_energy_nj_billed_to_me;
219 _Atomic uint64_t gpu_energy_nj_billed_to_others;
220
221 struct recount_coalition co_recount;
222
223 uint64_t task_count; /* tasks that have started in this coalition */
224 uint64_t dead_task_count; /* tasks that have exited in this coalition;
225 * subtract from task_count to get count
226 * of "active" tasks */
227 /*
228 * Count the length of time this coalition had at least one active task.
229 * This can be a 'denominator' to turn e.g. cpu_time to %cpu.
230 * */
231 uint64_t last_became_nonempty_time;
232 uint64_t time_nonempty;
233
234 queue_head_t tasks; /* List of active tasks in the coalition */
235 /*
236 * This ledger is used for triggering resource exception. For the tracked resources, this is updated
237 * when the member tasks' resource usage changes.
238 */
239 ledger_t resource_monitor_ledger;
240 #if CONFIG_PHYS_WRITE_ACCT
241 uint64_t fs_metadata_writes;
242 #endif /* CONFIG_PHYS_WRITE_ACCT */
243 };
244
245 /*
246 * COALITION_TYPE_JETSAM
247 */
248
249 static kern_return_t i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient);
250 static void i_coal_jetsam_dealloc(coalition_t coal);
251 static kern_return_t i_coal_jetsam_adopt_task(coalition_t coal, task_t task);
252 static kern_return_t i_coal_jetsam_remove_task(coalition_t coal, task_t task);
253 static kern_return_t i_coal_jetsam_set_taskrole(coalition_t coal,
254 task_t task, int role);
255 int i_coal_jetsam_get_taskrole(coalition_t coal, task_t task);
256 static task_t i_coal_jetsam_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block);
257
258 struct i_jetsam_coalition {
259 task_t leader;
260 queue_head_t extensions;
261 queue_head_t services;
262 queue_head_t other;
263 struct thread_group *thread_group;
264 bool swap_enabled;
265
266 struct coalition_requested_policy {
267 uint64_t crp_darwinbg :1, /* marked as darwinbg via setpriority */
268 crp_reserved :63;
269 } c_requested_policy;
270 struct coalition_effective_policy {
271 uint64_t cep_darwinbg :1, /* marked as darwinbg via setpriority */
272 cep_reserved :63;
273 } c_effective_policy;
274 };
275
276
277 /*
278 * main coalition structure
279 */
280 struct coalition {
281 uint64_t id; /* monotonically increasing */
282 uint32_t type;
283 uint32_t role; /* default task role (background, adaptive, interactive, etc) */
284 os_ref_atomic_t ref_count; /* Number of references to the memory containing this struct */
285 uint32_t active_count; /* Number of members of (tasks in) the
286 * coalition, plus vouchers referring
287 * to the coalition */
288 uint32_t focal_task_count; /* Number of TASK_FOREGROUND_APPLICATION tasks in the coalition */
289 uint32_t nonfocal_task_count; /* Number of TASK_BACKGROUND_APPLICATION tasks in the coalition */
290 uint32_t game_task_count; /* Number of GAME_MODE tasks in the coalition */
291 uint32_t carplay_task_count; /* Number of CARPLAY_MODE tasks in the coalition */
292
293 /* coalition flags */
294 uint32_t privileged : 1; /* Members of this coalition may create
295 * and manage coalitions and may posix_spawn
296 * processes into selected coalitions */
297 /* ast? */
298 /* voucher */
299 uint32_t termrequested : 1; /* launchd has requested termination when coalition becomes empty */
300 uint32_t terminated : 1; /* coalition became empty and spawns are now forbidden */
301 uint32_t reaped : 1; /* reaped, invisible to userspace, but waiting for ref_count to go to zero */
302 uint32_t notified : 1; /* no-more-processes notification was sent via special port */
303 uint32_t efficient : 1; /* launchd has marked the coalition as efficient */
304 #if DEVELOPMENT || DEBUG
305 uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */
306 #endif
307
308 struct smrq_slink link; /* global list of coalitions */
309
310 union {
311 lck_mtx_t lock; /* Coalition lock. */
312 struct smr_node smr_node;
313 };
314
315 /* put coalition type-specific structures here */
316 union {
317 struct i_resource_coalition r;
318 struct i_jetsam_coalition j;
319 };
320 };
321
322 os_refgrp_decl(static, coal_ref_grp, "coalitions", NULL);
323 #define coal_ref_init(coal, c) os_ref_init_count_raw(&(coal)->ref_count, &coal_ref_grp, c)
324 #define coal_ref_count(coal) os_ref_get_count_raw(&(coal)->ref_count)
325 #define coal_ref_try_retain(coal) os_ref_retain_try_raw(&(coal)->ref_count, &coal_ref_grp)
326 #define coal_ref_retain(coal) os_ref_retain_raw(&(coal)->ref_count, &coal_ref_grp)
327 #define coal_ref_release(coal) os_ref_release_raw(&(coal)->ref_count, &coal_ref_grp)
328 #define coal_ref_release_live(coal) os_ref_release_live_raw(&(coal)->ref_count, &coal_ref_grp)
329
330 #define COALITION_HASH_SIZE_MIN 16
331
332 static bool
coal_hash_obj_try_get(void * coal)333 coal_hash_obj_try_get(void *coal)
334 {
335 return coal_ref_try_retain((struct coalition *)coal);
336 }
337
338 SMRH_TRAITS_DEFINE_SCALAR(coal_hash_traits, struct coalition, id, link,
339 .domain = &smr_proc_task,
340 .obj_try_get = coal_hash_obj_try_get);
341
342 /*
343 * register different coalition types:
344 * these must be kept in the order specified in coalition.h
345 */
346 static const struct coalition_type s_coalition_types[COALITION_NUM_TYPES] = {
347 {
348 .type = COALITION_TYPE_RESOURCE,
349 .has_default = 1,
350 .init = i_coal_resource_init,
351 .dealloc = i_coal_resource_dealloc,
352 .adopt_task = i_coal_resource_adopt_task,
353 .remove_task = i_coal_resource_remove_task,
354 .set_taskrole = i_coal_resource_set_taskrole,
355 .get_taskrole = i_coal_resource_get_taskrole,
356 .iterate_tasks = i_coal_resource_iterate_tasks,
357 },
358 {
359 .type = COALITION_TYPE_JETSAM,
360 .has_default = 1,
361 .init = i_coal_jetsam_init,
362 .dealloc = i_coal_jetsam_dealloc,
363 .adopt_task = i_coal_jetsam_adopt_task,
364 .remove_task = i_coal_jetsam_remove_task,
365 .set_taskrole = i_coal_jetsam_set_taskrole,
366 .get_taskrole = i_coal_jetsam_get_taskrole,
367 .iterate_tasks = i_coal_jetsam_iterate_tasks,
368 },
369 };
370
371 static KALLOC_TYPE_DEFINE(coalition_zone, struct coalition, KT_PRIV_ACCT);
372
373 #define coal_call(coal, func, ...) \
374 (s_coalition_types[(coal)->type].func)(coal, ## __VA_ARGS__)
375
376
377 #define coalition_lock(c) lck_mtx_lock(&(c)->lock)
378 #define coalition_unlock(c) lck_mtx_unlock(&(c)->lock)
379
380 /*
381 * Define the coalition type to track focal tasks.
382 * On embedded, track them using jetsam coalitions since they have associated thread
383 * groups which reflect this property as a flag (and pass it down to CLPC).
384 * On non-embedded platforms, since not all coalitions have jetsam coalitions
385 * track focal counts on the resource coalition.
386 */
387 #if !XNU_TARGET_OS_OSX
388 #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_JETSAM
389 #else /* !XNU_TARGET_OS_OSX */
390 #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_RESOURCE
391 #endif /* !XNU_TARGET_OS_OSX */
392
393
394 /*
395 *
396 * Coalition ledger implementation
397 *
398 */
399
400 struct coalition_ledger_indices coalition_ledgers =
401 {.logical_writes = -1, };
402 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor);
403
404 ledger_t
coalition_ledger_get_from_task(task_t task)405 coalition_ledger_get_from_task(task_t task)
406 {
407 ledger_t ledger = LEDGER_NULL;
408 coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE];
409
410 if (coal != NULL && (!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]))) {
411 ledger = coal->r.resource_monitor_ledger;
412 ledger_reference(ledger);
413 }
414 return ledger;
415 }
416
417
418 enum {
419 COALITION_IO_LEDGER_ENABLE,
420 COALITION_IO_LEDGER_DISABLE
421 };
422
423 void
coalition_io_monitor_ctl(struct coalition * coalition,uint32_t flags,int64_t limit)424 coalition_io_monitor_ctl(struct coalition *coalition, uint32_t flags, int64_t limit)
425 {
426 ledger_t ledger = coalition->r.resource_monitor_ledger;
427
428 if (flags == COALITION_IO_LEDGER_ENABLE) {
429 /* Configure the logical I/O ledger */
430 ledger_set_limit(ledger, coalition_ledgers.logical_writes, (limit * 1024 * 1024), 0);
431 ledger_set_period(ledger, coalition_ledgers.logical_writes, (COALITION_LEDGER_MONITOR_INTERVAL_SECS * NSEC_PER_SEC));
432 } else if (flags == COALITION_IO_LEDGER_DISABLE) {
433 ledger_disable_refill(ledger, coalition_ledgers.logical_writes);
434 ledger_disable_callback(ledger, coalition_ledgers.logical_writes);
435 }
436 }
437
438 int
coalition_ledger_set_logical_writes_limit(struct coalition * coalition,int64_t limit)439 coalition_ledger_set_logical_writes_limit(struct coalition *coalition, int64_t limit)
440 {
441 int error = 0;
442
443 /* limit = -1 will be used to disable the limit and the callback */
444 if (limit > COALITION_MAX_LOGICAL_WRITES_LIMIT || limit == 0 || limit < -1) {
445 error = EINVAL;
446 goto out;
447 }
448
449 coalition_lock(coalition);
450 if (limit == -1) {
451 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_DISABLE, limit);
452 } else {
453 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_ENABLE, limit);
454 }
455 coalition_unlock(coalition);
456 out:
457 return error;
458 }
459
460 void __attribute__((noinline))
SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor)461 SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor)
462 {
463 int pid = proc_selfpid();
464 ledger_amount_t new_limit;
465 task_t task = current_task();
466 struct ledger_entry_info lei;
467 kern_return_t kr;
468 ledger_t ledger;
469 struct coalition *coalition = task->coalition[COALITION_TYPE_RESOURCE];
470
471 assert(coalition != NULL);
472 ledger = coalition->r.resource_monitor_ledger;
473
474 switch (flavor) {
475 case FLAVOR_IO_LOGICAL_WRITES:
476 ledger_get_entry_info(ledger, coalition_ledgers.logical_writes, &lei);
477 trace_resource_violation(RMON_LOGWRITES_VIOLATED, &lei);
478 break;
479 default:
480 goto Exit;
481 }
482
483 os_log(OS_LOG_DEFAULT, "Coalition [%lld] caught causing excessive I/O (flavor: %d). Task I/O: %lld MB. [Limit : %lld MB per %lld secs]. Triggered by process [%d]\n",
484 coalition->id, flavor, (lei.lei_balance / (1024 * 1024)), (lei.lei_limit / (1024 * 1024)),
485 (lei.lei_refill_period / NSEC_PER_SEC), pid);
486
487 kr = send_resource_violation(send_disk_writes_violation, task, &lei, kRNFlagsNone);
488 if (kr) {
489 os_log(OS_LOG_DEFAULT, "ERROR %#x returned from send_resource_violation(disk_writes, ...)\n", kr);
490 }
491
492 /*
493 * Continue to monitor the coalition after it hits the initital limit, but increase
494 * the limit exponentially so that we don't spam the listener.
495 */
496 new_limit = (lei.lei_limit / 1024 / 1024) * 4;
497 coalition_lock(coalition);
498 if (new_limit > COALITION_MAX_LOGICAL_WRITES_LIMIT) {
499 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_DISABLE, -1);
500 } else {
501 coalition_io_monitor_ctl(coalition, COALITION_IO_LEDGER_ENABLE, new_limit);
502 }
503 coalition_unlock(coalition);
504
505 Exit:
506 return;
507 }
508
509 void
coalition_io_rate_exceeded(int warning,const void * param0,__unused const void * param1)510 coalition_io_rate_exceeded(int warning, const void *param0, __unused const void *param1)
511 {
512 if (warning == 0) {
513 SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO((int)param0);
514 }
515 }
516
517 void
init_coalition_ledgers(void)518 init_coalition_ledgers(void)
519 {
520 ledger_template_t t;
521 assert(coalition_ledger_template == NULL);
522
523 if ((t = ledger_template_create("Per-coalition ledgers")) == NULL) {
524 panic("couldn't create coalition ledger template");
525 }
526
527 coalition_ledgers.logical_writes = ledger_entry_add(t, "logical_writes", "res", "bytes");
528
529 if (coalition_ledgers.logical_writes < 0) {
530 panic("couldn't create entries for coaliton ledger template");
531 }
532
533 ledger_set_callback(t, coalition_ledgers.logical_writes, coalition_io_rate_exceeded, (void *)FLAVOR_IO_LOGICAL_WRITES, NULL);
534 ledger_template_complete(t);
535
536 coalition_task_ledger_template = ledger_template_copy(task_ledger_template, "Coalition task ledgers");
537
538 if (coalition_task_ledger_template == NULL) {
539 panic("couldn't create coalition task ledger template");
540 }
541
542 ledger_template_complete(coalition_task_ledger_template);
543
544 coalition_ledger_template = t;
545 }
546
547 void
coalition_io_ledger_update(task_t task,int32_t flavor,boolean_t is_credit,uint32_t io_size)548 coalition_io_ledger_update(task_t task, int32_t flavor, boolean_t is_credit, uint32_t io_size)
549 {
550 ledger_t ledger;
551 coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE];
552
553 assert(coal != NULL);
554 ledger = coal->r.resource_monitor_ledger;
555 if (LEDGER_VALID(ledger)) {
556 if (flavor == FLAVOR_IO_LOGICAL_WRITES) {
557 if (is_credit) {
558 ledger_credit(ledger, coalition_ledgers.logical_writes, io_size);
559 } else {
560 ledger_debit(ledger, coalition_ledgers.logical_writes, io_size);
561 }
562 }
563 }
564 }
565
566 static void
coalition_notify_user(uint64_t id,uint32_t flags)567 coalition_notify_user(uint64_t id, uint32_t flags)
568 {
569 mach_port_t user_port;
570 kern_return_t kr;
571
572 kr = host_get_coalition_port(host_priv_self(), &user_port);
573 if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
574 return;
575 }
576
577 coalition_notification(user_port, id, flags);
578 ipc_port_release_send(user_port);
579 }
580
581 /*
582 *
583 * COALITION_TYPE_RESOURCE
584 *
585 */
586 static kern_return_t
i_coal_resource_init(coalition_t coal,boolean_t privileged,boolean_t efficient)587 i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient)
588 {
589 #pragma unused(privileged, efficient)
590
591 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
592
593 recount_coalition_init(&coal->r.co_recount);
594 coal->r.ledger = ledger_instantiate(coalition_task_ledger_template,
595 LEDGER_CREATE_ACTIVE_ENTRIES);
596 if (coal->r.ledger == NULL) {
597 return KERN_RESOURCE_SHORTAGE;
598 }
599
600 coal->r.resource_monitor_ledger = ledger_instantiate(coalition_ledger_template,
601 LEDGER_CREATE_ACTIVE_ENTRIES);
602 if (coal->r.resource_monitor_ledger == NULL) {
603 return KERN_RESOURCE_SHORTAGE;
604 }
605
606 queue_init(&coal->r.tasks);
607
608 return KERN_SUCCESS;
609 }
610
611 static void
i_coal_resource_dealloc(coalition_t coal)612 i_coal_resource_dealloc(coalition_t coal)
613 {
614 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
615
616 recount_coalition_deinit(&coal->r.co_recount);
617 ledger_dereference(coal->r.ledger);
618 ledger_dereference(coal->r.resource_monitor_ledger);
619 }
620
621 static kern_return_t
i_coal_resource_adopt_task(coalition_t coal,task_t task)622 i_coal_resource_adopt_task(coalition_t coal, task_t task)
623 {
624 struct i_resource_coalition *cr;
625
626 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
627 assert(queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]));
628
629 cr = &coal->r;
630 cr->task_count++;
631
632 if (cr->task_count < cr->dead_task_count) {
633 panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)",
634 __func__, coal, coal->id, coal_type_str(coal->type),
635 cr->task_count, cr->dead_task_count);
636 }
637
638 /* If moving from 0->1 active tasks */
639 if (cr->task_count - cr->dead_task_count == 1) {
640 cr->last_became_nonempty_time = mach_absolute_time();
641 }
642
643 /* put the task on the coalition's list of tasks */
644 enqueue_tail(&cr->tasks, &task->task_coalition[COALITION_TYPE_RESOURCE]);
645
646 coal_dbg("Added PID:%d to id:%llu, task_count:%llu, dead_count:%llu, nonempty_time:%llu",
647 task_pid(task), coal->id, cr->task_count, cr->dead_task_count,
648 cr->last_became_nonempty_time);
649
650 return KERN_SUCCESS;
651 }
652
653 static kern_return_t
i_coal_resource_remove_task(coalition_t coal,task_t task)654 i_coal_resource_remove_task(coalition_t coal, task_t task)
655 {
656 struct i_resource_coalition *cr;
657
658 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
659 assert(task->coalition[COALITION_TYPE_RESOURCE] == coal);
660 assert(!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]));
661
662 /*
663 * handle resource coalition accounting rollup for dead tasks
664 */
665 cr = &coal->r;
666
667 cr->dead_task_count++;
668
669 if (cr->task_count < cr->dead_task_count) {
670 panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)",
671 __func__, coal, coal->id, coal_type_str(coal->type), cr->task_count, cr->dead_task_count);
672 }
673
674 /* If moving from 1->0 active tasks */
675 if (cr->task_count - cr->dead_task_count == 0) {
676 uint64_t last_time_nonempty = mach_absolute_time() - cr->last_became_nonempty_time;
677 cr->last_became_nonempty_time = 0;
678 cr->time_nonempty += last_time_nonempty;
679 }
680
681 /* Do not roll up for exec'd task or exec copy task */
682 if (!task_is_exec_copy(task) && !task_did_exec(task)) {
683 ledger_rollup(cr->ledger, task->ledger);
684 cr->bytesread += task->task_io_stats->disk_reads.size;
685 cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
686 #if defined(__x86_64__)
687 cr->gpu_time += task_gpu_utilisation(task);
688 #endif /* defined(__x86_64__) */
689
690 cr->logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes;
691 cr->logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes;
692 cr->logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes;
693 cr->logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes;
694 cr->logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes;
695 cr->logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes;
696 cr->logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes;
697 cr->logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes;
698 #if CONFIG_PHYS_WRITE_ACCT
699 cr->fs_metadata_writes += task->task_fs_metadata_writes;
700 #endif /* CONFIG_PHYS_WRITE_ACCT */
701 task_update_cpu_time_qos_stats(task, cr->cpu_time_eqos, cr->cpu_time_rqos);
702 recount_coalition_rollup_task(&cr->co_recount, &task->tk_recount);
703 }
704
705 /* remove the task from the coalition's list */
706 remqueue(&task->task_coalition[COALITION_TYPE_RESOURCE]);
707 queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]);
708
709 coal_dbg("removed PID:%d from id:%llu, task_count:%llu, dead_count:%llu",
710 task_pid(task), coal->id, cr->task_count, cr->dead_task_count);
711
712 return KERN_SUCCESS;
713 }
714
715 static kern_return_t
i_coal_resource_set_taskrole(__unused coalition_t coal,__unused task_t task,__unused int role)716 i_coal_resource_set_taskrole(__unused coalition_t coal,
717 __unused task_t task, __unused int role)
718 {
719 return KERN_SUCCESS;
720 }
721
722 static int
i_coal_resource_get_taskrole(__unused coalition_t coal,__unused task_t task)723 i_coal_resource_get_taskrole(__unused coalition_t coal, __unused task_t task)
724 {
725 task_t t;
726
727 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
728
729 qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
730 if (t == task) {
731 return COALITION_TASKROLE_UNDEF;
732 }
733 }
734
735 return -1;
736 }
737
738 static task_t
i_coal_resource_iterate_tasks(coalition_t coal,coalition_for_each_task_block_t block)739 i_coal_resource_iterate_tasks(coalition_t coal, coalition_for_each_task_block_t block)
740 {
741 task_t t;
742 assert(coal && coal->type == COALITION_TYPE_RESOURCE);
743
744 qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
745 bool should_return = block(t);
746 if (should_return) {
747 return t;
748 }
749 }
750
751 return TASK_NULL;
752 }
753
754 #if CONFIG_PHYS_WRITE_ACCT
755 extern uint64_t kernel_pm_writes;
756 #endif /* CONFIG_PHYS_WRITE_ACCT */
757
758 kern_return_t
coalition_resource_usage_internal(coalition_t coal,struct coalition_resource_usage * cru_out)759 coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_usage *cru_out)
760 {
761 kern_return_t kr;
762 ledger_amount_t credit, debit;
763 int i;
764
765 if (coal->type != COALITION_TYPE_RESOURCE) {
766 return KERN_INVALID_ARGUMENT;
767 }
768
769 /* Return KERN_INVALID_ARGUMENT for Corpse coalition */
770 for (i = 0; i < COALITION_NUM_TYPES; i++) {
771 if (coal == corpse_coalition[i]) {
772 return KERN_INVALID_ARGUMENT;
773 }
774 }
775
776 ledger_t sum_ledger = ledger_instantiate(coalition_task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES);
777 if (sum_ledger == LEDGER_NULL) {
778 return KERN_RESOURCE_SHORTAGE;
779 }
780
781 coalition_lock(coal);
782
783 /*
784 * Start with the coalition's ledger, which holds the totals from all
785 * the dead tasks.
786 */
787 ledger_rollup(sum_ledger, coal->r.ledger);
788 uint64_t bytesread = coal->r.bytesread;
789 uint64_t byteswritten = coal->r.byteswritten;
790 uint64_t gpu_time = coal->r.gpu_time;
791 uint64_t logical_immediate_writes = coal->r.logical_immediate_writes;
792 uint64_t logical_deferred_writes = coal->r.logical_deferred_writes;
793 uint64_t logical_invalidated_writes = coal->r.logical_invalidated_writes;
794 uint64_t logical_metadata_writes = coal->r.logical_metadata_writes;
795 uint64_t logical_immediate_writes_to_external = coal->r.logical_immediate_writes_to_external;
796 uint64_t logical_deferred_writes_to_external = coal->r.logical_deferred_writes_to_external;
797 uint64_t logical_invalidated_writes_to_external = coal->r.logical_invalidated_writes_to_external;
798 uint64_t logical_metadata_writes_to_external = coal->r.logical_metadata_writes_to_external;
799 uint64_t ane_mach_time = os_atomic_load(&coal->r.ane_mach_time, relaxed);
800 uint64_t ane_energy_nj = os_atomic_load(&coal->r.ane_energy_nj, relaxed);
801 #if CONFIG_PHYS_WRITE_ACCT
802 uint64_t fs_metadata_writes = coal->r.fs_metadata_writes;
803 #endif /* CONFIG_PHYS_WRITE_ACCT */
804 int64_t conclave_mem = 0;
805 int64_t cpu_time_billed_to_me = 0;
806 int64_t cpu_time_billed_to_others = 0;
807 int64_t energy_billed_to_me = 0;
808 int64_t energy_billed_to_others = 0;
809 int64_t phys_footprint = 0;
810 struct recount_usage stats_sum = { 0 };
811 struct recount_usage stats_perf_only = { 0 };
812 recount_coalition_usage_perf_only(&coal->r.co_recount, &stats_sum,
813 &stats_perf_only);
814 uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 };
815 uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 };
816 /*
817 * Add to that all the active tasks' ledgers. Tasks cannot deallocate
818 * out from under us, since we hold the coalition lock.
819 */
820 task_t task;
821 qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) {
822 /*
823 * Rolling up stats for exec copy task or exec'd task will lead to double accounting.
824 * Cannot take task lock after taking coaliton lock
825 */
826 if (task_is_exec_copy(task) || task_did_exec(task)) {
827 continue;
828 }
829
830 ledger_rollup(sum_ledger, task->ledger);
831 bytesread += task->task_io_stats->disk_reads.size;
832 byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size;
833 #if defined(__x86_64__)
834 gpu_time += task_gpu_utilisation(task);
835 #endif /* defined(__x86_64__) */
836
837 logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes;
838 logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes;
839 logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes;
840 logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes;
841 logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes;
842 logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes;
843 logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes;
844 logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes;
845 #if CONFIG_PHYS_WRITE_ACCT
846 fs_metadata_writes += task->task_fs_metadata_writes;
847 #endif /* CONFIG_PHYS_WRITE_ACCT */
848
849 task_update_cpu_time_qos_stats(task, cpu_time_eqos, cpu_time_rqos);
850 recount_task_usage_perf_only(task, &stats_sum, &stats_perf_only);
851 }
852
853 kr = ledger_get_balance(sum_ledger, task_ledgers.conclave_mem, (int64_t *)&conclave_mem);
854 if (kr != KERN_SUCCESS || conclave_mem < 0) {
855 conclave_mem = 0;
856 }
857
858 kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_me, (int64_t *)&cpu_time_billed_to_me);
859 if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) {
860 cpu_time_billed_to_me = 0;
861 }
862
863 kr = ledger_get_balance(sum_ledger, task_ledgers.cpu_time_billed_to_others, (int64_t *)&cpu_time_billed_to_others);
864 if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) {
865 cpu_time_billed_to_others = 0;
866 }
867
868 kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_me, (int64_t *)&energy_billed_to_me);
869 if (kr != KERN_SUCCESS || energy_billed_to_me < 0) {
870 energy_billed_to_me = 0;
871 }
872
873 kr = ledger_get_balance(sum_ledger, task_ledgers.energy_billed_to_others, (int64_t *)&energy_billed_to_others);
874 if (kr != KERN_SUCCESS || energy_billed_to_others < 0) {
875 energy_billed_to_others = 0;
876 }
877
878 kr = ledger_get_balance(sum_ledger, task_ledgers.phys_footprint, (int64_t *)&phys_footprint);
879 if (kr != KERN_SUCCESS || phys_footprint < 0) {
880 phys_footprint = 0;
881 }
882
883 /* collect information from the coalition itself */
884 cru_out->tasks_started = coal->r.task_count;
885 cru_out->tasks_exited = coal->r.dead_task_count;
886
887 uint64_t time_nonempty = coal->r.time_nonempty;
888 uint64_t last_became_nonempty_time = coal->r.last_became_nonempty_time;
889
890 cru_out->gpu_energy_nj = os_atomic_load(&coal->r.gpu_energy_nj, relaxed);
891 cru_out->gpu_energy_nj_billed_to_me = os_atomic_load(&coal->r.gpu_energy_nj_billed_to_me, relaxed);
892 cru_out->gpu_energy_nj_billed_to_others = os_atomic_load(&coal->r.gpu_energy_nj_billed_to_others, relaxed);
893
894 coalition_unlock(coal);
895
896 /* Copy the totals out of sum_ledger */
897 kr = ledger_get_entries(sum_ledger, task_ledgers.cpu_time,
898 &credit, &debit);
899 if (kr != KERN_SUCCESS) {
900 credit = 0;
901 }
902 cru_out->conclave_mem = conclave_mem;
903 cru_out->cpu_time = credit;
904 cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me;
905 cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others;
906 cru_out->energy_billed_to_me = (uint64_t)energy_billed_to_me;
907 cru_out->energy_billed_to_others = (uint64_t)energy_billed_to_others;
908 cru_out->phys_footprint = phys_footprint;
909
910 kr = ledger_get_entries(sum_ledger, task_ledgers.interrupt_wakeups,
911 &credit, &debit);
912 if (kr != KERN_SUCCESS) {
913 credit = 0;
914 }
915 cru_out->interrupt_wakeups = credit;
916
917 kr = ledger_get_entries(sum_ledger, task_ledgers.platform_idle_wakeups,
918 &credit, &debit);
919 if (kr != KERN_SUCCESS) {
920 credit = 0;
921 }
922 cru_out->platform_idle_wakeups = credit;
923
924 cru_out->bytesread = bytesread;
925 cru_out->byteswritten = byteswritten;
926 cru_out->gpu_time = gpu_time;
927 cru_out->ane_mach_time = ane_mach_time;
928 cru_out->ane_energy_nj = ane_energy_nj;
929 cru_out->logical_immediate_writes = logical_immediate_writes;
930 cru_out->logical_deferred_writes = logical_deferred_writes;
931 cru_out->logical_invalidated_writes = logical_invalidated_writes;
932 cru_out->logical_metadata_writes = logical_metadata_writes;
933 cru_out->logical_immediate_writes_to_external = logical_immediate_writes_to_external;
934 cru_out->logical_deferred_writes_to_external = logical_deferred_writes_to_external;
935 cru_out->logical_invalidated_writes_to_external = logical_invalidated_writes_to_external;
936 cru_out->logical_metadata_writes_to_external = logical_metadata_writes_to_external;
937 #if CONFIG_PHYS_WRITE_ACCT
938 cru_out->fs_metadata_writes = fs_metadata_writes;
939 #else
940 cru_out->fs_metadata_writes = 0;
941 #endif /* CONFIG_PHYS_WRITE_ACCT */
942 cru_out->cpu_time_eqos_len = COALITION_NUM_THREAD_QOS_TYPES;
943 memcpy(cru_out->cpu_time_eqos, cpu_time_eqos, sizeof(cru_out->cpu_time_eqos));
944
945 cru_out->cpu_ptime = recount_usage_time_mach(&stats_perf_only);
946 #if CONFIG_PERVASIVE_CPI
947 cru_out->cpu_instructions = recount_usage_instructions(&stats_sum);
948 cru_out->cpu_cycles = recount_usage_cycles(&stats_sum);
949 cru_out->cpu_pinstructions = recount_usage_instructions(&stats_perf_only);
950 cru_out->cpu_pcycles = recount_usage_cycles(&stats_perf_only);
951 #endif // CONFIG_PERVASIVE_CPI
952
953 ledger_dereference(sum_ledger);
954 sum_ledger = LEDGER_NULL;
955
956 #if CONFIG_PERVASIVE_ENERGY
957 cru_out->energy = stats_sum.ru_energy_nj;
958 #endif /* CONFIG_PERVASIVE_ENERGY */
959
960 #if CONFIG_PHYS_WRITE_ACCT
961 // kernel_pm_writes are only recorded under kernel_task coalition
962 if (coalition_id(coal) == COALITION_ID_KERNEL) {
963 cru_out->pm_writes = kernel_pm_writes;
964 } else {
965 cru_out->pm_writes = 0;
966 }
967 #else
968 cru_out->pm_writes = 0;
969 #endif /* CONFIG_PHYS_WRITE_ACCT */
970
971 if (last_became_nonempty_time) {
972 time_nonempty += mach_absolute_time() - last_became_nonempty_time;
973 }
974 absolutetime_to_nanoseconds(time_nonempty, &cru_out->time_nonempty);
975
976 return KERN_SUCCESS;
977 }
978
979 bool
coalition_add_to_gpu_energy(uint64_t coalition_id,coalition_gpu_energy_t which,uint64_t energy)980 coalition_add_to_gpu_energy(uint64_t coalition_id,
981 coalition_gpu_energy_t which, uint64_t energy)
982 {
983 assert(coalition_id != 0);
984 bool exists = false;
985
986 smrh_key_t key = SMRH_SCALAR_KEY(coalition_id);
987
988 smrht_enter(&coal_hash_traits);
989
990 coalition_t coal = smr_hash_entered_find(&coalition_hash, key, &coal_hash_traits);
991 /* Preemption is disabled, and until we leave, coalition_free won't be called. */
992
993 if (coal != COALITION_NULL &&
994 coal->type == COALITION_TYPE_RESOURCE &&
995 !coal->terminated && !coal->reaped) {
996 /*
997 * After termination, there are no more member tasks and
998 * launchd is about to capture the final snapshot, so
999 * a terminated energy id considered invalid.
1000 *
1001 * Note that we don't lock against the termination transition
1002 * here, so it's possible for a race to occur where new energy
1003 * is accounted past the final snapshot.
1004 */
1005 exists = true;
1006
1007 if (which & CGE_SELF) {
1008 os_atomic_add(&coal->r.gpu_energy_nj, energy, relaxed);
1009 }
1010
1011 if (which & CGE_BILLED) {
1012 os_atomic_add(&coal->r.gpu_energy_nj_billed_to_me,
1013 energy, relaxed);
1014 }
1015
1016 if (which & CGE_OTHERS) {
1017 os_atomic_add(&coal->r.gpu_energy_nj_billed_to_others,
1018 energy, relaxed);
1019 }
1020 }
1021
1022 smrht_leave(&coal_hash_traits);
1023
1024 return exists;
1025 }
1026
1027
1028 kern_return_t
coalition_debug_info_internal(coalition_t coal,struct coalinfo_debuginfo * c_debuginfo)1029 coalition_debug_info_internal(coalition_t coal,
1030 struct coalinfo_debuginfo *c_debuginfo)
1031 {
1032 /* Return KERN_INVALID_ARGUMENT for Corpse coalition */
1033 for (int i = 0; i < COALITION_NUM_TYPES; i++) {
1034 if (coal == corpse_coalition[i]) {
1035 return KERN_INVALID_ARGUMENT;
1036 }
1037 }
1038
1039 if (coal->type == COALITION_FOCAL_TASKS_ACCOUNTING) {
1040 c_debuginfo->focal_task_count = coal->focal_task_count;
1041 c_debuginfo->nonfocal_task_count = coal->nonfocal_task_count;
1042 c_debuginfo->game_task_count = coal->game_task_count;
1043 c_debuginfo->carplay_task_count = coal->carplay_task_count;
1044 }
1045
1046 #if CONFIG_THREAD_GROUPS
1047 struct thread_group * group = coalition_get_thread_group(coal);
1048
1049 if (group != NULL) {
1050 c_debuginfo->thread_group_id = thread_group_id(group);
1051 c_debuginfo->thread_group_flags = thread_group_get_flags(group);
1052 c_debuginfo->thread_group_recommendation = thread_group_recommendation(group);
1053 }
1054 #endif /* CONFIG_THREAD_GROUPS */
1055
1056 return KERN_SUCCESS;
1057 }
1058
1059 /*
1060 *
1061 * COALITION_TYPE_JETSAM
1062 *
1063 */
1064 static kern_return_t
i_coal_jetsam_init(coalition_t coal,boolean_t privileged,boolean_t efficient)1065 i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient)
1066 {
1067 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1068 (void)privileged;
1069 (void)efficient;
1070
1071 coal->j.leader = TASK_NULL;
1072 queue_head_init(coal->j.extensions);
1073 queue_head_init(coal->j.services);
1074 queue_head_init(coal->j.other);
1075
1076 #if CONFIG_THREAD_GROUPS
1077 switch (coal->role) {
1078 case COALITION_ROLE_SYSTEM:
1079 coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_SYSTEM);
1080 break;
1081 case COALITION_ROLE_BACKGROUND:
1082 coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_BACKGROUND);
1083 break;
1084 default:
1085 coal->j.thread_group = thread_group_create_and_retain(efficient ? THREAD_GROUP_FLAGS_EFFICIENT : THREAD_GROUP_FLAGS_DEFAULT);
1086 }
1087 assert(coal->j.thread_group != NULL);
1088 #endif
1089 return KERN_SUCCESS;
1090 }
1091
1092 static void
i_coal_jetsam_dealloc(__unused coalition_t coal)1093 i_coal_jetsam_dealloc(__unused coalition_t coal)
1094 {
1095 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1096
1097 /* the coalition should be completely clear at this point */
1098 assert(queue_empty(&coal->j.extensions));
1099 assert(queue_empty(&coal->j.services));
1100 assert(queue_empty(&coal->j.other));
1101 assert(coal->j.leader == TASK_NULL);
1102
1103 #if CONFIG_THREAD_GROUPS
1104 /* disassociate from the thread group */
1105 assert(coal->j.thread_group != NULL);
1106 thread_group_release(coal->j.thread_group);
1107 coal->j.thread_group = NULL;
1108 #endif
1109 }
1110
1111 static kern_return_t
i_coal_jetsam_adopt_task(coalition_t coal,task_t task)1112 i_coal_jetsam_adopt_task(coalition_t coal, task_t task)
1113 {
1114 struct i_jetsam_coalition *cj;
1115 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1116
1117 cj = &coal->j;
1118
1119 assert(queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM]));
1120
1121 /* put each task initially in the "other" list */
1122 enqueue_tail(&cj->other, &task->task_coalition[COALITION_TYPE_JETSAM]);
1123 coal_dbg("coalition %lld adopted PID:%d as UNDEF",
1124 coal->id, task_pid(task));
1125
1126 return KERN_SUCCESS;
1127 }
1128
1129 static kern_return_t
i_coal_jetsam_remove_task(coalition_t coal,task_t task)1130 i_coal_jetsam_remove_task(coalition_t coal, task_t task)
1131 {
1132 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1133 assert(task->coalition[COALITION_TYPE_JETSAM] == coal);
1134
1135 coal_dbg("removing PID:%d from coalition id:%lld",
1136 task_pid(task), coal->id);
1137
1138 if (task == coal->j.leader) {
1139 coal->j.leader = NULL;
1140 coal_dbg(" PID:%d was the leader!", task_pid(task));
1141 } else {
1142 assert(!queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM]));
1143 }
1144
1145 /* remove the task from the specific coalition role queue */
1146 remqueue(&task->task_coalition[COALITION_TYPE_JETSAM]);
1147 queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]);
1148
1149 return KERN_SUCCESS;
1150 }
1151
1152 static kern_return_t
i_coal_jetsam_set_taskrole(coalition_t coal,task_t task,int role)1153 i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role)
1154 {
1155 struct i_jetsam_coalition *cj;
1156 queue_t q = NULL;
1157 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1158 assert(task->coalition[COALITION_TYPE_JETSAM] == coal);
1159
1160 cj = &coal->j;
1161
1162 switch (role) {
1163 case COALITION_TASKROLE_LEADER:
1164 coal_dbg("setting PID:%d as LEADER of %lld",
1165 task_pid(task), coal->id);
1166 if (cj->leader != TASK_NULL) {
1167 /* re-queue the exiting leader onto the "other" list */
1168 coal_dbg(" re-queue existing leader (%d) as OTHER",
1169 task_pid(cj->leader));
1170 re_queue_tail(&cj->other, &cj->leader->task_coalition[COALITION_TYPE_JETSAM]);
1171 }
1172 /*
1173 * remove the task from the "other" list
1174 * (where it was put by default)
1175 */
1176 remqueue(&task->task_coalition[COALITION_TYPE_JETSAM]);
1177 queue_chain_init(task->task_coalition[COALITION_TYPE_JETSAM]);
1178
1179 /* set the coalition leader */
1180 cj->leader = task;
1181 break;
1182 case COALITION_TASKROLE_XPC:
1183 coal_dbg("setting PID:%d as XPC in %lld",
1184 task_pid(task), coal->id);
1185 q = (queue_t)&cj->services;
1186 break;
1187 case COALITION_TASKROLE_EXT:
1188 coal_dbg("setting PID:%d as EXT in %lld",
1189 task_pid(task), coal->id);
1190 q = (queue_t)&cj->extensions;
1191 break;
1192 case COALITION_TASKROLE_NONE:
1193 /*
1194 * Tasks with a role of "none" should fall through to an
1195 * undefined role so long as the task is currently a member
1196 * of the coalition. This scenario can happen if a task is
1197 * killed (usually via jetsam) during exec.
1198 */
1199 if (task->coalition[COALITION_TYPE_JETSAM] != coal) {
1200 panic("%s: task %p attempting to set role %d "
1201 "in coalition %p to which it does not belong!", __func__, task, role, coal);
1202 }
1203 OS_FALLTHROUGH;
1204 case COALITION_TASKROLE_UNDEF:
1205 coal_dbg("setting PID:%d as UNDEF in %lld",
1206 task_pid(task), coal->id);
1207 q = (queue_t)&cj->other;
1208 break;
1209 default:
1210 panic("%s: invalid role(%d) for task", __func__, role);
1211 return KERN_INVALID_ARGUMENT;
1212 }
1213
1214 if (q != NULL) {
1215 re_queue_tail(q, &task->task_coalition[COALITION_TYPE_JETSAM]);
1216 }
1217
1218 return KERN_SUCCESS;
1219 }
1220
1221 int
i_coal_jetsam_get_taskrole(coalition_t coal,task_t task)1222 i_coal_jetsam_get_taskrole(coalition_t coal, task_t task)
1223 {
1224 struct i_jetsam_coalition *cj;
1225 task_t t;
1226
1227 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1228 assert(task->coalition[COALITION_TYPE_JETSAM] == coal);
1229
1230 cj = &coal->j;
1231
1232 if (task == cj->leader) {
1233 return COALITION_TASKROLE_LEADER;
1234 }
1235
1236 qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) {
1237 if (t == task) {
1238 return COALITION_TASKROLE_XPC;
1239 }
1240 }
1241
1242 qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) {
1243 if (t == task) {
1244 return COALITION_TASKROLE_EXT;
1245 }
1246 }
1247
1248 qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) {
1249 if (t == task) {
1250 return COALITION_TASKROLE_UNDEF;
1251 }
1252 }
1253
1254 /* task not in the coalition?! */
1255 return COALITION_TASKROLE_NONE;
1256 }
1257
1258 static task_t
i_coal_jetsam_iterate_tasks(coalition_t coal,coalition_for_each_task_block_t block)1259 i_coal_jetsam_iterate_tasks(
1260 coalition_t coal,
1261 coalition_for_each_task_block_t block)
1262 {
1263 struct i_jetsam_coalition *cj;
1264 task_t t;
1265
1266 assert(coal && coal->type == COALITION_TYPE_JETSAM);
1267
1268 cj = &coal->j;
1269
1270 if (cj->leader) {
1271 t = cj->leader;
1272 bool should_return = block( t);
1273 if (should_return) {
1274 return t;
1275 }
1276 }
1277
1278 qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) {
1279 bool should_return = block(t);
1280 if (should_return) {
1281 return t;
1282 }
1283 }
1284
1285 qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) {
1286 bool should_return = block(t);
1287 if (should_return) {
1288 return t;
1289 }
1290 }
1291
1292 qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) {
1293 bool should_return = block(t);
1294 if (should_return) {
1295 return t;
1296 }
1297 }
1298
1299 return TASK_NULL;
1300 }
1301
1302
1303 /*
1304 *
1305 * Main Coalition implementation
1306 *
1307 */
1308
1309 /*
1310 * coalition_create_internal
1311 * Returns: New coalition object, referenced for the caller and unlocked.
1312 * Condition: coalitions_list_lock must be UNLOCKED.
1313 */
1314 kern_return_t
coalition_create_internal(int type,int role,boolean_t privileged,boolean_t efficient,coalition_t * out,uint64_t * coalition_id)1315 coalition_create_internal(int type, int role, boolean_t privileged, boolean_t efficient, coalition_t *out, uint64_t *coalition_id)
1316 {
1317 kern_return_t kr;
1318 struct coalition *new_coal;
1319 uint64_t cid;
1320
1321 if (type < 0 || type > COALITION_TYPE_MAX) {
1322 return KERN_INVALID_ARGUMENT;
1323 }
1324
1325 new_coal = zalloc_flags(coalition_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1326
1327 new_coal->type = type;
1328 new_coal->role = role;
1329
1330 /* initialize type-specific resources */
1331 kr = coal_call(new_coal, init, privileged, efficient);
1332 if (kr != KERN_SUCCESS) {
1333 zfree(coalition_zone, new_coal);
1334 return kr;
1335 }
1336
1337 /* One for caller, one for coalitions list */
1338 coal_ref_init(new_coal, 2);
1339
1340 new_coal->privileged = privileged ? TRUE : FALSE;
1341 new_coal->efficient = efficient ? TRUE : FALSE;
1342 #if DEVELOPMENT || DEBUG
1343 new_coal->should_notify = 1;
1344 #endif
1345
1346 lck_mtx_init(&new_coal->lock, &coalitions_lck_grp, LCK_ATTR_NULL);
1347
1348 lck_mtx_lock(&coalitions_list_lock);
1349 new_coal->id = cid = coalition_next_id++;
1350
1351 smr_hash_serialized_insert(&coalition_hash, &new_coal->link,
1352 &coal_hash_traits);
1353
1354 #if CONFIG_THREAD_GROUPS
1355 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW),
1356 new_coal->id, new_coal->type,
1357 (new_coal->type == COALITION_TYPE_JETSAM && new_coal->j.thread_group) ?
1358 thread_group_get_id(new_coal->j.thread_group) : 0);
1359
1360 #else
1361 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW),
1362 new_coal->id, new_coal->type);
1363 #endif
1364
1365 if (smr_hash_serialized_should_grow(&coalition_hash, 1, 4)) {
1366 /* grow if more more than 4 elements per 1 bucket */
1367 smr_hash_grow_and_unlock(&coalition_hash,
1368 &coalitions_list_lock, &coal_hash_traits);
1369 } else {
1370 lck_mtx_unlock(&coalitions_list_lock);
1371 }
1372
1373 coal_dbg("id:%llu, type:%s", cid, coal_type_str(type));
1374
1375 if (coalition_id != NULL) {
1376 *coalition_id = cid;
1377 }
1378
1379 *out = new_coal;
1380 return KERN_SUCCESS;
1381 }
1382
1383 static void
coalition_free(struct smr_node * node)1384 coalition_free(struct smr_node *node)
1385 {
1386 struct coalition *coal;
1387
1388 coal = __container_of(node, struct coalition, smr_node);
1389 zfree(coalition_zone, coal);
1390 }
1391
1392 static __attribute__((noinline)) void
coalition_retire(coalition_t coal)1393 coalition_retire(coalition_t coal)
1394 {
1395 coalition_lock(coal);
1396
1397 coal_dbg("id:%llu type:%s active_count:%u%s",
1398 coal->id, coal_type_str(coal->type), coal->active_count,
1399 rc <= 0 ? ", will deallocate now" : "");
1400
1401 assert(coal->termrequested);
1402 assert(coal->terminated);
1403 assert(coal->active_count == 0);
1404 assert(coal->reaped);
1405 assert(coal->focal_task_count == 0);
1406 assert(coal->nonfocal_task_count == 0);
1407 assert(coal->game_task_count == 0);
1408 assert(coal->carplay_task_count == 0);
1409 #if CONFIG_THREAD_GROUPS
1410 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE),
1411 coal->id, coal->type,
1412 coal->type == COALITION_TYPE_JETSAM ?
1413 coal->j.thread_group : 0);
1414 #else
1415 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE),
1416 coal->id, coal->type);
1417 #endif
1418
1419 coal_call(coal, dealloc);
1420
1421 coalition_unlock(coal);
1422
1423 lck_mtx_destroy(&coal->lock, &coalitions_lck_grp);
1424
1425 smr_proc_task_call(&coal->smr_node, sizeof(*coal), coalition_free);
1426 }
1427
1428 /*
1429 * coalition_retain
1430 * */
1431 void
coalition_retain(coalition_t coal)1432 coalition_retain(coalition_t coal)
1433 {
1434 coal_ref_retain(coal);
1435 }
1436
1437 /*
1438 * coalition_release
1439 * Condition: coalition must be UNLOCKED.
1440 * */
1441 void
coalition_release(coalition_t coal)1442 coalition_release(coalition_t coal)
1443 {
1444 if (coal_ref_release(coal) > 0) {
1445 return;
1446 }
1447
1448 coalition_retire(coal);
1449 }
1450
1451 /*
1452 * coalition_find_by_id
1453 * Returns: Coalition object with specified id, referenced.
1454 */
1455 coalition_t
coalition_find_by_id(uint64_t cid)1456 coalition_find_by_id(uint64_t cid)
1457 {
1458 smrh_key_t key = SMRH_SCALAR_KEY(cid);
1459
1460 if (cid == 0) {
1461 return COALITION_NULL;
1462 }
1463
1464 return smr_hash_get(&coalition_hash, key, &coal_hash_traits);
1465 }
1466
1467 /*
1468 * coalition_find_and_activate_by_id
1469 * Returns: Coalition object with specified id, referenced, and activated.
1470 * This is the function to use when putting a 'new' thing into a coalition,
1471 * like posix_spawn of an XPC service by launchd.
1472 * See also coalition_extend_active.
1473 */
1474 coalition_t
coalition_find_and_activate_by_id(uint64_t cid)1475 coalition_find_and_activate_by_id(uint64_t cid)
1476 {
1477 coalition_t coal = coalition_find_by_id(cid);
1478
1479 if (coal == COALITION_NULL) {
1480 return COALITION_NULL;
1481 }
1482
1483 coalition_lock(coal);
1484
1485 if (coal->reaped || coal->terminated) {
1486 /* Too late to put something new into this coalition, it's
1487 * already on its way out the door */
1488 coalition_unlock(coal);
1489 coalition_release(coal);
1490 return COALITION_NULL;
1491 }
1492
1493 coal->active_count++;
1494
1495 #if COALITION_DEBUG
1496 uint32_t rc = coal_ref_count(coal);
1497 uint32_t ac = coal->active_count;
1498 #endif
1499 coalition_unlock(coal);
1500
1501 coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u",
1502 coal->id, coal_type_str(coal->type), rc, ac);
1503
1504 return coal;
1505 }
1506
1507 uint64_t
coalition_id(coalition_t coal)1508 coalition_id(coalition_t coal)
1509 {
1510 assert(coal != COALITION_NULL);
1511 return coal->id;
1512 }
1513
1514 void
task_coalition_ids(task_t task,uint64_t ids[COALITION_NUM_TYPES])1515 task_coalition_ids(task_t task, uint64_t ids[COALITION_NUM_TYPES])
1516 {
1517 int i;
1518 for (i = 0; i < COALITION_NUM_TYPES; i++) {
1519 if (task->coalition[i]) {
1520 ids[i] = task->coalition[i]->id;
1521 } else {
1522 ids[i] = 0;
1523 }
1524 }
1525 }
1526
1527 void
task_coalition_roles(task_t task,int roles[COALITION_NUM_TYPES])1528 task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES])
1529 {
1530 int i;
1531 memset(roles, 0, COALITION_NUM_TYPES * sizeof(roles[0]));
1532
1533 for (i = 0; i < COALITION_NUM_TYPES; i++) {
1534 if (task->coalition[i]) {
1535 coalition_lock(task->coalition[i]);
1536 roles[i] = coal_call(task->coalition[i],
1537 get_taskrole, task);
1538 coalition_unlock(task->coalition[i]);
1539 } else {
1540 roles[i] = COALITION_TASKROLE_NONE;
1541 }
1542 }
1543 }
1544
1545 int
task_coalition_role_for_type(task_t task,int coalition_type)1546 task_coalition_role_for_type(task_t task, int coalition_type)
1547 {
1548 coalition_t coal;
1549 int role;
1550 if (coalition_type >= COALITION_NUM_TYPES) {
1551 panic("Attempt to call task_coalition_role_for_type with invalid coalition_type: %d\n", coalition_type);
1552 }
1553 coal = task->coalition[coalition_type];
1554 if (coal == NULL) {
1555 return COALITION_TASKROLE_NONE;
1556 }
1557 coalition_lock(coal);
1558 role = coal_call(coal, get_taskrole, task);
1559 coalition_unlock(coal);
1560 return role;
1561 }
1562
1563 int
coalition_type(coalition_t coal)1564 coalition_type(coalition_t coal)
1565 {
1566 return coal->type;
1567 }
1568
1569 boolean_t
coalition_term_requested(coalition_t coal)1570 coalition_term_requested(coalition_t coal)
1571 {
1572 return coal->termrequested;
1573 }
1574
1575 boolean_t
coalition_is_terminated(coalition_t coal)1576 coalition_is_terminated(coalition_t coal)
1577 {
1578 return coal->terminated;
1579 }
1580
1581 boolean_t
coalition_is_reaped(coalition_t coal)1582 coalition_is_reaped(coalition_t coal)
1583 {
1584 return coal->reaped;
1585 }
1586
1587 boolean_t
coalition_is_privileged(coalition_t coal)1588 coalition_is_privileged(coalition_t coal)
1589 {
1590 return coal->privileged || unrestrict_coalition_syscalls;
1591 }
1592
1593 boolean_t
task_is_in_privileged_coalition(task_t task,int type)1594 task_is_in_privileged_coalition(task_t task, int type)
1595 {
1596 if (type < 0 || type > COALITION_TYPE_MAX) {
1597 return FALSE;
1598 }
1599 if (unrestrict_coalition_syscalls) {
1600 return TRUE;
1601 }
1602 if (!task->coalition[type]) {
1603 return FALSE;
1604 }
1605 return task->coalition[type]->privileged;
1606 }
1607
1608 void
task_coalition_update_gpu_stats(task_t task,uint64_t gpu_ns_delta)1609 task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta)
1610 {
1611 coalition_t coal;
1612
1613 assert(task != TASK_NULL);
1614 if (gpu_ns_delta == 0) {
1615 return;
1616 }
1617
1618 coal = task->coalition[COALITION_TYPE_RESOURCE];
1619 assert(coal != COALITION_NULL);
1620
1621 coalition_lock(coal);
1622 coal->r.gpu_time += gpu_ns_delta;
1623 coalition_unlock(coal);
1624 }
1625
1626 void
coalition_update_ane_stats(coalition_t coalition,uint64_t ane_mach_time,uint64_t ane_energy_nj)1627 coalition_update_ane_stats(coalition_t coalition, uint64_t ane_mach_time, uint64_t ane_energy_nj)
1628 {
1629 assert(coalition != COALITION_NULL);
1630
1631 os_atomic_add(&coalition->r.ane_mach_time, ane_mach_time, relaxed);
1632 os_atomic_add(&coalition->r.ane_energy_nj, ane_energy_nj, relaxed);
1633 }
1634
1635 boolean_t
task_coalition_adjust_focal_count(task_t task,int count,uint32_t * new_count)1636 task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count)
1637 {
1638 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1639 if (coal == COALITION_NULL) {
1640 return FALSE;
1641 }
1642
1643 *new_count = os_atomic_add(&coal->focal_task_count, count, relaxed);
1644 assert(*new_count != UINT32_MAX);
1645 return TRUE;
1646 }
1647
1648 uint32_t
task_coalition_focal_count(task_t task)1649 task_coalition_focal_count(task_t task)
1650 {
1651 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1652 if (coal == COALITION_NULL) {
1653 return 0;
1654 }
1655
1656 return coal->focal_task_count;
1657 }
1658
1659 uint32_t
task_coalition_game_mode_count(task_t task)1660 task_coalition_game_mode_count(task_t task)
1661 {
1662 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1663 if (coal == COALITION_NULL) {
1664 return 0;
1665 }
1666
1667 return coal->game_task_count;
1668 }
1669
1670 uint32_t
task_coalition_carplay_mode_count(task_t task)1671 task_coalition_carplay_mode_count(task_t task)
1672 {
1673 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1674 if (coal == COALITION_NULL) {
1675 return 0;
1676 }
1677 return coal->carplay_task_count;
1678 }
1679
1680 boolean_t
task_coalition_adjust_nonfocal_count(task_t task,int count,uint32_t * new_count)1681 task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count)
1682 {
1683 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1684 if (coal == COALITION_NULL) {
1685 return FALSE;
1686 }
1687
1688 *new_count = os_atomic_add(&coal->nonfocal_task_count, count, relaxed);
1689 assert(*new_count != UINT32_MAX);
1690 return TRUE;
1691 }
1692
1693 uint32_t
task_coalition_nonfocal_count(task_t task)1694 task_coalition_nonfocal_count(task_t task)
1695 {
1696 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1697 if (coal == COALITION_NULL) {
1698 return 0;
1699 }
1700
1701 return coal->nonfocal_task_count;
1702 }
1703
1704 bool
task_coalition_adjust_game_mode_count(task_t task,int count,uint32_t * new_count)1705 task_coalition_adjust_game_mode_count(task_t task, int count, uint32_t *new_count)
1706 {
1707 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1708 if (coal == COALITION_NULL) {
1709 return false;
1710 }
1711
1712 *new_count = os_atomic_add(&coal->game_task_count, count, relaxed);
1713 assert(*new_count != UINT32_MAX);
1714 return true;
1715 }
1716
1717 bool
task_coalition_adjust_carplay_mode_count(task_t task,int count,uint32_t * new_count)1718 task_coalition_adjust_carplay_mode_count(task_t task, int count, uint32_t *new_count)
1719 {
1720 coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING];
1721 if (coal == COALITION_NULL) {
1722 return false;
1723 }
1724
1725 *new_count = os_atomic_add(&coal->carplay_task_count, count, relaxed);
1726 assert(*new_count != UINT32_MAX);
1727 return true;
1728 }
1729
1730 #if CONFIG_THREAD_GROUPS
1731
1732 /* Thread group lives as long as the task is holding the coalition reference */
1733 struct thread_group *
task_coalition_get_thread_group(task_t task)1734 task_coalition_get_thread_group(task_t task)
1735 {
1736 coalition_t coal = task->coalition[COALITION_TYPE_JETSAM];
1737 /* return system thread group for non-jetsam coalitions */
1738 if (coal == COALITION_NULL) {
1739 return init_coalition[COALITION_TYPE_JETSAM]->j.thread_group;
1740 }
1741 return coal->j.thread_group;
1742 }
1743
1744
1745 struct thread_group *
kdp_coalition_get_thread_group(coalition_t coal)1746 kdp_coalition_get_thread_group(coalition_t coal)
1747 {
1748 if (coal->type != COALITION_TYPE_JETSAM) {
1749 return NULL;
1750 }
1751 assert(coal->j.thread_group != NULL);
1752 return coal->j.thread_group;
1753 }
1754
1755 /* Thread group lives as long as the coalition reference is held */
1756 struct thread_group *
coalition_get_thread_group(coalition_t coal)1757 coalition_get_thread_group(coalition_t coal)
1758 {
1759 if (coal->type != COALITION_TYPE_JETSAM) {
1760 return NULL;
1761 }
1762 assert(coal->j.thread_group != NULL);
1763 return coal->j.thread_group;
1764 }
1765
1766 /* Donates the thread group reference to the coalition */
1767 void
coalition_set_thread_group(coalition_t coal,struct thread_group * tg)1768 coalition_set_thread_group(coalition_t coal, struct thread_group *tg)
1769 {
1770 assert(coal != COALITION_NULL);
1771 assert(tg != NULL);
1772
1773 if (coal->type != COALITION_TYPE_JETSAM) {
1774 return;
1775 }
1776 struct thread_group *old_tg = coal->j.thread_group;
1777 assert(old_tg != NULL);
1778 coal->j.thread_group = tg;
1779
1780 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_THREAD_GROUP_SET),
1781 coal->id, coal->type, thread_group_get_id(tg));
1782
1783 thread_group_release(old_tg);
1784 }
1785
1786 void
task_coalition_thread_group_focal_update(task_t task)1787 task_coalition_thread_group_focal_update(task_t task)
1788 {
1789 assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL);
1790 thread_group_flags_update_lock();
1791 uint32_t focal_count = task_coalition_focal_count(task);
1792 if (focal_count) {
1793 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP);
1794 } else {
1795 thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP);
1796 }
1797 thread_group_flags_update_unlock();
1798 }
1799
1800 void
task_coalition_thread_group_game_mode_update(task_t task)1801 task_coalition_thread_group_game_mode_update(task_t task)
1802 {
1803 assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL);
1804 thread_group_flags_update_lock();
1805 if (task_coalition_game_mode_count(task)) {
1806 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE);
1807 } else {
1808 thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE);
1809 }
1810 thread_group_flags_update_unlock();
1811 }
1812
1813 void
task_coalition_thread_group_carplay_mode_update(task_t task)1814 task_coalition_thread_group_carplay_mode_update(task_t task)
1815 {
1816 assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL);
1817 thread_group_flags_update_lock();
1818 if (task_coalition_carplay_mode_count(task)) {
1819 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_CARPLAY_MODE);
1820 } else {
1821 thread_group_clear_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_CARPLAY_MODE);
1822 }
1823 thread_group_flags_update_unlock();
1824 }
1825
1826 void
task_coalition_thread_group_application_set(task_t task)1827 task_coalition_thread_group_application_set(task_t task)
1828 {
1829 /*
1830 * Setting the "Application" flag on the thread group is a one way transition.
1831 * Once a coalition has a single task with an application apptype, the
1832 * thread group associated with the coalition is tagged as Application.
1833 */
1834 thread_group_flags_update_lock();
1835 thread_group_set_flags_locked(task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_APPLICATION);
1836 thread_group_flags_update_unlock();
1837 }
1838
1839 #endif /* CONFIG_THREAD_GROUPS */
1840
1841 static void
coalition_for_each_task_locked(coalition_t coal,OS_NOESCAPE coalition_for_each_task_block_t block)1842 coalition_for_each_task_locked(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block)
1843 {
1844 assert(coal != COALITION_NULL);
1845
1846 coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u",
1847 coal, coal->id, coal_type_str(coal->type), coal->active_count);
1848
1849 __assert_only task_t selected_task;
1850
1851 selected_task = coal_call(coal, iterate_tasks, block);
1852
1853 assert(selected_task == TASK_NULL);
1854 }
1855
1856 void
coalition_for_each_task(coalition_t coal,OS_NOESCAPE coalition_for_each_task_block_t block)1857 coalition_for_each_task(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block)
1858 {
1859 assert(coal != COALITION_NULL);
1860
1861 coalition_lock(coal);
1862
1863 coalition_for_each_task_locked(coal, block);
1864
1865 coalition_unlock(coal);
1866 }
1867
1868 static task_t
coalition_select_task(coalition_t coal,OS_NOESCAPE coalition_for_each_task_block_t block)1869 coalition_select_task(coalition_t coal, OS_NOESCAPE coalition_for_each_task_block_t block)
1870 {
1871 assert(coal != COALITION_NULL);
1872
1873 coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u",
1874 coal, coal->id, coal_type_str(coal->type), coal->active_count);
1875
1876 coalition_lock(coal);
1877
1878 task_t selected_task = coal_call(coal, iterate_tasks, block);
1879
1880 if (selected_task != TASK_NULL) {
1881 task_reference(selected_task);
1882 }
1883
1884 coalition_unlock(coal);
1885 return selected_task;
1886 }
1887
1888
1889 void
coalition_remove_active(coalition_t coal)1890 coalition_remove_active(coalition_t coal)
1891 {
1892 coalition_lock(coal);
1893
1894 assert(!coalition_is_reaped(coal));
1895 assert(coal->active_count > 0);
1896
1897 coal->active_count--;
1898
1899 boolean_t do_notify = FALSE;
1900 uint64_t notify_id = 0;
1901 uint32_t notify_flags = 0;
1902 if (coal->termrequested && coal->active_count == 0) {
1903 /* We only notify once, when active_count reaches zero.
1904 * We just decremented, so if it reached zero, we mustn't have
1905 * notified already.
1906 */
1907 assert(!coal->terminated);
1908 coal->terminated = TRUE;
1909
1910 assert(!coal->notified);
1911
1912 coal->notified = TRUE;
1913 #if DEVELOPMENT || DEBUG
1914 do_notify = coal->should_notify;
1915 #else
1916 do_notify = TRUE;
1917 #endif
1918 notify_id = coal->id;
1919 notify_flags = 0;
1920 }
1921
1922 #if COALITION_DEBUG
1923 uint64_t cid = coal->id;
1924 uint32_t rc = coal_ref_count(coal);
1925 int ac = coal->active_count;
1926 int ct = coal->type;
1927 #endif
1928 coalition_unlock(coal);
1929
1930 coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u,%s",
1931 cid, coal_type_str(ct), rc, ac, do_notify ? " NOTIFY" : " ");
1932
1933 if (do_notify) {
1934 coalition_notify_user(notify_id, notify_flags);
1935 }
1936 }
1937
1938 /* Used for kernel_task, launchd, launchd's early boot tasks... */
1939 kern_return_t
coalitions_adopt_init_task(task_t task)1940 coalitions_adopt_init_task(task_t task)
1941 {
1942 kern_return_t kr;
1943 kr = coalitions_adopt_task(init_coalition, task);
1944 if (kr != KERN_SUCCESS) {
1945 panic("failed to adopt task %p into default coalition: %d", task, kr);
1946 }
1947 return kr;
1948 }
1949
1950 /* Used for forked corpses. */
1951 kern_return_t
coalitions_adopt_corpse_task(task_t task)1952 coalitions_adopt_corpse_task(task_t task)
1953 {
1954 kern_return_t kr;
1955 kr = coalitions_adopt_task(corpse_coalition, task);
1956 if (kr != KERN_SUCCESS) {
1957 panic("failed to adopt task %p into corpse coalition: %d", task, kr);
1958 }
1959 return kr;
1960 }
1961
1962 /*
1963 * coalition_adopt_task_internal
1964 * Condition: Coalition must be referenced and unlocked. Will fail if coalition
1965 * is already terminated.
1966 */
1967 static kern_return_t
coalition_adopt_task_internal(coalition_t coal,task_t task)1968 coalition_adopt_task_internal(coalition_t coal, task_t task)
1969 {
1970 kern_return_t kr;
1971
1972 if (task->coalition[coal->type]) {
1973 return KERN_ALREADY_IN_SET;
1974 }
1975
1976 coalition_lock(coal);
1977
1978 if (coal->reaped || coal->terminated) {
1979 coalition_unlock(coal);
1980 return KERN_TERMINATED;
1981 }
1982
1983 kr = coal_call(coal, adopt_task, task);
1984 if (kr != KERN_SUCCESS) {
1985 goto out_unlock;
1986 }
1987
1988 coal->active_count++;
1989
1990 coal_ref_retain(coal);
1991
1992 task->coalition[coal->type] = coal;
1993
1994 out_unlock:
1995 #if COALITION_DEBUG
1996 (void)coal; /* need expression after label */
1997 uint64_t cid = coal->id;
1998 uint32_t rc = coal_ref_count(coal);
1999 uint32_t ct = coal->type;
2000 #endif
2001 if (get_task_uniqueid(task) != UINT64_MAX) {
2002 /* On 32-bit targets, uniqueid will get truncated to 32 bits */
2003 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_ADOPT),
2004 coal->id, get_task_uniqueid(task));
2005 }
2006
2007 coalition_unlock(coal);
2008
2009 coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d",
2010 task_pid(task), cid, coal_type_str(ct), rc, kr);
2011 return kr;
2012 }
2013
2014 static kern_return_t
coalition_remove_task_internal(task_t task,int type)2015 coalition_remove_task_internal(task_t task, int type)
2016 {
2017 kern_return_t kr;
2018
2019 coalition_t coal = task->coalition[type];
2020
2021 if (!coal) {
2022 return KERN_SUCCESS;
2023 }
2024
2025 assert(coal->type == (uint32_t)type);
2026
2027 coalition_lock(coal);
2028
2029 kr = coal_call(coal, remove_task, task);
2030
2031 #if COALITION_DEBUG
2032 uint64_t cid = coal->id;
2033 uint32_t rc = coal_ref_count(coal);
2034 int ac = coal->active_count;
2035 int ct = coal->type;
2036 #endif
2037 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_REMOVE),
2038 coal->id, get_task_uniqueid(task));
2039 coalition_unlock(coal);
2040
2041 coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d",
2042 cid, coal_type_str(ct), rc, ac, kr);
2043
2044 coalition_remove_active(coal);
2045
2046 return kr;
2047 }
2048
2049 /*
2050 * coalitions_adopt_task
2051 * Condition: All coalitions must be referenced and unlocked.
2052 * Will fail if any coalition is already terminated.
2053 */
2054 kern_return_t
coalitions_adopt_task(coalition_t * coals,task_t task)2055 coalitions_adopt_task(coalition_t *coals, task_t task)
2056 {
2057 int i;
2058 kern_return_t kr;
2059
2060 if (!coals || coals[COALITION_TYPE_RESOURCE] == COALITION_NULL) {
2061 return KERN_INVALID_ARGUMENT;
2062 }
2063
2064 /* verify that the incoming coalitions are what they say they are */
2065 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2066 if (coals[i] && coals[i]->type != (uint32_t)i) {
2067 return KERN_INVALID_ARGUMENT;
2068 }
2069 }
2070
2071 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2072 kr = KERN_SUCCESS;
2073 if (coals[i]) {
2074 kr = coalition_adopt_task_internal(coals[i], task);
2075 }
2076 if (kr != KERN_SUCCESS) {
2077 /* dis-associate any coalitions that just adopted this task */
2078 while (--i >= 0) {
2079 if (task->coalition[i]) {
2080 coalition_remove_task_internal(task, i);
2081 }
2082 }
2083 break;
2084 }
2085 }
2086 return kr;
2087 }
2088
2089 /*
2090 * coalitions_remove_task
2091 * Condition: task must be referenced and UNLOCKED; all task's coalitions must be UNLOCKED
2092 */
2093 kern_return_t
coalitions_remove_task(task_t task)2094 coalitions_remove_task(task_t task)
2095 {
2096 kern_return_t kr;
2097 int i;
2098
2099 task_lock(task);
2100 if (!task_is_coalition_member(task)) {
2101 task_unlock(task);
2102 return KERN_SUCCESS;
2103 }
2104
2105 task_clear_coalition_member(task);
2106 task_unlock(task);
2107
2108 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2109 kr = coalition_remove_task_internal(task, i);
2110 assert(kr == KERN_SUCCESS);
2111 }
2112
2113 return kr;
2114 }
2115
2116 /*
2117 * task_release_coalitions
2118 * helper function to release references to all coalitions in which
2119 * 'task' is a member.
2120 */
2121 void
task_release_coalitions(task_t task)2122 task_release_coalitions(task_t task)
2123 {
2124 int i;
2125 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2126 if (task->coalition[i]) {
2127 coalition_release(task->coalition[i]);
2128 } else if (i == COALITION_TYPE_RESOURCE) {
2129 panic("deallocating task %p was not a member of a resource coalition", task);
2130 }
2131 }
2132 }
2133
2134 /*
2135 * coalitions_set_roles
2136 * for each type of coalition, if the task is a member of a coalition of
2137 * that type (given in the coalitions parameter) then set the role of
2138 * the task within that that coalition.
2139 */
2140 kern_return_t
coalitions_set_roles(coalition_t coalitions[COALITION_NUM_TYPES],task_t task,int roles[COALITION_NUM_TYPES])2141 coalitions_set_roles(coalition_t coalitions[COALITION_NUM_TYPES],
2142 task_t task, int roles[COALITION_NUM_TYPES])
2143 {
2144 kern_return_t kr = KERN_SUCCESS;
2145 int i;
2146
2147 for (i = 0; i < COALITION_NUM_TYPES; i++) {
2148 if (!coalitions[i]) {
2149 continue;
2150 }
2151 coalition_lock(coalitions[i]);
2152 kr = coal_call(coalitions[i], set_taskrole, task, roles[i]);
2153 coalition_unlock(coalitions[i]);
2154 assert(kr == KERN_SUCCESS);
2155 }
2156
2157 return kr;
2158 }
2159
2160 /*
2161 * coalition_terminate_internal
2162 * Condition: Coalition must be referenced and UNLOCKED.
2163 */
2164 kern_return_t
coalition_request_terminate_internal(coalition_t coal)2165 coalition_request_terminate_internal(coalition_t coal)
2166 {
2167 assert(coal->type >= 0 && coal->type <= COALITION_TYPE_MAX);
2168
2169 if (coal == init_coalition[coal->type]) {
2170 return KERN_DEFAULT_SET;
2171 }
2172
2173 coalition_lock(coal);
2174
2175 if (coal->reaped) {
2176 coalition_unlock(coal);
2177 return KERN_INVALID_NAME;
2178 }
2179
2180 if (coal->terminated || coal->termrequested) {
2181 coalition_unlock(coal);
2182 return KERN_TERMINATED;
2183 }
2184
2185 coal->termrequested = TRUE;
2186
2187 boolean_t do_notify = FALSE;
2188 uint64_t note_id = 0;
2189 uint32_t note_flags = 0;
2190
2191 if (coal->active_count == 0) {
2192 /*
2193 * We only notify once, when active_count reaches zero.
2194 * We just set termrequested to zero. If the active count
2195 * was already at zero (tasks died before we could request
2196 * a termination notification), we should notify.
2197 */
2198 assert(!coal->terminated);
2199 coal->terminated = TRUE;
2200
2201 assert(!coal->notified);
2202
2203 coal->notified = TRUE;
2204 #if DEVELOPMENT || DEBUG
2205 do_notify = coal->should_notify;
2206 #else
2207 do_notify = TRUE;
2208 #endif
2209 note_id = coal->id;
2210 note_flags = 0;
2211 }
2212
2213 coalition_unlock(coal);
2214
2215 if (do_notify) {
2216 coalition_notify_user(note_id, note_flags);
2217 }
2218
2219 return KERN_SUCCESS;
2220 }
2221
2222 /*
2223 * coalition_reap_internal
2224 * Condition: Coalition must be referenced and UNLOCKED.
2225 */
2226 kern_return_t
coalition_reap_internal(coalition_t coal)2227 coalition_reap_internal(coalition_t coal)
2228 {
2229 assert(coal->type <= COALITION_TYPE_MAX);
2230
2231 if (coal == init_coalition[coal->type]) {
2232 return KERN_DEFAULT_SET;
2233 }
2234
2235 coalition_lock(coal);
2236 if (coal->reaped) {
2237 coalition_unlock(coal);
2238 return KERN_TERMINATED;
2239 }
2240 if (!coal->terminated) {
2241 coalition_unlock(coal);
2242 return KERN_FAILURE;
2243 }
2244 assert(coal->termrequested);
2245 if (coal->active_count > 0) {
2246 coalition_unlock(coal);
2247 return KERN_FAILURE;
2248 }
2249
2250 coal->reaped = TRUE;
2251
2252 coalition_unlock(coal);
2253
2254 lck_mtx_lock(&coalitions_list_lock);
2255 smr_hash_serialized_remove(&coalition_hash, &coal->link,
2256 &coal_hash_traits);
2257 if (smr_hash_serialized_should_shrink(&coalition_hash,
2258 COALITION_HASH_SIZE_MIN, 2, 1)) {
2259 /* shrink if more than 2 buckets per 1 element */
2260 smr_hash_shrink_and_unlock(&coalition_hash,
2261 &coalitions_list_lock, &coal_hash_traits);
2262 } else {
2263 lck_mtx_unlock(&coalitions_list_lock);
2264 }
2265
2266 /* Release the list's reference and launchd's reference. */
2267 coal_ref_release_live(coal);
2268 coalition_release(coal);
2269
2270 return KERN_SUCCESS;
2271 }
2272
2273 #if DEVELOPMENT || DEBUG
2274 int
coalition_should_notify(coalition_t coal)2275 coalition_should_notify(coalition_t coal)
2276 {
2277 int should;
2278 if (!coal) {
2279 return -1;
2280 }
2281 coalition_lock(coal);
2282 should = coal->should_notify;
2283 coalition_unlock(coal);
2284
2285 return should;
2286 }
2287
2288 void
coalition_set_notify(coalition_t coal,int notify)2289 coalition_set_notify(coalition_t coal, int notify)
2290 {
2291 if (!coal) {
2292 return;
2293 }
2294 coalition_lock(coal);
2295 coal->should_notify = !!notify;
2296 coalition_unlock(coal);
2297 }
2298 #endif
2299
2300 void
coalitions_init(void)2301 coalitions_init(void)
2302 {
2303 kern_return_t kr;
2304 int i;
2305 const struct coalition_type *ctype;
2306
2307 smr_hash_init(&coalition_hash, COALITION_HASH_SIZE_MIN);
2308
2309 init_task_ledgers();
2310
2311 init_coalition_ledgers();
2312
2313 for (i = 0, ctype = &s_coalition_types[0]; i < COALITION_NUM_TYPES; ctype++, i++) {
2314 /* verify the entry in the global coalition types array */
2315 if (ctype->type != i ||
2316 !ctype->init ||
2317 !ctype->dealloc ||
2318 !ctype->adopt_task ||
2319 !ctype->remove_task) {
2320 panic("%s: Malformed coalition type %s(%d) in slot for type:%s(%d)",
2321 __func__, coal_type_str(ctype->type), ctype->type, coal_type_str(i), i);
2322 }
2323 if (!ctype->has_default) {
2324 continue;
2325 }
2326 kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, TRUE, FALSE, &init_coalition[ctype->type], NULL);
2327 if (kr != KERN_SUCCESS) {
2328 panic("%s: could not create init %s coalition: kr:%d",
2329 __func__, coal_type_str(i), kr);
2330 }
2331 if (i == COALITION_TYPE_RESOURCE) {
2332 assert(COALITION_ID_KERNEL == init_coalition[ctype->type]->id);
2333 }
2334 kr = coalition_create_internal(ctype->type, COALITION_ROLE_SYSTEM, FALSE, FALSE, &corpse_coalition[ctype->type], NULL);
2335 if (kr != KERN_SUCCESS) {
2336 panic("%s: could not create corpse %s coalition: kr:%d",
2337 __func__, coal_type_str(i), kr);
2338 }
2339 }
2340
2341 /* "Leak" our reference to the global object */
2342 }
2343
2344 /*
2345 * BSD Kernel interface functions
2346 *
2347 */
2348 static void
coalition_fill_procinfo(struct coalition * coal,struct procinfo_coalinfo * coalinfo)2349 coalition_fill_procinfo(struct coalition *coal,
2350 struct procinfo_coalinfo *coalinfo)
2351 {
2352 coalinfo->coalition_id = coal->id;
2353 coalinfo->coalition_type = coal->type;
2354 coalinfo->coalition_tasks = coalition_get_task_count(coal);
2355 }
2356
2357
2358 size_t
coalitions_get_list(int type,struct procinfo_coalinfo * coal_list,size_t list_sz)2359 coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz)
2360 {
2361 size_t ncoals = 0;
2362 struct coalition *coal;
2363
2364 lck_mtx_lock(&coalitions_list_lock);
2365 smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) {
2366 if (!coal->reaped && (type < 0 || type == (int)coal->type)) {
2367 if (coal_list && ncoals < list_sz) {
2368 coalition_fill_procinfo(coal, &coal_list[ncoals]);
2369 }
2370 ++ncoals;
2371 }
2372 }
2373 lck_mtx_unlock(&coalitions_list_lock);
2374
2375 return ncoals;
2376 }
2377
2378 /*
2379 * Return the coaltion of the given type to which the task belongs.
2380 */
2381 coalition_t
task_get_coalition(task_t task,int coal_type)2382 task_get_coalition(task_t task, int coal_type)
2383 {
2384 coalition_t c;
2385
2386 if (task == NULL || coal_type > COALITION_TYPE_MAX) {
2387 return COALITION_NULL;
2388 }
2389
2390 c = task->coalition[coal_type];
2391 assert(c == COALITION_NULL || (int)c->type == coal_type);
2392 return c;
2393 }
2394
2395 /*
2396 * Report if the given task is the leader of the given jetsam coalition.
2397 */
2398 boolean_t
coalition_is_leader(task_t task,coalition_t coal)2399 coalition_is_leader(task_t task, coalition_t coal)
2400 {
2401 boolean_t ret = FALSE;
2402
2403 if (coal != COALITION_NULL) {
2404 coalition_lock(coal);
2405
2406 ret = (coal->type == COALITION_TYPE_JETSAM && coal->j.leader == task);
2407
2408 coalition_unlock(coal);
2409 }
2410
2411 return ret;
2412 }
2413
2414 kern_return_t
coalition_iterate_stackshot(coalition_iterate_fn_t callout,void * arg,uint32_t coalition_type)2415 coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type)
2416 {
2417 coalition_t coal;
2418 int i = 0;
2419
2420 smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) {
2421 if (coal == NULL || !ml_validate_nofault((vm_offset_t)coal, sizeof(struct coalition))) {
2422 return KERN_FAILURE;
2423 }
2424
2425 if (coalition_type == coal->type) {
2426 callout(arg, i++, coal);
2427 }
2428 }
2429
2430 return KERN_SUCCESS;
2431 }
2432
2433 task_t
kdp_coalition_get_leader(coalition_t coal)2434 kdp_coalition_get_leader(coalition_t coal)
2435 {
2436 if (!coal) {
2437 return TASK_NULL;
2438 }
2439
2440 if (coal->type == COALITION_TYPE_JETSAM) {
2441 return coal->j.leader;
2442 }
2443 return TASK_NULL;
2444 }
2445
2446 task_t
coalition_get_leader(coalition_t coal)2447 coalition_get_leader(coalition_t coal)
2448 {
2449 task_t leader = TASK_NULL;
2450
2451 if (!coal) {
2452 return TASK_NULL;
2453 }
2454
2455 coalition_lock(coal);
2456 if (coal->type != COALITION_TYPE_JETSAM) {
2457 goto out_unlock;
2458 }
2459
2460 leader = coal->j.leader;
2461 if (leader != TASK_NULL) {
2462 task_reference(leader);
2463 }
2464
2465 out_unlock:
2466 coalition_unlock(coal);
2467 return leader;
2468 }
2469
2470
2471 int
coalition_get_task_count(coalition_t coal)2472 coalition_get_task_count(coalition_t coal)
2473 {
2474 int ntasks = 0;
2475 struct queue_entry *qe;
2476 if (!coal) {
2477 return 0;
2478 }
2479
2480 coalition_lock(coal);
2481 switch (coal->type) {
2482 case COALITION_TYPE_RESOURCE:
2483 qe_foreach(qe, &coal->r.tasks)
2484 ntasks++;
2485 break;
2486 case COALITION_TYPE_JETSAM:
2487 if (coal->j.leader) {
2488 ntasks++;
2489 }
2490 qe_foreach(qe, &coal->j.other)
2491 ntasks++;
2492 qe_foreach(qe, &coal->j.extensions)
2493 ntasks++;
2494 qe_foreach(qe, &coal->j.services)
2495 ntasks++;
2496 break;
2497 default:
2498 break;
2499 }
2500 coalition_unlock(coal);
2501
2502 return ntasks;
2503 }
2504
2505
2506 static uint64_t
i_get_list_footprint(queue_t list,int type,int * ntasks)2507 i_get_list_footprint(queue_t list, int type, int *ntasks)
2508 {
2509 task_t task;
2510 uint64_t bytes = 0;
2511
2512 qe_foreach_element(task, list, task_coalition[type]) {
2513 bytes += get_task_phys_footprint(task);
2514 coal_dbg(" [%d] task_pid:%d, type:%d, footprint:%lld",
2515 *ntasks, task_pid(task), type, bytes);
2516 *ntasks += 1;
2517 }
2518
2519 return bytes;
2520 }
2521
2522 uint64_t
coalition_get_page_count(coalition_t coal,int * ntasks)2523 coalition_get_page_count(coalition_t coal, int *ntasks)
2524 {
2525 uint64_t bytes = 0;
2526 int num_tasks = 0;
2527
2528 if (ntasks) {
2529 *ntasks = 0;
2530 }
2531 if (!coal) {
2532 return bytes;
2533 }
2534
2535 coalition_lock(coal);
2536
2537 switch (coal->type) {
2538 case COALITION_TYPE_RESOURCE:
2539 bytes += i_get_list_footprint(&coal->r.tasks, COALITION_TYPE_RESOURCE, &num_tasks);
2540 break;
2541 case COALITION_TYPE_JETSAM:
2542 if (coal->j.leader) {
2543 bytes += get_task_phys_footprint(coal->j.leader);
2544 num_tasks = 1;
2545 }
2546 bytes += i_get_list_footprint(&coal->j.extensions, COALITION_TYPE_JETSAM, &num_tasks);
2547 bytes += i_get_list_footprint(&coal->j.services, COALITION_TYPE_JETSAM, &num_tasks);
2548 bytes += i_get_list_footprint(&coal->j.other, COALITION_TYPE_JETSAM, &num_tasks);
2549 break;
2550 default:
2551 break;
2552 }
2553
2554 coalition_unlock(coal);
2555
2556 if (ntasks) {
2557 *ntasks = num_tasks;
2558 }
2559
2560 return bytes / PAGE_SIZE_64;
2561 }
2562
2563 struct coal_sort_s {
2564 int pid;
2565 int usr_order;
2566 uint64_t bytes;
2567 };
2568
2569 /*
2570 * return < 0 for a < b
2571 * 0 for a == b
2572 * > 0 for a > b
2573 */
2574 typedef int (*cmpfunc_t)(const void *a, const void *b);
2575
2576 extern void
2577 qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
2578
2579 static int
dflt_cmp(const void * a,const void * b)2580 dflt_cmp(const void *a, const void *b)
2581 {
2582 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2583 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2584
2585 /*
2586 * if both A and B are equal, use a memory descending sort
2587 */
2588 if (csA->usr_order == csB->usr_order) {
2589 return (int)((int64_t)csB->bytes - (int64_t)csA->bytes);
2590 }
2591
2592 /* otherwise, return the relationship between user specified orders */
2593 return csA->usr_order - csB->usr_order;
2594 }
2595
2596 static int
mem_asc_cmp(const void * a,const void * b)2597 mem_asc_cmp(const void *a, const void *b)
2598 {
2599 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2600 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2601
2602 return (int)((int64_t)csA->bytes - (int64_t)csB->bytes);
2603 }
2604
2605 static int
mem_dec_cmp(const void * a,const void * b)2606 mem_dec_cmp(const void *a, const void *b)
2607 {
2608 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2609 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2610
2611 return (int)((int64_t)csB->bytes - (int64_t)csA->bytes);
2612 }
2613
2614 static int
usr_asc_cmp(const void * a,const void * b)2615 usr_asc_cmp(const void *a, const void *b)
2616 {
2617 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2618 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2619
2620 return csA->usr_order - csB->usr_order;
2621 }
2622
2623 static int
usr_dec_cmp(const void * a,const void * b)2624 usr_dec_cmp(const void *a, const void *b)
2625 {
2626 const struct coal_sort_s *csA = (const struct coal_sort_s *)a;
2627 const struct coal_sort_s *csB = (const struct coal_sort_s *)b;
2628
2629 return csB->usr_order - csA->usr_order;
2630 }
2631
2632 /* avoid dynamic allocation in this path */
2633 #define MAX_SORTED_PIDS 80
2634
2635 static int
coalition_get_sort_list(coalition_t coal,int sort_order,queue_t list,struct coal_sort_s * sort_array,int array_sz)2636 coalition_get_sort_list(coalition_t coal, int sort_order, queue_t list,
2637 struct coal_sort_s *sort_array, int array_sz)
2638 {
2639 int ntasks = 0;
2640 task_t task;
2641
2642 assert(sort_array != NULL);
2643
2644 if (array_sz <= 0) {
2645 return 0;
2646 }
2647
2648 if (!list) {
2649 /*
2650 * this function will only be called with a NULL
2651 * list for JETSAM-type coalitions, and is intended
2652 * to investigate the leader process
2653 */
2654 if (coal->type != COALITION_TYPE_JETSAM ||
2655 coal->j.leader == TASK_NULL) {
2656 return 0;
2657 }
2658 sort_array[0].pid = task_pid(coal->j.leader);
2659 switch (sort_order) {
2660 case COALITION_SORT_DEFAULT:
2661 sort_array[0].usr_order = 0;
2662 OS_FALLTHROUGH;
2663 case COALITION_SORT_MEM_ASC:
2664 case COALITION_SORT_MEM_DEC:
2665 sort_array[0].bytes = get_task_phys_footprint(coal->j.leader);
2666 break;
2667 case COALITION_SORT_USER_ASC:
2668 case COALITION_SORT_USER_DEC:
2669 sort_array[0].usr_order = 0;
2670 break;
2671 default:
2672 break;
2673 }
2674 return 1;
2675 }
2676
2677 qe_foreach_element(task, list, task_coalition[coal->type]) {
2678 if (ntasks >= array_sz) {
2679 printf("WARNING: more than %d pids in coalition %llu\n",
2680 MAX_SORTED_PIDS, coal->id);
2681 break;
2682 }
2683
2684 sort_array[ntasks].pid = task_pid(task);
2685
2686 switch (sort_order) {
2687 case COALITION_SORT_DEFAULT:
2688 sort_array[ntasks].usr_order = 0;
2689 OS_FALLTHROUGH;
2690 case COALITION_SORT_MEM_ASC:
2691 case COALITION_SORT_MEM_DEC:
2692 sort_array[ntasks].bytes = get_task_phys_footprint(task);
2693 break;
2694 case COALITION_SORT_USER_ASC:
2695 case COALITION_SORT_USER_DEC:
2696 sort_array[ntasks].usr_order = 0;
2697 break;
2698 default:
2699 break;
2700 }
2701
2702 ntasks++;
2703 }
2704
2705 return ntasks;
2706 }
2707
2708 int
coalition_get_pid_list(coalition_t coal,uint32_t rolemask,int sort_order,int * pid_list,int list_sz)2709 coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order,
2710 int *pid_list, int list_sz)
2711 {
2712 struct i_jetsam_coalition *cj;
2713 int ntasks = 0;
2714 cmpfunc_t cmp_func = NULL;
2715 struct coal_sort_s sort_array[MAX_SORTED_PIDS] = { {0, 0, 0} }; /* keep to < 2k */
2716
2717 if (!coal ||
2718 !(rolemask & COALITION_ROLEMASK_ALLROLES) ||
2719 !pid_list || list_sz < 1) {
2720 coal_dbg("Invalid parameters: coal:%p, type:%d, rolemask:0x%x, "
2721 "pid_list:%p, list_sz:%d", coal, coal ? coal->type : -1,
2722 rolemask, pid_list, list_sz);
2723 return -EINVAL;
2724 }
2725
2726 switch (sort_order) {
2727 case COALITION_SORT_NOSORT:
2728 cmp_func = NULL;
2729 break;
2730 case COALITION_SORT_DEFAULT:
2731 cmp_func = dflt_cmp;
2732 break;
2733 case COALITION_SORT_MEM_ASC:
2734 cmp_func = mem_asc_cmp;
2735 break;
2736 case COALITION_SORT_MEM_DEC:
2737 cmp_func = mem_dec_cmp;
2738 break;
2739 case COALITION_SORT_USER_ASC:
2740 cmp_func = usr_asc_cmp;
2741 break;
2742 case COALITION_SORT_USER_DEC:
2743 cmp_func = usr_dec_cmp;
2744 break;
2745 default:
2746 return -ENOTSUP;
2747 }
2748
2749 coalition_lock(coal);
2750
2751 if (coal->type == COALITION_TYPE_RESOURCE) {
2752 ntasks += coalition_get_sort_list(coal, sort_order, &coal->r.tasks,
2753 sort_array, MAX_SORTED_PIDS);
2754 goto unlock_coal;
2755 }
2756
2757 cj = &coal->j;
2758
2759 if (rolemask & COALITION_ROLEMASK_UNDEF) {
2760 ntasks += coalition_get_sort_list(coal, sort_order, &cj->other,
2761 sort_array + ntasks,
2762 MAX_SORTED_PIDS - ntasks);
2763 }
2764
2765 if (rolemask & COALITION_ROLEMASK_XPC) {
2766 ntasks += coalition_get_sort_list(coal, sort_order, &cj->services,
2767 sort_array + ntasks,
2768 MAX_SORTED_PIDS - ntasks);
2769 }
2770
2771 if (rolemask & COALITION_ROLEMASK_EXT) {
2772 ntasks += coalition_get_sort_list(coal, sort_order, &cj->extensions,
2773 sort_array + ntasks,
2774 MAX_SORTED_PIDS - ntasks);
2775 }
2776
2777 if (rolemask & COALITION_ROLEMASK_LEADER) {
2778 ntasks += coalition_get_sort_list(coal, sort_order, NULL,
2779 sort_array + ntasks,
2780 MAX_SORTED_PIDS - ntasks);
2781 }
2782
2783 unlock_coal:
2784 coalition_unlock(coal);
2785
2786 /* sort based on the chosen criterion (no sense sorting 1 item) */
2787 if (cmp_func && ntasks > 1) {
2788 qsort(sort_array, ntasks, sizeof(struct coal_sort_s), cmp_func);
2789 }
2790
2791 for (int i = 0; i < ntasks; i++) {
2792 if (i >= list_sz) {
2793 break;
2794 }
2795 coal_dbg(" [%d] PID:%d, footprint:%lld, usr_order:%d",
2796 i, sort_array[i].pid, sort_array[i].bytes,
2797 sort_array[i].usr_order);
2798 pid_list[i] = sort_array[i].pid;
2799 }
2800
2801 return ntasks;
2802 }
2803
2804 void
coalition_mark_swappable(coalition_t coal)2805 coalition_mark_swappable(coalition_t coal)
2806 {
2807 struct i_jetsam_coalition *cj = NULL;
2808
2809 coalition_lock(coal);
2810 assert(coal && coal->type == COALITION_TYPE_JETSAM);
2811
2812 cj = &coal->j;
2813 cj->swap_enabled = true;
2814
2815 i_coal_jetsam_iterate_tasks(coal, ^bool (task_t task) {
2816 vm_task_set_selfdonate_pages(task, true);
2817 return false;
2818 });
2819
2820 coalition_unlock(coal);
2821 }
2822
2823 bool
coalition_is_swappable(coalition_t coal)2824 coalition_is_swappable(coalition_t coal)
2825 {
2826 struct i_jetsam_coalition *cj = NULL;
2827
2828 coalition_lock(coal);
2829 assert(coal && coal->type == COALITION_TYPE_JETSAM);
2830
2831 cj = &coal->j;
2832 bool enabled = cj->swap_enabled;
2833
2834 coalition_unlock(coal);
2835
2836 return enabled;
2837 }
2838
2839
2840 /*
2841 * Coalition policy functions
2842 */
2843
2844 static uintptr_t
crequested(coalition_t coal)2845 crequested(coalition_t coal)
2846 {
2847 static_assert(sizeof(struct coalition_requested_policy) == sizeof(uint64_t), "size invariant violated");
2848
2849 uintptr_t* raw = (uintptr_t*)&coal->j.c_requested_policy;
2850
2851 return raw[0];
2852 }
2853
2854 static uintptr_t
ceffective(coalition_t coal)2855 ceffective(coalition_t coal)
2856 {
2857 static_assert(sizeof(struct coalition_requested_policy) == sizeof(uint64_t), "size invariant violated");
2858
2859 uintptr_t* raw = (uintptr_t*)&coal->j.c_effective_policy;
2860
2861 return raw[0];
2862 }
2863
2864 static uint32_t
cpending(coalition_pend_token_t pend_token)2865 cpending(coalition_pend_token_t pend_token)
2866 {
2867 static_assert(sizeof(struct coalition_pend_token) == sizeof(uint32_t), "size invariant violated");
2868
2869 return *(uint32_t*)(void*)(pend_token);
2870 }
2871
2872
2873 static void
jetsam_coalition_policy_update_locked(coalition_t coal,coalition_pend_token_t pend_token)2874 jetsam_coalition_policy_update_locked(coalition_t coal, coalition_pend_token_t pend_token)
2875 {
2876 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2877 (IMPORTANCE_CODE(IMP_UPDATE, TASK_POLICY_COALITION) | DBG_FUNC_START),
2878 coalition_id(coal), ceffective(coal), 0, 0, 0);
2879
2880 struct coalition_requested_policy requested = coal->j.c_requested_policy;
2881 struct coalition_effective_policy next = {};
2882
2883 /* Update darwinbg state */
2884 next.cep_darwinbg = requested.crp_darwinbg;
2885
2886 /* TODO: should termrequested cause darwinbg to be unset */
2887
2888 struct coalition_effective_policy prev = coal->j.c_effective_policy;
2889
2890 coal->j.c_effective_policy = next;
2891
2892 /* */
2893 if (prev.cep_darwinbg != next.cep_darwinbg) {
2894 pend_token->cpt_update_j_coal_tasks = 1;
2895 }
2896
2897 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2898 (IMPORTANCE_CODE(IMP_UPDATE, TASK_POLICY_COALITION)) | DBG_FUNC_END,
2899 coalition_id(coal), ceffective(coal), 0, 0, 0);
2900 }
2901
2902 /*
2903 * Returns task reference or TASK_NULL if none left.
2904 * Called without coalition lock held.
2905 * Called with global coalition_policy_lock held.
2906 */
2907 static task_t
coalition_find_next_task_with_pending_work(coalition_t coal)2908 coalition_find_next_task_with_pending_work(coalition_t coal)
2909 {
2910 return coalition_select_task(coal, ^bool (task_t task){
2911 if (task->pended_coalition_changes.tpt_value != 0) {
2912 return true;
2913 }
2914 return false;
2915 });
2916 }
2917
2918 /*
2919 * Called with coalition unlocked to do things that can't be done
2920 * while holding the coalition lock.
2921 * Called with global coalition_policy_lock held.
2922 */
2923 static void
coalition_policy_update_complete_unlocked(coalition_t coal,coalition_pend_token_t coalition_pend_token)2924 coalition_policy_update_complete_unlocked(
2925 coalition_t coal,
2926 coalition_pend_token_t coalition_pend_token)
2927 {
2928 /*
2929 * Note that because we dropped the coalition lock, a task may have
2930 * exited since we set the pending bit, so we can't assert that
2931 * no task exits with a change pending.
2932 */
2933
2934 if (coalition_pend_token->cpt_update_j_coal_tasks) {
2935 task_t task;
2936
2937 while ((task = coalition_find_next_task_with_pending_work(coal)) != TASK_NULL) {
2938 task_policy_update_complete_unlocked(task, &task->pended_coalition_changes);
2939 task->pended_coalition_changes.tpt_value = 0;
2940 task_deallocate(task);
2941 }
2942 }
2943
2944
2945 if (coalition_pend_token->cpt_update_timers) {
2946 ml_timer_evaluate();
2947 }
2948 }
2949
2950 /*
2951 * This lock covers the coalition_pend_token fields
2952 * in all tasks, so that we don't have to have stack storage
2953 * for all of them. We could also use a gate or busy bit
2954 * on each coalition, but this is less complex.
2955 */
2956 LCK_MTX_DECLARE(coalition_policy_lock, &coalitions_lck_grp);
2957
2958 kern_return_t
jetsam_coalition_set_policy(coalition_t coal,int flavor,int value)2959 jetsam_coalition_set_policy(coalition_t coal,
2960 int flavor,
2961 int value)
2962 {
2963 struct coalition_pend_token pend_token = {};
2964 coalition_pend_token_t pend_token_ref = &pend_token;
2965
2966 assert(coalition_type(coal) == COALITION_TYPE_JETSAM);
2967
2968 lck_mtx_lock(&coalition_policy_lock);
2969
2970 coalition_lock(coal);
2971
2972 struct coalition_requested_policy requested = coal->j.c_requested_policy;
2973
2974 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2975 (IMPORTANCE_CODE(flavor, TASK_POLICY_COALITION)) | DBG_FUNC_START,
2976 coalition_id(coal), crequested(coal), 0, value, 0);
2977
2978 switch (flavor) {
2979 case TASK_POLICY_DARWIN_BG:
2980 requested.crp_darwinbg = value;
2981 break;
2982 default:
2983 panic("unknown coalition policy: %d %d", flavor, value);
2984 break;
2985 }
2986
2987 coal->j.c_requested_policy = requested;
2988
2989 jetsam_coalition_policy_update_locked(coal, &pend_token);
2990
2991 if (pend_token.cpt_update_j_coal_tasks) {
2992 coalition_for_each_task_locked(coal, ^bool (task_t task) {
2993 coalition_policy_update_task(task, pend_token_ref);
2994 return false;
2995 });
2996 }
2997
2998 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
2999 (IMPORTANCE_CODE(flavor, TASK_POLICY_COALITION)) | DBG_FUNC_END,
3000 coalition_id(coal), crequested(coal), 0, cpending(&pend_token), 0);
3001
3002 coalition_unlock(coal);
3003
3004 coalition_policy_update_complete_unlocked(coal, &pend_token);
3005
3006 lck_mtx_unlock(&coalition_policy_lock);
3007
3008 return KERN_SUCCESS;
3009 }
3010
3011 kern_return_t
jetsam_coalition_get_policy(coalition_t coal,int flavor,int * value)3012 jetsam_coalition_get_policy(coalition_t coal, int flavor, int *value)
3013 {
3014 lck_mtx_lock(&coalition_policy_lock);
3015
3016 coalition_lock(coal);
3017
3018 struct coalition_requested_policy requested = coal->j.c_requested_policy;
3019
3020 switch (flavor) {
3021 case TASK_POLICY_DARWIN_BG:
3022 *value = requested.crp_darwinbg;
3023 break;
3024 default:
3025 panic("unknown coalition policy: %d", flavor);
3026 break;
3027 }
3028
3029 coalition_unlock(coal);
3030
3031 lck_mtx_unlock(&coalition_policy_lock);
3032
3033 return KERN_SUCCESS;
3034 }
3035
3036 bool
task_get_effective_jetsam_coalition_policy(task_t task,int flavor)3037 task_get_effective_jetsam_coalition_policy(task_t task, int flavor)
3038 {
3039 coalition_t coal = task->coalition[COALITION_TYPE_JETSAM];
3040
3041 if (coal == NULL) {
3042 return false;
3043 }
3044
3045 struct coalition_effective_policy effective = coal->j.c_effective_policy;
3046
3047 switch (flavor) {
3048 case TASK_POLICY_DARWIN_BG:
3049 return effective.cep_darwinbg;
3050 default:
3051 panic("unknown coalition policy: %d", flavor);
3052 break;
3053 }
3054
3055 return false;
3056 }
3057