1 /*
2 * C (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/task.h>
30 #include <kern/task_ref.h>
31 #include <libkern/OSKextLibPrivate.h>
32
33 #include <os/refcnt.h>
34
35 /*
36 * Task references.
37 *
38 * Each task reference/deallocate pair has an associated reference group:
39 * TASK_GRP_INTERNAL This group is used exclusively to track long-term
40 * references which are almost always present.
41 * Specifically, the importance task reference, the owning
42 * task reference and the thread references.
43 * TASK_GRP_EXTERNAL For kext references
44 * TASK_KERNEL For at-large kernel references other than those tracked
45 * by task_internal.
46 * TASK_GRP_MIG For references from the MIG layer
47 *
48 * Depending on configuration (see task_refgrp_config) os_refgrps are used to
49 * keep track of the context of the reference/deallocation.
50 *
51 * TASK_REF_CONFIG_OFF
52 * No refgrps are used other than the single 'task' reference group.
53 *
54 * TASK_REF_CONFIG_DEFAULT
55 * Global refgrps are used for 'kernel' and 'external' references. The
56 * primary 'task' reference group is set as their parent. Each kext also gets
57 * its own refgrp parented to the 'external' group.
58 * Each task gets two reference groups - one for 'kernel' references parented to
59 * the global 'kernel' group and as second which is dynamically assigned. All
60 * references tagged with TASK_GRP_INTERNAL, TASK_GRP_KERNEL and TASK_GRP_MIG
61 * use the task 'kernel' group. The dynamic group is initialized for the first
62 * 'external' reference to a kext specific group parented to the matching global
63 * kext group. For 'external' references not matching that group, the global
64 * 'external' group is used.
65 * This is the default configuration.
66 *
67 * TASK_REF_CONFIG_FULL
68 * Global refgrps are used for 'kernel', 'external', 'internal' and 'mig'
69 * references. The primary 'task' reference group is set as their parent. Each
70 * kext also gets is own refgrp parented to the 'external' group.
71 * Each task gets eight reference groups - one each mirroring the four global
72 * reference groups and four dynamic groups which are assigned to kexts. For
73 * 'external' references not matching any of the four dynamic groups, the global
74 * 'external' group is used.
75 *
76 * Kext callers have the calls which take or release task references mapped
77 * to '_external' equivalents via the .exports file.
78 *
79 * At-large kernel callers see calls redefined to call the '_kernel' variants
80 * (see task_ref.h).
81 *
82 * The mig layer generates code which uses the '_mig' variants.
83 *
84 * Other groups are selected explicitly.
85 *
86 * Reference groups support recording of back traces via the rlog boot arg.
87 * For example: rlog=task_external would keep a backtrace log of all external
88 * references.
89 */
90
91 #define TASK_REF_COUNT_INITIAL (2u)
92
93 extern void task_deallocate_internal(task_t task, os_ref_count_t refs);
94
95 #if DEVELOPMENT || DEBUG
96
97 #include <stdbool.h>
98
99 #define DYNAMIC_COUNT 4
100
101 /*
102 * Controlled by the boot arg 'task_refgrp=X'.
103 *
104 * Unspecified/default
105 * There are two task reference groups. One kext specific reference group, the
106 * other used for kernel/internal and mig references.
107 *
108 * "off"
109 * No task specific reference groups are used.
110 *
111 * "full"
112 * Each task gets its own set of kernel/internal/mig and external groups.
113 * Additionally four dynamic reference groups are made available to identify kext
114 * references.
115 */
116 __attribute__((used))
117 static enum {
118 TASK_REF_CONFIG_DEFAULT,
119 TASK_REF_CONFIG_FULL,
120 TASK_REF_CONFIG_OFF,
121 } task_refgrp_config = TASK_REF_CONFIG_DEFAULT;
122
123 /* Global reference groups. */
124 os_refgrp_decl_flags(static, task_primary_refgrp, "task", NULL, OS_REFGRP_F_ALWAYS_ENABLED);
125 os_refgrp_decl_flags(static, task_kernel_refgrp, "task_kernel", &task_primary_refgrp, OS_REFGRP_F_ALWAYS_ENABLED);
126 os_refgrp_decl_flags(static, task_internal_refgrp, "task_internal", &task_primary_refgrp, OS_REFGRP_F_ALWAYS_ENABLED);
127 os_refgrp_decl_flags(static, task_mig_refgrp, "task_mig", &task_primary_refgrp, OS_REFGRP_F_ALWAYS_ENABLED);
128 os_refgrp_decl_flags(, task_external_refgrp, "task_external", &task_primary_refgrp, OS_REFGRP_F_ALWAYS_ENABLED);
129
130
131 /* 'task_refgrp' is used by lldb macros. */
132 __attribute__((used))
133 static struct os_refgrp * const task_refgrp[TASK_GRP_COUNT] = {
134 [TASK_GRP_KERNEL] = &task_kernel_refgrp,
135 [TASK_GRP_INTERNAL] = &task_internal_refgrp,
136 [TASK_GRP_MIG] = &task_mig_refgrp,
137 [TASK_GRP_EXTERNAL] = &task_external_refgrp,
138 };
139
140 /* Names used by local reference groups. */
141 static const char * const local_name[TASK_GRP_COUNT] = {
142 [TASK_GRP_KERNEL] = "task_local_kernel",
143 [TASK_GRP_INTERNAL] = "task_local_internal",
144 [TASK_GRP_MIG] = "task_local_mig",
145 [TASK_GRP_EXTERNAL] = "task_local_external",
146 };
147
148 /* Walk back the callstack calling cb for each address. */
149 static inline void
150 walk_kext_callstack(int (^cb)(uintptr_t))
151 {
152 uintptr_t* frameptr;
153 uintptr_t* frameptr_next;
154 uintptr_t retaddr;
155 uintptr_t kstackb, kstackt;
156 thread_t cthread;
157
158 cthread = current_thread();
159 assert3p(cthread, !=, NULL);
160
161 kstackb = thread_get_kernel_stack(cthread);
162 kstackt = kstackb + kernel_stack_size;
163
164 /* Load stack frame pointer (EBP on x86) into frameptr */
165 frameptr = __builtin_frame_address(0);
166
167 while (frameptr != NULL) {
168 /* Verify thread stack bounds */
169 if (((uintptr_t)(frameptr + 2) > kstackt) ||
170 ((uintptr_t)frameptr < kstackb)) {
171 break;
172 }
173
174 /* Next frame pointer is pointed to by the previous one */
175 frameptr_next = (uintptr_t*) *frameptr;
176
177 /* Pull return address from one spot above the frame pointer */
178 retaddr = *(frameptr + 1);
179
180 #if defined(HAS_APPLE_PAC)
181 retaddr = (uintptr_t) ptrauth_strip((void *)retaddr,
182 ptrauth_key_return_address);
183 #endif
184
185 if (((retaddr < vm_kernel_builtinkmod_text_end) &&
186 (retaddr >= vm_kernel_builtinkmod_text)) ||
187 (retaddr < vm_kernel_stext) || (retaddr > vm_kernel_top)) {
188 if (cb(retaddr) != 0) {
189 return;
190 }
191 }
192 frameptr = frameptr_next;
193 }
194
195 return;
196 }
197
198 /* Return the reference group associated with the 'closest' kext. */
199 static struct os_refgrp *
lookup_kext_refgrp(void)200 lookup_kext_refgrp(void)
201 {
202 __block struct os_refgrp *refgrp = NULL;
203
204 /* Get the kext specific group based on the current stack. */
205 walk_kext_callstack(^(uintptr_t retaddr) {
206 OSKextGetRefGrpForCaller(retaddr, ^(struct os_refgrp *kext_grp) {
207 assert(kext_grp != NULL);
208 refgrp = kext_grp;
209 });
210 return 1;
211 });
212 return refgrp;
213 }
214
215
216 /*
217 * Given an array of reference groups, find one that matches the specified kext
218 * group. If there is no match and there is a empty slot, initialize a new
219 * refgrp with the kext group as the parent (only when `can_allocate` is true).
220 */
221 static struct os_refgrp *
lookup_dynamic_refgrp(struct os_refgrp * kext,struct os_refgrp * dynamic,int dynamic_count,bool can_allocate)222 lookup_dynamic_refgrp(struct os_refgrp *kext,
223 struct os_refgrp *dynamic, int dynamic_count, bool can_allocate)
224 {
225 /* First see if it exists. */
226 for (int i = 0; i < dynamic_count; i++) {
227 if (dynamic[i].grp_parent == kext) {
228 return &dynamic[i];
229 }
230 }
231
232 if (!can_allocate) {
233 return NULL;
234 }
235
236 /* Grab an empty one, if available. */
237 for (int i = 0; i < dynamic_count; i++) {
238 if (dynamic[i].grp_name == NULL) {
239 dynamic[i] = (struct os_refgrp)
240 os_refgrp_initializer(kext->grp_name, kext,
241 OS_REFGRP_F_ALWAYS_ENABLED);
242 return &dynamic[i];
243 }
244 }
245
246 return NULL;
247 }
248
249 /*
250 * Find the best external reference group.
251 * - Task specific kext ref group
252 * else
253 * - Kext ref group
254 * else
255 * - Global external ref group
256 */
257 static struct os_refgrp *
find_external_refgrp(struct os_refgrp * dynamic,int dynamic_count,bool can_allocate)258 find_external_refgrp(struct os_refgrp *dynamic, int dynamic_count,
259 bool can_allocate)
260 {
261 struct os_refgrp *kext_refgrp = lookup_kext_refgrp();
262 if (kext_refgrp == NULL) {
263 return task_refgrp[TASK_GRP_EXTERNAL];
264 }
265
266 struct os_refgrp *refgrp = lookup_dynamic_refgrp(kext_refgrp, dynamic,
267 dynamic_count, can_allocate);
268 if (refgrp == NULL) {
269 return kext_refgrp;
270 }
271
272 return refgrp;
273 }
274
275 void
task_reference_grp(task_t task,task_grp_t grp)276 task_reference_grp(task_t task, task_grp_t grp)
277 {
278 assert3u(grp, <, TASK_GRP_COUNT);
279 assert(
280 task_refgrp_config == TASK_REF_CONFIG_OFF ||
281 task_refgrp_config == TASK_REF_CONFIG_DEFAULT ||
282 task_refgrp_config == TASK_REF_CONFIG_FULL);
283
284 struct os_refgrp *refgrp = NULL;
285
286 if (task == TASK_NULL) {
287 return;
288 }
289
290 task_require(task);
291
292 /*
293 * External ref groups need to search and potentially allocate from the
294 * dynamic task ref groups. This must be protected by a lock.
295 */
296 if (task_refgrp_config != TASK_REF_CONFIG_OFF &&
297 grp == TASK_GRP_EXTERNAL) {
298 lck_spin_lock(&task->ref_group_lock);
299 }
300
301 switch (task_refgrp_config) {
302 case TASK_REF_CONFIG_OFF:
303 refgrp = NULL;
304 break;
305
306 case TASK_REF_CONFIG_DEFAULT:
307
308 refgrp = (grp == TASK_GRP_EXTERNAL) ?
309 find_external_refgrp(&task->ref_group[1], 1, true) :
310 &task->ref_group[TASK_GRP_KERNEL];
311 break;
312
313 case TASK_REF_CONFIG_FULL:
314
315 refgrp = (grp == TASK_GRP_EXTERNAL) ?
316 find_external_refgrp(&task->ref_group[TASK_GRP_COUNT], DYNAMIC_COUNT, true) :
317 &task->ref_group[grp];
318 break;
319 }
320
321 os_ref_retain_raw(&task->ref_count.ref_count, refgrp);
322
323 if (task_refgrp_config != TASK_REF_CONFIG_OFF &&
324 grp == TASK_GRP_EXTERNAL) {
325 lck_spin_unlock(&task->ref_group_lock);
326 }
327 }
328
329 void
task_deallocate_grp(task_t task,task_grp_t grp)330 task_deallocate_grp(task_t task, task_grp_t grp)
331 {
332 assert3u(grp, <, TASK_GRP_COUNT);
333 assert(
334 task_refgrp_config == TASK_REF_CONFIG_OFF ||
335 task_refgrp_config == TASK_REF_CONFIG_DEFAULT ||
336 task_refgrp_config == TASK_REF_CONFIG_FULL);
337
338 os_ref_count_t refs = -1;
339 struct os_refgrp *refgrp = NULL;
340
341 if (task == TASK_NULL) {
342 return;
343 }
344
345 /*
346 * There is no need to take the ref_group_lock when de-allocating. The
347 * lock is only required when allocating a group.
348 */
349 switch (task_refgrp_config) {
350 case TASK_REF_CONFIG_OFF:
351 refgrp = NULL;
352 break;
353
354 case TASK_REF_CONFIG_DEFAULT:
355 refgrp = (grp == TASK_GRP_EXTERNAL) ?
356 find_external_refgrp(&task->ref_group[1], 1, false) :
357 &task->ref_group[TASK_GRP_KERNEL];
358 break;
359
360 case TASK_REF_CONFIG_FULL:
361 refgrp = (grp == TASK_GRP_EXTERNAL) ?
362 find_external_refgrp(&task->ref_group[TASK_GRP_COUNT], DYNAMIC_COUNT, false) :
363 &task->ref_group[grp];
364 break;
365 }
366
367
368 refs = os_ref_release_raw(&task->ref_count.ref_count, refgrp);
369 /* Beware - the task may have been freed after this point. */
370
371 task_deallocate_internal(task, refs);
372 }
373
374 void
task_reference_external(task_t task)375 task_reference_external(task_t task)
376 {
377 task_reference_grp(task, TASK_GRP_EXTERNAL);
378 }
379
380 void
task_deallocate_external(task_t task)381 task_deallocate_external(task_t task)
382 {
383 task_deallocate_grp(task, TASK_GRP_EXTERNAL);
384 }
385
386 static void
allocate_refgrp_default(task_t task)387 allocate_refgrp_default(task_t task)
388 {
389 /* Just one static group and one dynamic group. */
390 task->ref_group = kalloc_type(struct os_refgrp, 2,
391 Z_WAITOK | Z_ZERO | Z_NOFAIL);
392
393 task->ref_group[TASK_GRP_KERNEL] = (struct os_refgrp)
394 os_refgrp_initializer(local_name[TASK_GRP_KERNEL],
395 task_refgrp[TASK_GRP_KERNEL], OS_REFGRP_F_ALWAYS_ENABLED);
396 os_ref_log_init(&task->ref_group[TASK_GRP_KERNEL]);
397 }
398
399 static void
free_refgrp_default(task_t task)400 free_refgrp_default(task_t task)
401 {
402 os_ref_log_fini(&task->ref_group[TASK_GRP_KERNEL]);
403 /* Just one static group and one dynamic group. */
404 kfree_type(struct os_refgrp, 2, task->ref_group);
405 }
406
407 static void
allocate_refgrp_full(task_t task)408 allocate_refgrp_full(task_t task)
409 {
410 task->ref_group = kalloc_type(struct os_refgrp,
411 TASK_GRP_COUNT + DYNAMIC_COUNT, Z_WAITOK | Z_ZERO | Z_NOFAIL);
412
413 for (int i = 0; i < TASK_GRP_COUNT; i++) {
414 task->ref_group[i] = (struct os_refgrp)
415 os_refgrp_initializer(local_name[i], task_refgrp[i],
416 OS_REFGRP_F_ALWAYS_ENABLED);
417 os_ref_log_init(&task->ref_group[i]);
418 }
419 }
420
421 static void
free_refgrp_full(task_t task)422 free_refgrp_full(task_t task)
423 {
424 for (int i = 0; i < TASK_GRP_COUNT; i++) {
425 os_ref_log_fini(&task->ref_group[i]);
426 }
427 kfree_type(struct os_refgrp, TASK_GRP_COUNT + DYNAMIC_COUNT, task->ref_group);
428 }
429
430 kern_return_t
task_ref_count_init(task_t task)431 task_ref_count_init(task_t task)
432 {
433 assert(
434 task_refgrp_config == TASK_REF_CONFIG_OFF ||
435 task_refgrp_config == TASK_REF_CONFIG_DEFAULT ||
436 task_refgrp_config == TASK_REF_CONFIG_FULL);
437
438 switch (task_refgrp_config) {
439 case TASK_REF_CONFIG_OFF:
440 os_ref_init_count(&task->ref_count, &task_primary_refgrp,
441 TASK_REF_COUNT_INITIAL);
442 return KERN_SUCCESS;
443
444
445 case TASK_REF_CONFIG_DEFAULT:
446 allocate_refgrp_default(task);
447 lck_spin_init(&task->ref_group_lock, &task_lck_grp, LCK_ATTR_NULL);
448 os_ref_init_count(&task->ref_count, &task->ref_group[TASK_GRP_KERNEL],
449 TASK_REF_COUNT_INITIAL);
450 return KERN_SUCCESS;
451
452 case TASK_REF_CONFIG_FULL:
453 allocate_refgrp_full(task);
454 lck_spin_init(&task->ref_group_lock, &task_lck_grp, LCK_ATTR_NULL);
455
456 os_ref_init_count_internal(&task->ref_count.ref_count,
457 &task->ref_group[TASK_GRP_KERNEL], 1);
458
459 task_reference_grp(task, TASK_GRP_INTERNAL);
460
461 return KERN_SUCCESS;
462 }
463 }
464
465 void
task_ref_count_fini(task_t task)466 task_ref_count_fini(task_t task)
467 {
468 assert(
469 task_refgrp_config == TASK_REF_CONFIG_OFF ||
470 task_refgrp_config == TASK_REF_CONFIG_DEFAULT ||
471 task_refgrp_config == TASK_REF_CONFIG_FULL);
472
473 switch (task_refgrp_config) {
474 case TASK_REF_CONFIG_OFF:
475 return;
476
477 case TASK_REF_CONFIG_DEFAULT:
478 lck_spin_destroy(&task->ref_group_lock, &task_lck_grp);
479 free_refgrp_default(task);
480 return;
481
482 case TASK_REF_CONFIG_FULL:
483 lck_spin_destroy(&task->ref_group_lock, &task_lck_grp);
484 free_refgrp_full(task);
485 return;
486 }
487 }
488
489 void
task_ref_init(void)490 task_ref_init(void)
491 {
492 char config[16] = {0};
493
494 /* Allow task reference group logging to be configured. */
495 (void) PE_parse_boot_arg_str("task_refgrp", config,
496 sizeof(config));
497
498 if (strncmp(config, "full", sizeof(config)) == 0) {
499 task_refgrp_config = TASK_REF_CONFIG_FULL;
500 }
501 if (strncmp(config, "off", sizeof(config)) == 0) {
502 task_refgrp_config = TASK_REF_CONFIG_OFF;
503 }
504
505 if (task_refgrp_config == TASK_REF_CONFIG_OFF) {
506 return;
507 }
508
509 for (int i = 0; i < TASK_GRP_COUNT; i++) {
510 os_ref_log_init(task_refgrp[i]);
511 }
512 }
513
514 #else /* DEVELOPMENT || DEBUG */
515
516 kern_return_t
task_ref_count_init(task_t task)517 task_ref_count_init(task_t task)
518 {
519 /* One ref for our caller, one for being alive. */
520 os_ref_init_count(&task->ref_count, &task_primary_refgrp,
521 TASK_REF_COUNT_INITIAL);
522 return KERN_SUCCESS;
523 }
524
525 void
task_reference_grp(task_t task,task_grp_t grp)526 task_reference_grp(task_t task, __attribute__((__unused__)) task_grp_t grp)
527 {
528 if (task == TASK_NULL) {
529 return;
530 }
531
532 task_require(task);
533 os_ref_retain(&task->ref_count);
534 }
535
536 void
task_deallocate_grp(task_t task,task_grp_t grp)537 task_deallocate_grp(task_t task, __attribute__((__unused__)) task_grp_t grp)
538 {
539 if (task == TASK_NULL) {
540 return;
541 }
542
543 os_ref_count_t refs = os_ref_release(&task->ref_count);
544 task_deallocate_internal(task, refs);
545 }
546
547 void
task_reference_external(task_t task)548 task_reference_external(task_t task)
549 {
550 task_reference_grp(task, 0);
551 }
552
553 void
task_deallocate_external(task_t task)554 task_deallocate_external(task_t task)
555 {
556 task_deallocate_grp(task, 0);
557 }
558
559 void
task_ref_count_fini(task_t task)560 task_ref_count_fini(__attribute__((__unused__)) task_t task)
561 {
562 }
563
564 void
task_ref_init(void)565 task_ref_init(void)
566 {
567 }
568
569 #endif /* DEVELOPMENT || DEBUG */
570