1 /*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #define LOCK_PRIVATE 1
58
59 #include <mach_ldebug.h>
60 #include <debug.h>
61
62 #include <mach/mach_host_server.h>
63 #include <mach_debug/lockgroup_info.h>
64
65 #include <kern/kalloc.h>
66 #include <kern/locks.h>
67 #include <kern/lock_stat.h>
68 #include <os/atomic_private.h>
69
70 static KALLOC_TYPE_DEFINE(KT_LCK_GRP_ATTR, lck_grp_attr_t, KT_PRIV_ACCT);
71 static KALLOC_TYPE_DEFINE(KT_LCK_GRP, lck_grp_t, KT_PRIV_ACCT);
72 static KALLOC_TYPE_DEFINE(KT_LCK_ATTR, lck_attr_t, KT_PRIV_ACCT);
73
74 SECURITY_READ_ONLY_LATE(lck_attr_t) LockDefaultLckAttr;
75 static SECURITY_READ_ONLY_LATE(lck_grp_attr_t) lck_grp_attr_default;
76 static lck_grp_t lck_grp_compat_grp;
77 queue_head_t lck_grp_queue;
78 unsigned int lck_grp_cnt;
79 static LCK_MTX_DECLARE(lck_grp_lock, &lck_grp_compat_grp);
80
81 #pragma mark lock group attributes
82
83 lck_grp_attr_t *
lck_grp_attr_alloc_init(void)84 lck_grp_attr_alloc_init(void)
85 {
86 lck_grp_attr_t *attr;
87
88 attr = zalloc(KT_LCK_GRP_ATTR);
89 lck_grp_attr_setdefault(attr);
90 return attr;
91 }
92
93 void
lck_grp_attr_setdefault(lck_grp_attr_t * attr)94 lck_grp_attr_setdefault(lck_grp_attr_t *attr)
95 {
96 attr->grp_attr_val = lck_grp_attr_default.grp_attr_val;
97 }
98
99 void
lck_grp_attr_setstat(lck_grp_attr_t * attr __unused)100 lck_grp_attr_setstat(lck_grp_attr_t *attr __unused)
101 {
102 attr->grp_attr_val |= LCK_GRP_ATTR_STAT;
103 }
104
105
106 void
lck_grp_attr_free(lck_grp_attr_t * attr)107 lck_grp_attr_free(lck_grp_attr_t *attr)
108 {
109 zfree(KT_LCK_GRP_ATTR, attr);
110 }
111
112 #pragma mark lock groups
113
114 __startup_func
115 static void
lck_group_init(void)116 lck_group_init(void)
117 {
118 queue_init(&lck_grp_queue);
119
120 if (LcksOpts & enaLkStat) {
121 lck_grp_attr_default.grp_attr_val |= LCK_GRP_ATTR_STAT;
122 }
123 if (LcksOpts & enaLkTimeStat) {
124 lck_grp_attr_default.grp_attr_val |= LCK_GRP_ATTR_TIME_STAT;
125 }
126
127 #if __arm__ || __arm64__
128 /* <rdar://problem/4404579>: Using LCK_ATTR_DEBUG here causes panic at boot time for arm */
129 LcksOpts &= ~enaLkDeb;
130 #endif
131 if (LcksOpts & enaLkDeb) {
132 LockDefaultLckAttr.lck_attr_val = LCK_ATTR_DEBUG;
133 } else {
134 LockDefaultLckAttr.lck_attr_val = LCK_ATTR_NONE;
135 }
136
137 lck_grp_init(&lck_grp_compat_grp, "Compatibility APIs",
138 &lck_grp_attr_default);
139 }
140 STARTUP(LOCKS_EARLY, STARTUP_RANK_FIRST, lck_group_init);
141
142 __startup_func
143 void
lck_grp_startup_init(struct lck_grp_spec * sp)144 lck_grp_startup_init(struct lck_grp_spec *sp)
145 {
146 lck_grp_init_flags(sp->grp, sp->grp_name, sp->grp_flags |
147 lck_grp_attr_default.grp_attr_val);
148 }
149
150 bool
lck_grp_has_stats(lck_grp_t * grp)151 lck_grp_has_stats(lck_grp_t *grp)
152 {
153 return grp->lck_grp_attr & LCK_GRP_ATTR_STAT;
154 }
155
156 lck_grp_t *
lck_grp_alloc_init(const char * grp_name,lck_grp_attr_t * attr)157 lck_grp_alloc_init(const char *grp_name, lck_grp_attr_t *attr)
158 {
159 lck_grp_t *grp;
160
161 if (attr == LCK_GRP_ATTR_NULL) {
162 attr = &lck_grp_attr_default;
163 }
164 grp = zalloc(KT_LCK_GRP);
165 lck_grp_init_flags(grp, grp_name,
166 attr->grp_attr_val | LCK_GRP_ATTR_ALLOCATED);
167 return grp;
168 }
169
170 void
lck_grp_init(lck_grp_t * grp,const char * grp_name,lck_grp_attr_t * attr)171 lck_grp_init(lck_grp_t *grp, const char *grp_name, lck_grp_attr_t *attr)
172 {
173 if (attr == LCK_GRP_ATTR_NULL) {
174 attr = &lck_grp_attr_default;
175 }
176 lck_grp_init_flags(grp, grp_name, attr->grp_attr_val);
177 }
178
179 lck_grp_t *
lck_grp_init_flags(lck_grp_t * grp,const char * grp_name,lck_grp_options_t flags)180 lck_grp_init_flags(lck_grp_t *grp, const char *grp_name, lck_grp_options_t flags)
181 {
182 bzero(grp, sizeof(lck_grp_t));
183
184 (void)strlcpy(grp->lck_grp_name, grp_name, LCK_GRP_MAX_NAME);
185
186 grp->lck_grp_attr = flags;
187
188 if (grp->lck_grp_attr & LCK_GRP_ATTR_STAT) {
189 lck_grp_stats_t *stats = &grp->lck_grp_stats;
190
191 #if LOCK_STATS
192 lck_grp_stat_enable(&stats->lgss_spin_held);
193 lck_grp_stat_enable(&stats->lgss_spin_miss);
194 #endif /* LOCK_STATS */
195
196 lck_grp_stat_enable(&stats->lgss_mtx_held);
197 lck_grp_stat_enable(&stats->lgss_mtx_miss);
198 lck_grp_stat_enable(&stats->lgss_mtx_direct_wait);
199 lck_grp_stat_enable(&stats->lgss_mtx_wait);
200 }
201 if (grp->lck_grp_attr & LCK_GRP_ATTR_TIME_STAT) {
202 #if LOCK_STATS
203 lck_grp_stats_t *stats = &grp->lck_grp_stats;
204 lck_grp_stat_enable(&stats->lgss_spin_spin);
205 #endif /* LOCK_STATS */
206 }
207
208 os_ref_init(&grp->lck_grp_refcnt, NULL);
209
210 if (startup_phase >= STARTUP_SUB_LOCKS) {
211 lck_mtx_lock(&lck_grp_lock);
212 }
213
214 enqueue_tail(&lck_grp_queue, &grp->lck_grp_link);
215 lck_grp_cnt++;
216
217 if (startup_phase >= STARTUP_SUB_LOCKS) {
218 lck_mtx_unlock(&lck_grp_lock);
219 }
220
221 return grp;
222 }
223
224 static void
lck_grp_destroy(lck_grp_t * grp)225 lck_grp_destroy(lck_grp_t *grp)
226 {
227 lck_mtx_lock(&lck_grp_lock);
228 lck_grp_cnt--;
229 remque(&grp->lck_grp_link);
230 lck_mtx_unlock(&lck_grp_lock);
231 zfree(KT_LCK_GRP, grp);
232 }
233
234 void
lck_grp_free(lck_grp_t * grp)235 lck_grp_free(lck_grp_t *grp)
236 {
237 lck_grp_deallocate(grp, NULL);
238 }
239
240
241 void
lck_grp_reference(lck_grp_t * grp,uint32_t * cnt)242 lck_grp_reference(lck_grp_t *grp, uint32_t *cnt)
243 {
244 if (cnt) {
245 os_atomic_inc(cnt, relaxed);
246 }
247 if (grp->lck_grp_attr & LCK_GRP_ATTR_ALLOCATED) {
248 os_ref_retain(&grp->lck_grp_refcnt);
249 }
250 }
251
252 void
lck_grp_deallocate(lck_grp_t * grp,uint32_t * cnt)253 lck_grp_deallocate(lck_grp_t *grp, uint32_t *cnt)
254 {
255 if (cnt) {
256 os_atomic_dec(cnt, relaxed);
257 }
258 if ((grp->lck_grp_attr & LCK_GRP_ATTR_ALLOCATED) &&
259 os_ref_release(&grp->lck_grp_refcnt) == 0) {
260 lck_grp_destroy(grp);
261 }
262 }
263
264 static void
265 lck_grp_foreach_locked(bool (^block)(lck_grp_t *))
266 {
267 lck_grp_t *grp;
268
269 qe_foreach_element(grp, &lck_grp_queue, lck_grp_link) {
270 if (!block(grp)) {
271 return;
272 }
273 }
274 }
275
276
277 void
278 lck_grp_foreach(bool (^block)(lck_grp_t *))
279 {
280 lck_mtx_lock(&lck_grp_lock);
281 lck_grp_foreach_locked(block);
282 lck_mtx_unlock(&lck_grp_lock);
283 }
284
285 kern_return_t
host_lockgroup_info(host_t host,lockgroup_info_array_t * lockgroup_infop,mach_msg_type_number_t * lockgroup_infoCntp)286 host_lockgroup_info(
287 host_t host,
288 lockgroup_info_array_t *lockgroup_infop,
289 mach_msg_type_number_t *lockgroup_infoCntp)
290 {
291 lockgroup_info_t *info;
292 vm_offset_t addr;
293 vm_size_t size, used;
294 vm_size_t vmsize, vmused;
295 uint32_t needed;
296 __block uint32_t count = 0;
297 vm_map_copy_t copy;
298 kern_return_t kr;
299
300 if (host == HOST_NULL) {
301 return KERN_INVALID_HOST;
302 }
303
304 needed = os_atomic_load(&lck_grp_cnt, relaxed);
305
306 for (;;) {
307 size = needed * sizeof(lockgroup_info_t);
308 vmsize = vm_map_round_page(size, VM_MAP_PAGE_MASK(ipc_kernel_map));
309 kr = kernel_memory_allocate(ipc_kernel_map, &addr, vmsize,
310 0, KMA_ZERO, VM_KERN_MEMORY_IPC);
311 if (kr != KERN_SUCCESS) {
312 return kr;
313 }
314
315 lck_mtx_lock(&lck_grp_lock);
316 if (needed >= lck_grp_cnt) {
317 break;
318 }
319 needed = lck_grp_cnt;
320 lck_mtx_unlock(&lck_grp_lock);
321
322 kmem_free(ipc_kernel_map, addr, vmsize);
323 }
324
325 info = (lockgroup_info_t *)addr;
326
327 lck_grp_foreach_locked(^bool (lck_grp_t *grp) {
328 info[count].lock_spin_cnt = grp->lck_grp_spincnt;
329 info[count].lock_rw_cnt = grp->lck_grp_rwcnt;
330 info[count].lock_mtx_cnt = grp->lck_grp_mtxcnt;
331
332 #if LOCK_STATS
333 info[count].lock_spin_held_cnt = grp->lck_grp_stats.lgss_spin_held.lgs_count;
334 info[count].lock_spin_miss_cnt = grp->lck_grp_stats.lgss_spin_miss.lgs_count;
335 #endif /* LOCK_STATS */
336
337 // Historically on x86, held was used for "direct wait" and util for "held"
338 info[count].lock_mtx_util_cnt = grp->lck_grp_stats.lgss_mtx_held.lgs_count;
339 info[count].lock_mtx_held_cnt = grp->lck_grp_stats.lgss_mtx_direct_wait.lgs_count;
340 info[count].lock_mtx_miss_cnt = grp->lck_grp_stats.lgss_mtx_miss.lgs_count;
341 info[count].lock_mtx_wait_cnt = grp->lck_grp_stats.lgss_mtx_wait.lgs_count;
342
343 memcpy(info[count].lockgroup_name, grp->lck_grp_name, LOCKGROUP_MAX_NAME);
344
345 count++;
346 return true;
347 });
348
349 lck_mtx_unlock(&lck_grp_lock);
350
351 /*
352 * We might have found less groups than `needed`
353 * get rid of the excess now:
354 * - [0, used) is what we want to return
355 * - [0, size) is what we allocated
356 */
357 used = count * sizeof(lockgroup_info_t);
358 vmused = vm_map_round_page(used, VM_MAP_PAGE_MASK(ipc_kernel_map));
359
360 if (vmused < vmsize) {
361 kmem_free(ipc_kernel_map, addr + vmused, vmsize - vmused);
362 }
363
364 kr = vm_map_unwire(ipc_kernel_map, addr, addr + vmused, FALSE);
365 assert(kr == KERN_SUCCESS);
366
367 kr = vm_map_copyin(ipc_kernel_map, addr, used, TRUE, ©);
368 assert(kr == KERN_SUCCESS);
369
370 *lockgroup_infop = (lockgroup_info_t *)copy;
371 *lockgroup_infoCntp = count;
372
373 return KERN_SUCCESS;
374 }
375
376 #pragma mark lock attributes
377
378 __startup_func
379 void
lck_attr_startup_init(struct lck_attr_startup_spec * sp)380 lck_attr_startup_init(struct lck_attr_startup_spec *sp)
381 {
382 lck_attr_t *attr = sp->lck_attr;
383 lck_attr_setdefault(attr);
384 attr->lck_attr_val |= sp->lck_attr_set_flags;
385 attr->lck_attr_val &= ~sp->lck_attr_clear_flags;
386 }
387
388 lck_attr_t *
lck_attr_alloc_init(void)389 lck_attr_alloc_init(void)
390 {
391 lck_attr_t *attr;
392
393 attr = zalloc(KT_LCK_ATTR);
394 lck_attr_setdefault(attr);
395 return attr;
396 }
397
398
399 void
lck_attr_setdefault(lck_attr_t * attr)400 lck_attr_setdefault(lck_attr_t *attr)
401 {
402 attr->lck_attr_val = LockDefaultLckAttr.lck_attr_val;
403 }
404
405
406 void
lck_attr_setdebug(lck_attr_t * attr)407 lck_attr_setdebug(lck_attr_t *attr)
408 {
409 os_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG, relaxed);
410 }
411
412 void
lck_attr_cleardebug(lck_attr_t * attr)413 lck_attr_cleardebug(lck_attr_t *attr)
414 {
415 os_atomic_andnot(&attr->lck_attr_val, LCK_ATTR_DEBUG, relaxed);
416 }
417
418 void
lck_attr_rw_shared_priority(lck_attr_t * attr)419 lck_attr_rw_shared_priority(lck_attr_t *attr)
420 {
421 os_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY, relaxed);
422 }
423
424
425 void
lck_attr_free(lck_attr_t * attr)426 lck_attr_free(lck_attr_t *attr)
427 {
428 zfree(KT_LCK_ATTR, attr);
429 }
430 #pragma mark lock stat
431
432 void
lck_grp_stat_enable(lck_grp_stat_t * stat)433 lck_grp_stat_enable(lck_grp_stat_t *stat)
434 {
435 /* callers ensure this is properly synchronized */
436 stat->lgs_enablings++;
437 }
438
439 void
lck_grp_stat_disable(lck_grp_stat_t * stat)440 lck_grp_stat_disable(lck_grp_stat_t *stat)
441 {
442 stat->lgs_enablings--;
443 }
444
445 bool
lck_grp_stat_enabled(lck_grp_stat_t * stat)446 lck_grp_stat_enabled(lck_grp_stat_t *stat)
447 {
448 return stat->lgs_enablings != 0;
449 }
450
451 #if CONFIG_DTRACE || LOCK_STATS
452 #if LOCK_STATS || __x86_64__
453
454 static inline void
lck_grp_inc_stats(lck_grp_t * grp,lck_grp_stat_t * stat)455 lck_grp_inc_stats(lck_grp_t *grp, lck_grp_stat_t *stat)
456 {
457 #pragma unused(grp)
458 if (lck_grp_stat_enabled(stat)) {
459 __unused uint64_t val = os_atomic_inc_orig(&stat->lgs_count, relaxed);
460 #if CONFIG_DTRACE && LOCK_STATS
461 if (__improbable(stat->lgs_limit && (val % (stat->lgs_limit)) == 0)) {
462 lockprof_invoke(grp, stat, val);
463 }
464 #endif /* CONFIG_DTRACE && LOCK_STATS */
465 }
466 }
467
468 #endif
469 #if LOCK_STATS
470
471 static inline void
lck_grp_inc_time_stats(lck_grp_t * grp,lck_grp_stat_t * stat,uint64_t time)472 lck_grp_inc_time_stats(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t time)
473 {
474 if (lck_grp_stat_enabled(stat)) {
475 __unused uint64_t val = os_atomic_add_orig(&stat->lgs_count, time, relaxed);
476 #if CONFIG_DTRACE
477 if (__improbable(stat->lgs_limit)) {
478 while (__improbable(time > stat->lgs_limit)) {
479 time -= stat->lgs_limit;
480 lockprof_invoke(grp, stat, val);
481 }
482 if (__improbable(((val % stat->lgs_limit) + time) > stat->lgs_limit)) {
483 lockprof_invoke(grp, stat, val);
484 }
485 }
486 #endif /* CONFIG_DTRACE */
487 }
488 }
489
490 void
__lck_grp_spin_update_held(lck_grp_t * grp)491 __lck_grp_spin_update_held(lck_grp_t *grp)
492 {
493 if (grp) {
494 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_spin_held);
495 }
496 }
497
498 void
__lck_grp_spin_update_miss(lck_grp_t * grp)499 __lck_grp_spin_update_miss(lck_grp_t *grp)
500 {
501 if (grp) {
502 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_spin_miss);
503 }
504 }
505
506 void
__lck_grp_spin_update_spin(lck_grp_t * grp,uint64_t time)507 __lck_grp_spin_update_spin(lck_grp_t *grp, uint64_t time)
508 {
509 if (grp) {
510 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_spin_spin;
511 lck_grp_inc_time_stats(grp, stat, time);
512 }
513 }
514
515 void
__lck_grp_ticket_update_held(lck_grp_t * grp)516 __lck_grp_ticket_update_held(lck_grp_t *grp)
517 {
518 if (grp) {
519 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_ticket_held);
520 }
521 }
522
523 void
__lck_grp_ticket_update_miss(lck_grp_t * grp)524 __lck_grp_ticket_update_miss(lck_grp_t *grp)
525 {
526 if (grp) {
527 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_ticket_miss);
528 }
529 }
530
531 void
__lck_grp_ticket_update_spin(lck_grp_t * grp,uint64_t time)532 __lck_grp_ticket_update_spin(lck_grp_t *grp, uint64_t time)
533 {
534 if (grp) {
535 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_ticket_spin;
536 lck_grp_inc_time_stats(grp, stat, time);
537 }
538 }
539
540 #endif /* LOCK_STATS */
541 #if __x86_64__
542
543 void
__lck_grp_mtx_update_miss(lck_grp_t * grp)544 __lck_grp_mtx_update_miss(lck_grp_t *grp)
545 {
546 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_mtx_miss);
547 }
548
549 void
__lck_grp_mtx_update_direct_wait(lck_grp_t * grp)550 __lck_grp_mtx_update_direct_wait(lck_grp_t *grp)
551 {
552 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_mtx_direct_wait);
553 }
554
555 void
__lck_grp_mtx_update_wait(lck_grp_t * grp)556 __lck_grp_mtx_update_wait(lck_grp_t *grp)
557 {
558 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_mtx_wait);
559 }
560
561 void
__lck_grp_mtx_update_held(lck_grp_t * grp)562 __lck_grp_mtx_update_held(lck_grp_t *grp)
563 {
564 lck_grp_inc_stats(grp, &grp->lck_grp_stats.lgss_mtx_held);
565 }
566
567 #endif /* __x86_64__ */
568 #endif /* CONFIG_DTRACE || LOCK_STATS */
569