1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <sys/ioctl.h>
29
30 #include <sys/stat.h>
31 #include <miscfs/devfs/devfs.h>
32 #include <sys/conf.h>
33 #include <sys/systm.h>
34 #include <sys/dtrace.h>
35 #include <sys/dtrace_impl.h>
36 #include <kern/lock_group.h>
37 #include <kern/lock_stat.h>
38
39 #if LOCK_STATS
40
41 #define LP_NODE "lockprof"
42
43 #define LOCKPROF_AFRAMES 3
44 #define LOCKPROF_LEN 64
45
46 static dtrace_provider_id_t lockprof_id;
47
48 decl_lck_mtx_data(extern, lck_grp_lock);
49 extern queue_head_t lck_grp_queue;
50 extern unsigned int lck_grp_cnt;
51
52 extern void lck_grp_reference(lck_grp_t *grp);
53 extern void lck_grp_deallocate(lck_grp_t *grp);
54
55 #define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */
56 static uint32_t lockprof_count; /* current number of lockprof probes */
57
58 enum probe_flags {
59 /*
60 * Counts time spent spinning/blocking
61 */
62 TIME_EVENT = 0x01,
63 /*
64 * Requires LCK_GRP_ATTR_STAT to be set on the lock
65 * group, either via lck_grp_attr_setsta on the lock group,
66 * or globally via the lcks=3 boot-arg
67 */
68 STAT_NEEDED = 0x02
69 };
70
71 static const struct {
72 const char *prefix;
73 int flags;
74 size_t count_offset;
75 size_t stat_offset;
76 } probes[] = {
77 {"spin-held-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_held)},
78 {"spin-miss-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_miss)},
79 {"spin-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_spin)},
80 {"ticket-held-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_held)},
81 {"ticket-miss-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_miss)},
82 {"ticket-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_spin)},
83 #if HAS_EXT_MUTEXES
84 {"adaptive-held-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_held)},
85 {"adaptive-miss-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_miss)},
86 {"adaptive-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_wait)},
87 {"adaptive-direct-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_direct_wait)},
88 #endif /* HAS_EXT_MUTEXES */
89 {NULL, false, 0, 0}
90 };
91
92 /*
93 * Default defined probes for counting events
94 */
95 const static int hold_defaults[] = {
96 10000 /* 10000 events */
97 };
98
99 /*
100 * Default defined probes for time events
101 */
102 const static struct {
103 unsigned int time;
104 const char *suffix;
105 uint64_t mult;
106 } cont_defaults[] = {
107 {100, "ms", NANOSEC / MILLISEC} /* 100 ms */
108 };
109
110 typedef struct lockprof_probe {
111 int lockprof_kind;
112 dtrace_id_t lockprof_id;
113 uint64_t lockprof_limit;
114 lck_grp_t *lockprof_grp;
115 } lockprof_probe_t;
116
117 void
lockprof_invoke(lck_grp_t * grp,lck_grp_stat_t * stat,uint64_t val)118 lockprof_invoke(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t val)
119 {
120 dtrace_probe(stat->lgs_probeid, (uintptr_t)grp, val, 0, 0, 0);
121 }
122
123 static int
lockprof_lock_count(lck_grp_t * grp,int kind)124 lockprof_lock_count(lck_grp_t *grp, int kind)
125 {
126 return *(int*)((uintptr_t)(grp) + probes[kind].count_offset);
127 }
128
129 static void
probe_create(int kind,const char * suffix,const char * grp_name,uint64_t count,uint64_t mult)130 probe_create(int kind, const char *suffix, const char *grp_name, uint64_t count, uint64_t mult)
131 {
132 char name[LOCKPROF_LEN];
133 lck_mtx_lock(&lck_grp_lock);
134 lck_grp_t *grp = (lck_grp_t*)queue_first(&lck_grp_queue);
135 uint64_t limit = count * mult;
136
137 if (probes[kind].flags & TIME_EVENT) {
138 nanoseconds_to_absolutetime(limit, &limit);
139 }
140
141 for (unsigned int i = 0; i < lck_grp_cnt; i++, grp = (lck_grp_t*)queue_next((queue_entry_t)grp)) {
142 if (!grp_name || grp_name[0] == '\0' || strcmp(grp_name, grp->lck_grp_name) == 0) {
143 snprintf(name, sizeof(name), "%s%llu%s", probes[kind].prefix, count, suffix ?: "");
144
145 if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) {
146 continue;
147 }
148 if (lockprof_lock_count(grp, kind) == 0) {
149 continue;
150 }
151 if ((probes[kind].flags & STAT_NEEDED) && !(grp->lck_grp_attr & LCK_GRP_ATTR_STAT)) {
152 continue;
153 }
154 if (lockprof_count >= LOCKPROF_MAX) {
155 break;
156 }
157
158 lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP);
159 probe->lockprof_kind = kind;
160 probe->lockprof_limit = limit;
161 probe->lockprof_grp = grp;
162
163 lck_grp_reference(grp);
164
165 probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name,
166 LOCKPROF_AFRAMES, probe);
167
168 lockprof_count++;
169 }
170 }
171 lck_mtx_unlock(&lck_grp_lock);
172 }
173
174 static void
lockprof_provide(void * arg,const dtrace_probedesc_t * desc)175 lockprof_provide(void *arg, const dtrace_probedesc_t *desc)
176 {
177 #pragma unused(arg)
178 size_t event_id, i, j, len;
179
180 if (desc == NULL) {
181 for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) {
182 for (j = 0; probes[j].prefix != NULL; j++) {
183 if (!(probes[j].flags & TIME_EVENT)) {
184 probe_create(j, NULL, NULL, hold_defaults[i], 1);
185 }
186 }
187 }
188 for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) {
189 for (j = 0; probes[j].prefix != NULL; j++) {
190 if (probes[j].flags & TIME_EVENT) {
191 probe_create(j, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult);
192 }
193 }
194 }
195 return;
196 }
197
198 const char *name, *suffix = NULL;
199 hrtime_t val = 0, mult = 1;
200
201 const struct {
202 const char *name;
203 hrtime_t mult;
204 } suffixes[] = {
205 { "us", NANOSEC / MICROSEC },
206 { "usec", NANOSEC / MICROSEC },
207 { "ms", NANOSEC / MILLISEC },
208 { "msec", NANOSEC / MILLISEC },
209 { "s", NANOSEC / SEC },
210 { "sec", NANOSEC / SEC },
211 { NULL, 0 }
212 };
213
214 name = desc->dtpd_name;
215
216 for (event_id = 0; probes[event_id].prefix != NULL; event_id++) {
217 len = strlen(probes[event_id].prefix);
218
219 if (strncmp(name, probes[event_id].prefix, len) != 0) {
220 continue;
221 }
222 break;
223 }
224
225 if (probes[event_id].prefix == NULL) {
226 return;
227 }
228
229
230 /*
231 * We need to start before any time suffix.
232 */
233 for (i = strlen(name); i >= len; i--) {
234 if (name[i] >= '0' && name[i] <= '9') {
235 break;
236 }
237 suffix = &name[i];
238 }
239
240 /*
241 * Now determine the numerical value present in the probe name.
242 */
243 for (uint64_t m = 1; i >= len; i--) {
244 if (name[i] < '0' || name[i] > '9') {
245 return;
246 }
247
248 val += (name[i] - '0') * m;
249 m *= (hrtime_t)10;
250 }
251
252 if (val == 0) {
253 return;
254 }
255
256 if (probes[event_id].flags & TIME_EVENT) {
257 for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
258 if (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0) {
259 mult = suffixes[i].mult;
260 break;
261 }
262 }
263 if (suffixes[i].name == NULL) {
264 return;
265 }
266 } else if (*suffix != '\0') {
267 return;
268 }
269
270 probe_create(event_id, suffix, desc->dtpd_mod, val, mult);
271 }
272
273
274 static lck_grp_stat_t*
lockprof_stat(lck_grp_t * grp,int kind)275 lockprof_stat(lck_grp_t *grp, int kind)
276 {
277 return (lck_grp_stat_t*)((uintptr_t)&grp->lck_grp_stats + probes[kind].stat_offset);
278 }
279
280 static int
lockprof_enable(void * arg,dtrace_id_t id,void * parg)281 lockprof_enable(void *arg, dtrace_id_t id, void *parg)
282 {
283 #pragma unused(arg, id, parg)
284 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
285 lck_grp_t *grp = probe->lockprof_grp;
286 lck_grp_stat_t *stat;
287
288 if (grp == NULL) {
289 return -1;
290 }
291
292 if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
293 return -1;
294 }
295
296 /*
297 * lockprof_enable/disable are called with
298 * dtrace_lock held
299 */
300 if (stat->lgs_limit != 0) {
301 return -1;
302 }
303
304 stat->lgs_limit = probe->lockprof_limit;
305 stat->lgs_enablings++;
306 stat->lgs_probeid = probe->lockprof_id;
307
308 return 0;
309 }
310
311 static void
lockprof_disable(void * arg,dtrace_id_t id,void * parg)312 lockprof_disable(void *arg, dtrace_id_t id, void *parg)
313 {
314 #pragma unused(arg, id)
315 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
316 lck_grp_t *grp = probe->lockprof_grp;
317 lck_grp_stat_t *stat;
318
319 if (grp == NULL) {
320 return;
321 }
322
323 if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
324 return;
325 }
326
327 if (stat->lgs_limit == 0 || stat->lgs_enablings == 0) {
328 return;
329 }
330
331 stat->lgs_limit = 0;
332 stat->lgs_enablings--;
333 stat->lgs_probeid = 0;
334 }
335
336 static void
lockprof_destroy(void * arg,dtrace_id_t id,void * parg)337 lockprof_destroy(void *arg, dtrace_id_t id, void *parg)
338 {
339 #pragma unused(arg, id)
340 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
341 lck_grp_deallocate(probe->lockprof_grp);
342 kmem_free(probe, sizeof(lockprof_probe_t));
343 lockprof_count--;
344 }
345
346 static void
lockprof_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)347 lockprof_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
348 {
349 #pragma unused(arg, id, parg)
350 const char *argdesc = NULL;
351 switch (desc->dtargd_ndx) {
352 case 0:
353 argdesc = "lck_grp_t*";
354 break;
355 case 1:
356 argdesc = "uint64_t";
357 break;
358 }
359
360 if (argdesc) {
361 strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
362 } else {
363 desc->dtargd_ndx = DTRACE_ARGNONE;
364 }
365 }
366 static dtrace_pattr_t lockprof_attr = {
367 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
368 { DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
369 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
370 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
371 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
372 };
373
374 static dtrace_pops_t lockprof_pops = {
375 .dtps_provide = lockprof_provide,
376 .dtps_provide_module = NULL,
377 .dtps_enable = lockprof_enable,
378 .dtps_disable = lockprof_disable,
379 .dtps_suspend = NULL,
380 .dtps_resume = NULL,
381 .dtps_getargdesc = lockprof_getargdesc,
382 .dtps_getargval = NULL,
383 .dtps_usermode = NULL,
384 .dtps_destroy = lockprof_destroy
385 };
386
387 static int
_lockprof_open(dev_t dev,int flags,int devtype,struct proc * p)388 _lockprof_open(dev_t dev, int flags, int devtype, struct proc *p)
389 {
390 #pragma unused(dev,flags,devtype,p)
391 return 0;
392 }
393
394 static const struct cdevsw lockprof_cdevsw =
395 {
396 .d_open = _lockprof_open,
397 .d_close = eno_opcl,
398 .d_read = eno_rdwrt,
399 .d_write = eno_rdwrt,
400 .d_ioctl = eno_ioctl,
401 .d_stop = (stop_fcn_t *)nulldev,
402 .d_reset = (reset_fcn_t *)nulldev,
403 .d_select = eno_select,
404 .d_mmap = eno_mmap,
405 .d_strategy = eno_strat,
406 .d_reserved_1 = eno_getc,
407 .d_reserved_2 = eno_putc,
408 };
409
410
411 #endif /* LOCK_STATS */
412 void lockprof_init(void);
413 void
lockprof_init(void)414 lockprof_init(void)
415 {
416 #if LOCK_STATS
417 int majorno = cdevsw_add(-1, &lockprof_cdevsw);
418
419 if (majorno < 0) {
420 panic("dtrace: failed to allocate a major number");
421 return;
422 }
423
424 if (dtrace_register(LP_NODE, &lockprof_attr, DTRACE_PRIV_KERNEL,
425 NULL, &lockprof_pops, NULL, &lockprof_id) != 0) {
426 panic("dtrace: failed to register lockprof provider");
427 }
428
429 dev_t dev = makedev(majorno, 0);
430
431 if (devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
432 LP_NODE, 0 ) == NULL) {
433 panic("dtrace: devfs_make_node failed for lockprof");
434 }
435
436 #endif /* LOCK_STATS */
437 }
438