1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <sys/ioctl.h>
29
30 #include <sys/stat.h>
31 #include <miscfs/devfs/devfs.h>
32 #include <sys/conf.h>
33 #include <sys/systm.h>
34 #include <sys/dtrace.h>
35 #include <sys/dtrace_impl.h>
36 #include <kern/lock_group.h>
37 #include <kern/lock_stat.h>
38
39 #if LOCK_STATS
40
41 #define LP_NODE "lockprof"
42
43 #define LOCKPROF_AFRAMES 3
44 #define LOCKPROF_LEN 64
45
46 static dtrace_provider_id_t lockprof_id;
47
48 #define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */
49 static uint32_t lockprof_count; /* current number of lockprof probes */
50
51 enum probe_flags {
52 /*
53 * Counts time spent spinning/blocking
54 */
55 TIME_EVENT = 0x01,
56 /*
57 * Requires LCK_GRP_ATTR_STAT to be set on the lock
58 * group, either via lck_grp_attr_setsta on the lock group,
59 * or globally via the lcks=3 boot-arg
60 */
61 STAT_NEEDED = 0x02
62 };
63
64 static const struct {
65 const char *prefix;
66 int flags;
67 size_t count_offset;
68 size_t stat_offset;
69 } probes[] = {
70 {"spin-held-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_held)},
71 {"spin-miss-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_miss)},
72 {"spin-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_spin)},
73 {"ticket-held-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_held)},
74 {"ticket-miss-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_miss)},
75 {"ticket-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_spin)},
76 #if HAS_EXT_MUTEXES
77 {"adaptive-held-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_held)},
78 {"adaptive-miss-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_miss)},
79 {"adaptive-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_wait)},
80 {"adaptive-direct-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_direct_wait)},
81 #endif /* HAS_EXT_MUTEXES */
82 {NULL, false, 0, 0}
83 };
84
85 /*
86 * Default defined probes for counting events
87 */
88 const static int hold_defaults[] = {
89 10000 /* 10000 events */
90 };
91
92 /*
93 * Default defined probes for time events
94 */
95 const static struct {
96 unsigned int time;
97 const char *suffix;
98 uint64_t mult;
99 } cont_defaults[] = {
100 {100, "ms", NANOSEC / MILLISEC} /* 100 ms */
101 };
102
103 typedef struct lockprof_probe {
104 int lockprof_kind;
105 dtrace_id_t lockprof_id;
106 uint64_t lockprof_limit;
107 lck_grp_t *lockprof_grp;
108 } lockprof_probe_t;
109
110 void
lockprof_invoke(lck_grp_t * grp,lck_grp_stat_t * stat,uint64_t val)111 lockprof_invoke(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t val)
112 {
113 dtrace_probe(stat->lgs_probeid, (uintptr_t)grp, val, 0, 0, 0);
114 }
115
116 static int
lockprof_lock_count(lck_grp_t * grp,int kind)117 lockprof_lock_count(lck_grp_t *grp, int kind)
118 {
119 return *(int*)((uintptr_t)(grp) + probes[kind].count_offset);
120 }
121
122 static void
probe_create(int kind,const char * suffix,const char * grp_name,uint64_t count,uint64_t mult)123 probe_create(int kind, const char *suffix, const char *grp_name, uint64_t count, uint64_t mult)
124 {
125 uint64_t limit = count * mult;
126
127 if (probes[kind].flags & TIME_EVENT) {
128 nanoseconds_to_absolutetime(limit, &limit);
129 }
130
131 lck_grp_foreach(^bool (lck_grp_t *grp) {
132 char name[LOCKPROF_LEN];
133
134 if (!grp_name || grp_name[0] == '\0' || strcmp(grp_name, grp->lck_grp_name) == 0) {
135 snprintf(name, sizeof(name), "%s%llu%s", probes[kind].prefix, count, suffix ?: "");
136
137 if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) {
138 return true;
139 }
140 if (lockprof_lock_count(grp, kind) == 0) {
141 return true;
142 }
143 if ((probes[kind].flags & STAT_NEEDED) && !lck_grp_has_stats(grp)) {
144 return true;
145 }
146 if (lockprof_count >= LOCKPROF_MAX) {
147 return false;
148 }
149
150 lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP);
151 probe->lockprof_kind = kind;
152 probe->lockprof_limit = limit;
153 probe->lockprof_grp = grp;
154
155 lck_grp_reference(grp, NULL);
156
157 probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name,
158 LOCKPROF_AFRAMES, probe);
159
160 lockprof_count++;
161 }
162
163 return true;
164 });
165 }
166
167 static void
lockprof_provide(void * arg,const dtrace_probedesc_t * desc)168 lockprof_provide(void *arg, const dtrace_probedesc_t *desc)
169 {
170 #pragma unused(arg)
171 size_t event_id, i, j, len;
172
173 if (desc == NULL) {
174 for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) {
175 for (j = 0; probes[j].prefix != NULL; j++) {
176 if (!(probes[j].flags & TIME_EVENT)) {
177 probe_create(j, NULL, NULL, hold_defaults[i], 1);
178 }
179 }
180 }
181 for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) {
182 for (j = 0; probes[j].prefix != NULL; j++) {
183 if (probes[j].flags & TIME_EVENT) {
184 probe_create(j, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult);
185 }
186 }
187 }
188 return;
189 }
190
191 const char *name, *suffix = NULL;
192 hrtime_t val = 0, mult = 1;
193
194 const struct {
195 const char *name;
196 hrtime_t mult;
197 } suffixes[] = {
198 { "us", NANOSEC / MICROSEC },
199 { "usec", NANOSEC / MICROSEC },
200 { "ms", NANOSEC / MILLISEC },
201 { "msec", NANOSEC / MILLISEC },
202 { "s", NANOSEC / SEC },
203 { "sec", NANOSEC / SEC },
204 { NULL, 0 }
205 };
206
207 name = desc->dtpd_name;
208
209 for (event_id = 0; probes[event_id].prefix != NULL; event_id++) {
210 len = strlen(probes[event_id].prefix);
211
212 if (strncmp(name, probes[event_id].prefix, len) != 0) {
213 continue;
214 }
215 break;
216 }
217
218 if (probes[event_id].prefix == NULL) {
219 return;
220 }
221
222
223 /*
224 * We need to start before any time suffix.
225 */
226 for (i = strlen(name); i >= len; i--) {
227 if (name[i] >= '0' && name[i] <= '9') {
228 break;
229 }
230 suffix = &name[i];
231 }
232
233 /*
234 * Now determine the numerical value present in the probe name.
235 */
236 for (uint64_t m = 1; i >= len; i--) {
237 if (name[i] < '0' || name[i] > '9') {
238 return;
239 }
240
241 val += (name[i] - '0') * m;
242 m *= (hrtime_t)10;
243 }
244
245 if (val == 0) {
246 return;
247 }
248
249 if (probes[event_id].flags & TIME_EVENT) {
250 for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
251 if (suffix && (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0)) {
252 mult = suffixes[i].mult;
253 break;
254 }
255 }
256 if (suffixes[i].name == NULL) {
257 return;
258 }
259 } else if (suffix && (*suffix != '\0')) {
260 return;
261 }
262
263 probe_create(event_id, suffix, desc->dtpd_mod, val, mult);
264 }
265
266
267 static lck_grp_stat_t*
lockprof_stat(lck_grp_t * grp,int kind)268 lockprof_stat(lck_grp_t *grp, int kind)
269 {
270 return (lck_grp_stat_t*)((uintptr_t)&grp->lck_grp_stats + probes[kind].stat_offset);
271 }
272
273 static int
lockprof_enable(void * arg,dtrace_id_t id,void * parg)274 lockprof_enable(void *arg, dtrace_id_t id, void *parg)
275 {
276 #pragma unused(arg, id, parg)
277 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
278 lck_grp_t *grp = probe->lockprof_grp;
279 lck_grp_stat_t *stat;
280
281 if (grp == NULL) {
282 return -1;
283 }
284
285 if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
286 return -1;
287 }
288
289 /*
290 * lockprof_enable/disable are called with
291 * dtrace_lock held
292 */
293 if (stat->lgs_limit != 0) {
294 return -1;
295 }
296
297 stat->lgs_limit = probe->lockprof_limit;
298 stat->lgs_probeid = probe->lockprof_id;
299 lck_grp_stat_enable(stat);
300
301 return 0;
302 }
303
304 static void
lockprof_disable(void * arg,dtrace_id_t id,void * parg)305 lockprof_disable(void *arg, dtrace_id_t id, void *parg)
306 {
307 #pragma unused(arg, id)
308 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
309 lck_grp_t *grp = probe->lockprof_grp;
310 lck_grp_stat_t *stat;
311
312 if (grp == NULL) {
313 return;
314 }
315
316 if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
317 return;
318 }
319
320 if (stat->lgs_limit == 0 || !lck_grp_stat_enabled(stat)) {
321 return;
322 }
323
324 stat->lgs_limit = 0;
325 stat->lgs_probeid = 0;
326 lck_grp_stat_disable(stat);
327 }
328
329 static void
lockprof_destroy(void * arg,dtrace_id_t id,void * parg)330 lockprof_destroy(void *arg, dtrace_id_t id, void *parg)
331 {
332 #pragma unused(arg, id)
333 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
334 lck_grp_deallocate(probe->lockprof_grp, NULL);
335 kmem_free(probe, sizeof(lockprof_probe_t));
336 lockprof_count--;
337 }
338
339 static void
lockprof_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)340 lockprof_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
341 {
342 #pragma unused(arg, id, parg)
343 const char *argdesc = NULL;
344 switch (desc->dtargd_ndx) {
345 case 0:
346 argdesc = "lck_grp_t*";
347 break;
348 case 1:
349 argdesc = "uint64_t";
350 break;
351 }
352
353 if (argdesc) {
354 strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
355 } else {
356 desc->dtargd_ndx = DTRACE_ARGNONE;
357 }
358 }
359 static dtrace_pattr_t lockprof_attr = {
360 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
361 { DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
362 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
363 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
364 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
365 };
366
367 static dtrace_pops_t lockprof_pops = {
368 .dtps_provide = lockprof_provide,
369 .dtps_provide_module = NULL,
370 .dtps_enable = lockprof_enable,
371 .dtps_disable = lockprof_disable,
372 .dtps_suspend = NULL,
373 .dtps_resume = NULL,
374 .dtps_getargdesc = lockprof_getargdesc,
375 .dtps_getargval = NULL,
376 .dtps_usermode = NULL,
377 .dtps_destroy = lockprof_destroy
378 };
379
380 static int
_lockprof_open(dev_t dev,int flags,int devtype,struct proc * p)381 _lockprof_open(dev_t dev, int flags, int devtype, struct proc *p)
382 {
383 #pragma unused(dev,flags,devtype,p)
384 return 0;
385 }
386
387 static const struct cdevsw lockprof_cdevsw =
388 {
389 .d_open = _lockprof_open,
390 .d_close = eno_opcl,
391 .d_read = eno_rdwrt,
392 .d_write = eno_rdwrt,
393 .d_ioctl = eno_ioctl,
394 .d_stop = (stop_fcn_t *)nulldev,
395 .d_reset = (reset_fcn_t *)nulldev,
396 .d_select = eno_select,
397 .d_mmap = eno_mmap,
398 .d_strategy = eno_strat,
399 .d_reserved_1 = eno_getc,
400 .d_reserved_2 = eno_putc,
401 };
402
403
404 #endif /* LOCK_STATS */
405 void lockprof_init(void);
406 void
lockprof_init(void)407 lockprof_init(void)
408 {
409 #if LOCK_STATS
410 int majorno = cdevsw_add(-1, &lockprof_cdevsw);
411
412 if (majorno < 0) {
413 panic("dtrace: failed to allocate a major number");
414 return;
415 }
416
417 if (dtrace_register(LP_NODE, &lockprof_attr, DTRACE_PRIV_KERNEL,
418 NULL, &lockprof_pops, NULL, &lockprof_id) != 0) {
419 panic("dtrace: failed to register lockprof provider");
420 }
421
422 dev_t dev = makedev(majorno, 0);
423
424 if (devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
425 LP_NODE ) == NULL) {
426 panic("dtrace: devfs_make_node failed for lockprof");
427 }
428
429 #endif /* LOCK_STATS */
430 }
431