1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/param.h>
27 #include <sys/systm.h>
28 #include <sys/errno.h>
29 #include <sys/stat.h>
30 #include <sys/ioctl.h>
31 #include <sys/conf.h>
32 #include <sys/fcntl.h>
33 #include <miscfs/devfs/devfs.h>
34
35 #if defined(__arm__) || defined(__arm64__)
36 #include <arm/caches_internal.h>
37 #endif /* defined(__arm__) || defined(__arm64__) */
38
39 #include <sys/dtrace.h>
40 #include <sys/dtrace_impl.h>
41
42 #include <sys/dtrace_glue.h>
43
44 #include <sys/sdt_impl.h>
45 extern int dtrace_kernel_symbol_mode;
46
47 #include <ptrauth.h>
48
49 /* #include <machine/trap.h */
50 struct savearea_t; /* Used anonymously */
51
52 #if defined(__arm__)
53 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
54 extern perfCallback tempDTraceTrapHook;
55 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
56 #define SDT_PATCHVAL 0xdefc
57 #define SDT_AFRAMES 7
58 #elif defined(__arm64__)
59 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
60 extern perfCallback tempDTraceTrapHook;
61 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
62 #define SDT_PATCHVAL 0xe7eeee7e
63 #define SDT_AFRAMES 7
64 #elif defined(__x86_64__)
65 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int);
66 extern perfCallback tempDTraceTrapHook;
67 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int);
68 #define SDT_PATCHVAL 0xf0
69 #define SDT_AFRAMES 6
70 #else
71 #error Unknown architecture
72 #endif
73
74 #define SDT_PROBETAB_SIZE 0x1000 /* 4k entries -- 16K total */
75
76 static int sdt_verbose = 0;
77 sdt_probe_t **sdt_probetab;
78 int sdt_probetab_size;
79 int sdt_probetab_mask;
80
81 /*ARGSUSED*/
82 static void
__sdt_provide_module(void * arg,struct modctl * ctl)83 __sdt_provide_module(void *arg, struct modctl *ctl)
84 {
85 #pragma unused(arg)
86 char *modname = ctl->mod_modname;
87 sdt_probedesc_t *sdpd;
88 sdt_probe_t *sdp, *old;
89 sdt_provider_t *prov;
90
91 /*
92 * One for all, and all for one: if we haven't yet registered all of
93 * our providers, we'll refuse to provide anything.
94 */
95 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
96 if (prov->sdtp_id == DTRACE_PROVNONE) {
97 return;
98 }
99 }
100
101 /* Nothing to do. Module is either invalid or we haven't found any SDT probe descriptions. */
102 if (!ctl || ctl->mod_sdtprobecnt != 0 || (sdpd = ctl->mod_sdtdesc) == NULL) {
103 return;
104 }
105
106 for (sdpd = ctl->mod_sdtdesc; sdpd != NULL; sdpd = sdpd->sdpd_next) {
107 dtrace_id_t id;
108
109 /* Validate probe's provider name. Do not provide probes for unknown providers. */
110 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
111 if (strcmp(prov->sdtp_prefix, sdpd->sdpd_prov) == 0) {
112 break;
113 }
114 }
115
116 if (prov->sdtp_name == NULL) {
117 printf("Ignoring probes from unsupported provider %s\n", sdpd->sdpd_prov);
118 continue;
119 }
120
121 if (sdpd->sdpd_func == NULL) {
122 /*
123 * Ignore probes for which we don't have any symbol. That's likely some problem with
124 * __sdt section processing.
125 */
126 printf("Ignoring probe %s (no symbol name)\n", sdpd->sdpd_name);
127 continue;
128 }
129
130 sdp = kmem_zalloc(sizeof(sdt_probe_t), KM_SLEEP);
131 sdp->sdp_loadcnt = ctl->mod_loadcnt;
132 sdp->sdp_ctl = ctl;
133 sdp->sdp_name = kmem_alloc(strlen(sdpd->sdpd_name) + 1, KM_SLEEP);
134 (void) strlcpy(sdp->sdp_name, sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
135 sdp->sdp_namelen = strlen(sdpd->sdpd_name) + 1;
136 sdp->sdp_provider = prov;
137
138 /*
139 * We have our provider. Now create the probe.
140 */
141 if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
142 sdpd->sdpd_func, sdp->sdp_name)) != DTRACE_IDNONE) {
143 old = dtrace_probe_arg(prov->sdtp_id, id);
144 ASSERT(old != NULL);
145
146 sdp->sdp_next = old->sdp_next;
147 sdp->sdp_id = id;
148 old->sdp_next = sdp;
149 } else {
150 sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
151 modname, sdpd->sdpd_func, sdp->sdp_name, SDT_AFRAMES, sdp);
152
153 ctl->mod_sdtprobecnt++;
154 }
155
156 #if 0
157 printf("__sdt_provide_module: sdpd=0x%p sdp=0x%p name=%s, id=%d\n", sdpd, sdp,
158 sdp->sdp_name, sdp->sdp_id);
159 #endif
160
161 sdp->sdp_hashnext =
162 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
163 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
164
165 sdp->sdp_patchval = SDT_PATCHVAL;
166 sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
167 sdp->sdp_savedval = *sdp->sdp_patchpoint;
168 }
169 }
170
171 /*ARGSUSED*/
172 static void
sdt_destroy(void * arg,dtrace_id_t id,void * parg)173 sdt_destroy(void *arg, dtrace_id_t id, void *parg)
174 {
175 #pragma unused(arg,id)
176 sdt_probe_t *sdp = parg, *old, *last, *hash;
177 int ndx;
178
179 struct modctl *ctl = sdp->sdp_ctl;
180
181 /*
182 * Decrement SDT probe counts only when a probe being destroyed belongs to the
183 * currently loaded version of a module and not the stale one.
184 */
185 if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt && ctl->mod_loaded) {
186 ctl->mod_sdtprobecnt--;
187 }
188
189 while (sdp != NULL) {
190 old = sdp;
191
192 /*
193 * Now we need to remove this probe from the sdt_probetab.
194 */
195 ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
196 last = NULL;
197 hash = sdt_probetab[ndx];
198
199 while (hash != sdp) {
200 ASSERT(hash != NULL);
201 last = hash;
202 hash = hash->sdp_hashnext;
203 }
204
205 if (last != NULL) {
206 last->sdp_hashnext = sdp->sdp_hashnext;
207 } else {
208 sdt_probetab[ndx] = sdp->sdp_hashnext;
209 }
210
211 kmem_free(sdp->sdp_name, sdp->sdp_namelen);
212 sdp = sdp->sdp_next;
213 kmem_free(old, sizeof(sdt_probe_t));
214 }
215 }
216
217 /*ARGSUSED*/
218 static int
sdt_enable(void * arg,dtrace_id_t id,void * parg)219 sdt_enable(void *arg, dtrace_id_t id, void *parg)
220 {
221 #pragma unused(arg,id)
222 sdt_probe_t *sdp = parg;
223 struct modctl *ctl = sdp->sdp_ctl;
224
225 ctl->mod_nenabled++;
226
227 /*
228 * If this module has disappeared since we discovered its probes,
229 * refuse to enable it.
230 */
231 if (!ctl->mod_loaded) {
232 if (sdt_verbose) {
233 cmn_err(CE_NOTE, "sdt is failing for probe %s "
234 "(module %s unloaded)",
235 sdp->sdp_name, ctl->mod_modname);
236 }
237 goto err;
238 }
239
240 /*
241 * Now check that our modctl has the expected load count. If it
242 * doesn't, this module must have been unloaded and reloaded -- and
243 * we're not going to touch it.
244 */
245 if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
246 if (sdt_verbose) {
247 cmn_err(CE_NOTE, "sdt is failing for probe %s "
248 "(module %s reloaded)",
249 sdp->sdp_name, ctl->mod_modname);
250 }
251 goto err;
252 }
253
254 dtrace_casptr(&tempDTraceTrapHook, NULL, ptrauth_nop_cast(void *, &fbt_perfCallback));
255 if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
256 if (sdt_verbose) {
257 cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
258 "in module %s: tempDTraceTrapHook already occupied.",
259 sdp->sdp_name, ctl->mod_modname);
260 }
261 return 0;
262 }
263
264 while (sdp != NULL) {
265 (void)ml_nofault_copy((vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint,
266 (vm_size_t)sizeof(sdp->sdp_patchval));
267
268 /*
269 * Make the patched instruction visible via a data + instruction
270 * cache fush on platforms that need it
271 */
272 flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0);
273 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0);
274
275 sdp = sdp->sdp_next;
276 }
277
278 err:
279 return 0;
280 }
281
282 /*ARGSUSED*/
283 static void
sdt_disable(void * arg,dtrace_id_t id,void * parg)284 sdt_disable(void *arg, dtrace_id_t id, void *parg)
285 {
286 #pragma unused(arg,id)
287 sdt_probe_t *sdp = parg;
288 struct modctl *ctl = sdp->sdp_ctl;
289
290 ctl->mod_nenabled--;
291
292 if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt) {
293 goto err;
294 }
295
296 while (sdp != NULL) {
297 (void)ml_nofault_copy((vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint,
298 (vm_size_t)sizeof(sdp->sdp_savedval));
299 /*
300 * Make the patched instruction visible via a data + instruction
301 * cache flush on platforms that need it
302 */
303 flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0);
304 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0);
305 sdp = sdp->sdp_next;
306 }
307
308 err:
309 ;
310 }
311
312 static dtrace_pops_t sdt_pops = {
313 .dtps_provide = NULL,
314 .dtps_provide_module = sdt_provide_module,
315 .dtps_enable = sdt_enable,
316 .dtps_disable = sdt_disable,
317 .dtps_suspend = NULL,
318 .dtps_resume = NULL,
319 .dtps_getargdesc = sdt_getargdesc,
320 .dtps_getargval = sdt_getarg,
321 .dtps_usermode = NULL,
322 .dtps_destroy = sdt_destroy,
323 };
324
325 /*ARGSUSED*/
326 static int
sdt_attach(dev_info_t * devi)327 sdt_attach(dev_info_t *devi)
328 {
329 sdt_provider_t *prov;
330
331 if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
332 0, DDI_PSEUDO, 0) == DDI_FAILURE) {
333 cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
334 ddi_remove_minor_node(devi, NULL);
335 return DDI_FAILURE;
336 }
337
338 if (sdt_probetab_size == 0) {
339 sdt_probetab_size = SDT_PROBETAB_SIZE;
340 }
341
342 sdt_probetab_mask = sdt_probetab_size - 1;
343 sdt_probetab =
344 kmem_zalloc(sdt_probetab_size * sizeof(sdt_probe_t *), KM_SLEEP);
345 dtrace_invop_add(sdt_invop);
346
347 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
348 if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
349 DTRACE_PRIV_KERNEL, NULL,
350 &sdt_pops, prov, &prov->sdtp_id) != 0) {
351 cmn_err(CE_WARN, "failed to register sdt provider %s",
352 prov->sdtp_name);
353 }
354 }
355
356 return DDI_SUCCESS;
357 }
358
359 /*
360 * APPLE NOTE: sdt_detach not implemented
361 */
362 #if !defined(__APPLE__)
363 /*ARGSUSED*/
364 static int
sdt_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)365 sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
366 {
367 sdt_provider_t *prov;
368
369 switch (cmd) {
370 case DDI_DETACH:
371 break;
372
373 case DDI_SUSPEND:
374 return DDI_SUCCESS;
375
376 default:
377 return DDI_FAILURE;
378 }
379
380 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
381 if (prov->sdtp_id != DTRACE_PROVNONE) {
382 if (dtrace_unregister(prov->sdtp_id) != 0) {
383 return DDI_FAILURE;
384 }
385
386 prov->sdtp_id = DTRACE_PROVNONE;
387 }
388 }
389
390 dtrace_invop_remove(sdt_invop);
391 kmem_free(sdt_probetab, sdt_probetab_size * sizeof(sdt_probe_t *));
392
393 return DDI_SUCCESS;
394 }
395 #endif /* __APPLE__ */
396
397 d_open_t _sdt_open;
398
399 int
_sdt_open(dev_t dev,int flags,int devtype,struct proc * p)400 _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
401 {
402 #pragma unused(dev,flags,devtype,p)
403 return 0;
404 }
405
406 #define SDT_MAJOR -24 /* let the kernel pick the device number */
407
408 static const struct cdevsw sdt_cdevsw =
409 {
410 .d_open = _sdt_open,
411 .d_close = eno_opcl,
412 .d_read = eno_rdwrt,
413 .d_write = eno_rdwrt,
414 .d_ioctl = eno_ioctl,
415 .d_stop = (stop_fcn_t *)nulldev,
416 .d_reset = (reset_fcn_t *)nulldev,
417 .d_select = eno_select,
418 .d_mmap = eno_mmap,
419 .d_strategy = eno_strat,
420 .d_reserved_1 = eno_getc,
421 .d_reserved_2 = eno_putc,
422 };
423
424
425 #include <mach-o/nlist.h>
426 #include <libkern/kernel_mach_header.h>
427
428 /*
429 * Represents single record in __DATA,__sdt section.
430 */
431 typedef struct dtrace_sdt_def {
432 uintptr_t dsd_addr; /* probe site location */
433 const char *dsd_prov; /* provider's name */
434 const char *dsd_name; /* probe's name */
435 } __attribute__((__packed__)) dtrace_sdt_def_t;
436
437 /*
438 * Creates a copy of name and unescapes '-' characters.
439 */
440 static char *
sdt_strdup_name(const char * name)441 sdt_strdup_name(const char *name)
442 {
443 size_t len = strlen(name) + 1;
444 size_t i, j;
445 char *nname = kmem_alloc(len, KM_SLEEP);
446
447 for (i = 0, j = 0; name[j] != '\0'; i++) {
448 if (name[j] == '_' && name[j + 1] == '_') {
449 nname[i] = '-';
450 j += 2;
451 } else {
452 nname[i] = name[j++];
453 }
454 }
455
456 nname[i] = '\0';
457 return nname;
458 }
459
460 /*
461 * Returns Mach-O header that should be used for given modctl.
462 */
463 static kernel_mach_header_t *
sdt_get_module_mh(struct modctl * ctl)464 sdt_get_module_mh(struct modctl *ctl)
465 {
466 kernel_mach_header_t *mh = (kernel_mach_header_t *)ctl->mod_address;
467
468 /* Static KEXTs have their __sdt section merged into kernel's __sdt. */
469 if (MOD_IS_STATIC_KEXT(ctl)) {
470 mh = &_mh_execute_header;
471 }
472
473 if (mh->magic != MH_MAGIC_KERNEL) {
474 return NULL;
475 }
476
477 return mh;
478 }
479
480 /*
481 * Finds symbol table for given kernel module.
482 */
483 static uint32_t
sdt_find_symbol_table(struct modctl * ctl,kernel_nlist_t ** sym,char ** strings)484 sdt_find_symbol_table(struct modctl *ctl, kernel_nlist_t **sym, char **strings)
485 {
486 kernel_mach_header_t *mh = sdt_get_module_mh(ctl);
487 struct load_command *cmd = (struct load_command *)&mh[1];
488 kernel_segment_command_t *orig_le = NULL;
489 struct symtab_command *orig_st = NULL;
490
491 for (int i = 0; i < mh->ncmds; i++) {
492 if (cmd->cmd == LC_SEGMENT_KERNEL) {
493 kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
494
495 if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) {
496 orig_le = orig_sg;
497 }
498 } else if (cmd->cmd == LC_SYMTAB) {
499 orig_st = (struct symtab_command *) cmd;
500 }
501
502 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
503 }
504
505 if ((orig_st == NULL) || (orig_le == NULL)) {
506 return 0;
507 }
508
509 *sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
510 *strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
511
512 return orig_st->nsyms;
513 }
514
515 /* Last kernel address. */
516 static SECURITY_READ_ONLY_LATE(vm_address_t) kern_end = (vm_address_t)-1;
517
518 void
sdt_early_init(void)519 sdt_early_init(void)
520 {
521 kernel_mach_header_t *mh = &_mh_execute_header;
522 kernel_section_t *sec_ks = NULL;
523 kc_format_t kc_format;
524
525 if (!PE_get_primary_kc_format(&kc_format)) {
526 kc_format = KCFormatUnknown;
527 }
528
529 /*
530 * Detects end of kernel's text in static kernel cache. It is the last text address before
531 * the first kext text section start.
532 */
533 if (kc_format == KCFormatStatic) {
534 if ((sec_ks = getsectbynamefromheader(mh, "__PRELINK_INFO", "__kmod_start")) == NULL) {
535 printf("SDT: unable to find prelink info\n");
536 return;
537 }
538
539 /* find the MIN(start_address) of all kexts in this image. */
540 const uint64_t *start_addr = (const uint64_t *)sec_ks->addr;
541 for (int i = 0; i < sec_ks->size / sizeof(uint64_t); i++) {
542 if (kern_end > start_addr[i]) {
543 kern_end = start_addr[i];
544 }
545 }
546 }
547 }
548
549 /*
550 * Finds TEXT range that belongs to given module.
551 */
552 static int
sdt_find_module_text_range(struct modctl * ctl,vm_address_t * start,vm_address_t * end)553 sdt_find_module_text_range(struct modctl *ctl, vm_address_t *start, vm_address_t *end)
554 {
555 kc_format_t kc_format;
556
557 if (!PE_get_primary_kc_format(&kc_format)) {
558 kc_format = KCFormatUnknown;
559 }
560
561 /* Adjust kernel region for static kernel cache. */
562 *start = ctl->mod_address;
563
564 if (MOD_IS_MACH_KERNEL(ctl) && kc_format == KCFormatStatic) {
565 *end = kern_end;
566 } else {
567 *end = ctl->mod_address + ctl->mod_size;
568 }
569
570 return 1;
571 }
572
573 /*
574 * Processes SDT section in given Mach-O header
575 */
576 void
sdt_load_machsect(struct modctl * ctl)577 sdt_load_machsect(struct modctl *ctl)
578 {
579 kernel_mach_header_t *mh = sdt_get_module_mh(ctl);
580 kernel_section_t *sec_sdt = NULL;
581 char *strings = NULL;
582 kernel_nlist_t *sym = NULL;
583 vm_address_t text_start, text_end;
584 unsigned int len;
585 uint32_t nsyms = 0;
586
587 if (mh == NULL) {
588 return;
589 }
590
591 /* Ignore SDT definitions if we don't know where they belong. */
592 if (!sdt_find_module_text_range(ctl, &text_start, &text_end)) {
593 printf("SDT: Unable to determine text range for %s\n", ctl->mod_modname);
594 return;
595 }
596
597 /* Do not load SDTs when asked to use kernel symbols but symbol table is not available. */
598 if (MOD_HAS_KERNEL_SYMBOLS(ctl) && (nsyms = sdt_find_symbol_table(ctl, &sym, &strings)) == 0) {
599 printf("SDT: No kernel symbols for %s\n", ctl->mod_modname);
600 return;
601 }
602
603 /* Locate DTrace SDT section in the object. */
604 if ((sec_sdt = getsectbynamefromheader(mh, "__DATA", "__sdt")) == NULL) {
605 return;
606 }
607
608 /*
609 * Iterate over SDT section and establish all SDT probe descriptions.
610 */
611 dtrace_sdt_def_t *sdtdef = (dtrace_sdt_def_t *)(sec_sdt->addr);
612 for (size_t k = 0; k < sec_sdt->size / sizeof(dtrace_sdt_def_t); k++, sdtdef++) {
613 const char *funcname;
614 unsigned long best = 0;
615
616 #if defined(__arm__)
617 /* PR8353094 - mask off thumb-bit */
618 sdtdef->dsd_addr &= ~0x1U;
619 #elif defined(__arm64__)
620 sdtdef->dsd_addr &= ~0x1LU;
621 #endif /* __arm__ */
622
623 /*
624 * Static KEXTs share __sdt section with kernel after linking. It is required
625 * to filter out description and pick only those that belong to requested
626 * module or kernel itself.
627 */
628 if (MOD_IS_STATIC_KEXT(ctl) || MOD_IS_MACH_KERNEL(ctl)) {
629 if ((sdtdef->dsd_addr < text_start) || (sdtdef->dsd_addr > text_end)) {
630 continue;
631 }
632 } else {
633 /* Skip over probe descripton that do not belong to current module. */
634 if (!dtrace_addr_in_module((void *)sdtdef->dsd_addr, ctl)) {
635 continue;
636 }
637 }
638
639 sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
640
641 /* Unescape probe name and keep a note of the size of original memory allocation. */
642 sdpd->sdpd_name = sdt_strdup_name(sdtdef->dsd_name);
643 sdpd->sdpd_namelen = strlen(sdtdef->dsd_name) + 1;
644
645 /* Used only for provider structure lookup so there is no need to make dynamic copy. */
646 sdpd->sdpd_prov = sdtdef->dsd_prov;
647
648 /*
649 * Find the symbol immediately preceding the sdt probe site just discovered,
650 * that symbol names the function containing the sdt probe.
651 */
652 sdpd->sdpd_func = NULL;
653
654 if (MOD_HAS_KERNEL_SYMBOLS(ctl)) {
655 funcname = NULL;
656 for (int i = 0; i < nsyms; i++) {
657 uint8_t jn_type = sym[i].n_type & N_TYPE;
658 char *jname = strings + sym[i].n_un.n_strx;
659
660 if ((N_SECT != jn_type && N_ABS != jn_type)) {
661 continue;
662 }
663
664 if (0 == sym[i].n_un.n_strx) { /* iff a null, "", name. */
665 continue;
666 }
667
668 if (*jname == '_') {
669 jname += 1;
670 }
671
672 if (sdtdef->dsd_addr <= (unsigned long)sym[i].n_value) {
673 continue;
674 }
675
676 if ((unsigned long)sym[i].n_value > best) {
677 best = (unsigned long)sym[i].n_value;
678 funcname = jname;
679 }
680 }
681
682 if (funcname) {
683 len = strlen(funcname) + 1;
684 sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP);
685 (void) strlcpy(sdpd->sdpd_func, funcname, len);
686 }
687 }
688
689 sdpd->sdpd_offset = sdtdef->dsd_addr;
690
691 sdpd->sdpd_next = (sdt_probedesc_t *)ctl->mod_sdtdesc;
692 ctl->mod_sdtdesc = sdpd;
693 }
694 }
695
696 void
sdt_init(void)697 sdt_init( void )
698 {
699 int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
700
701 if (majdevno < 0) {
702 printf("sdt_init: failed to allocate a major number!\n");
703 return;
704 }
705
706 if (dtrace_sdt_probes_restricted()) {
707 return;
708 }
709
710 sdt_attach((dev_info_t*)(uintptr_t)majdevno);
711 }
712
713 #undef SDT_MAJOR
714
715 /*
716 * Provide SDT modules with userspace symbols.
717 *
718 * A module contains only partially filled in SDT probe descriptions because symbols were
719 * not available at the time when __sdt section was loaded. Fixup descriptons before providing
720 * the probes.
721 */
722 static void
sdt_provide_module_user_syms(void * arg,struct modctl * ctl)723 sdt_provide_module_user_syms(void *arg, struct modctl *ctl)
724 {
725 sdt_probedesc_t *sdpd;
726 dtrace_module_symbols_t *mod_sym = ctl->mod_user_symbols;
727
728 if (mod_sym == NULL) {
729 printf("DTrace missing userspace symbols for module %s\n", ctl->mod_modname);
730 return;
731 }
732
733 /* Fixup missing probe description parts. */
734 for (sdpd = ctl->mod_sdtdesc; sdpd != NULL; sdpd = sdpd->sdpd_next) {
735 ASSERT(sdpd->sdpd_func == NULL);
736
737 /* Look for symbol that contains SDT probe offset. */
738 for (int i = 0; i < mod_sym->dtmodsyms_count; i++) {
739 dtrace_symbol_t *symbol = &mod_sym->dtmodsyms_symbols[i];
740 char *name = symbol->dtsym_name;
741
742 /*
743 * Every function symbol gets extra '_' prepended in the Mach-O symbol table.
744 * Strip it away to make a probe's function name match source code.
745 */
746 if (*name == '_') {
747 name += 1;
748 }
749
750 if (!symbol->dtsym_addr) {
751 continue;
752 }
753
754 /* Ignore symbols that do not belong to this module. */
755 if (!dtrace_addr_in_module((void *)symbol->dtsym_addr, ctl)) {
756 continue;
757 }
758
759 /* Pick symbol name when we found match. */
760 if ((symbol->dtsym_addr <= sdpd->sdpd_offset) &&
761 (sdpd->sdpd_offset < symbol->dtsym_addr + symbol->dtsym_size)) {
762 size_t len = strlen(name) + 1;
763 sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP);
764 (void) strlcpy(sdpd->sdpd_func, name, len);
765 break;
766 }
767 }
768 }
769
770 /* Probe descriptionds are now fixed up. Provide them as usual. */
771 __sdt_provide_module(arg, ctl);
772 }
773
774 /*ARGSUSED*/
775 void
sdt_provide_module(void * arg,struct modctl * ctl)776 sdt_provide_module(void *arg, struct modctl *ctl)
777 {
778 ASSERT(ctl != NULL);
779 ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);
780 LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED);
781
782 if (MOD_SDT_DONE(ctl)) {
783 return;
784 }
785
786 if (MOD_HAS_KERNEL_SYMBOLS(ctl)) {
787 __sdt_provide_module(arg, ctl);
788 ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
789 return;
790 }
791
792 if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) {
793 sdt_provide_module_user_syms(arg, ctl);
794 ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
795 return;
796 }
797
798 /*
799 * The SDT provider's module is not detachable so we don't have to re-provide SDT
800 * probes if that happens. After succesfull providing, the probe descriptions are
801 * no longer required. If module gets re-loaded it will get a new set of probe
802 * descriptions from its __sdt section.
803 */
804 if (MOD_SDT_PROBES_PROVIDED(ctl)) {
805 sdt_probedesc_t *sdpd = ctl->mod_sdtdesc;
806 while (sdpd) {
807 sdt_probedesc_t *this_sdpd = sdpd;
808 kmem_free((void *)sdpd->sdpd_name, sdpd->sdpd_namelen);
809 if (sdpd->sdpd_func) {
810 kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
811 }
812 sdpd = sdpd->sdpd_next;
813 kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
814 }
815 ctl->mod_sdtdesc = NULL;
816 }
817 }
818