1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <ptrauth.h>
27
28 #include <kern/thread.h>
29 #include <mach/thread_status.h>
30
31 /* XXX All of these should really be derived from syscall_sw.h */
32 #if defined (__x86_64__)
33 #define SYSCALL_CLASS_SHIFT 24
34 #define SYSCALL_CLASS_MASK (0xFF << SYSCALL_CLASS_SHIFT)
35 #define SYSCALL_NUMBER_MASK (~SYSCALL_CLASS_MASK)
36 #define I386_SYSCALL_NUMBER_MASK (0xFFFF)
37 #endif
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/proc.h>
42 #include <sys/errno.h>
43 #include <sys/ioctl.h>
44 #include <sys/conf.h>
45 #include <sys/fcntl.h>
46 #include <sys/syscall.h>
47 #include <miscfs/devfs/devfs.h>
48
49 #include <sys/dtrace.h>
50 #include <sys/dtrace_impl.h>
51 #include <sys/systrace_args.h>
52 #include "systrace.h"
53 #include <sys/stat.h>
54 #include <sys/systm.h>
55 #include <sys/conf.h>
56 #include <sys/user.h>
57
58 #include <machine/pal_routines.h>
59
60 #if defined (__x86_64__)
61 #define SYSTRACE_ARTIFICIAL_FRAMES 2
62 #define MACHTRACE_ARTIFICIAL_FRAMES 3
63 #elif defined(__arm__) || defined(__arm64__)
64 #define SYSTRACE_ARTIFICIAL_FRAMES 2
65 #define MACHTRACE_ARTIFICIAL_FRAMES 3
66 #else
67 #error Unknown Architecture
68 #endif
69
70 #define SYSTRACE_NARGS (int)(sizeof(((uthread_t)NULL)->uu_arg) / sizeof(((uthread_t)NULL)->uu_arg[0]))
71
72 #include <sys/sysent.h>
73 #define sy_callc sy_call /* Map Solaris slot name to Darwin's */
74 #define NSYSCALL nsysent /* and is less than 500 or so */
75
76 extern const char *syscallnames[];
77
78 #include <sys/dtrace_glue.h>
79 #define casptr dtrace_casptr
80 #define membar_enter dtrace_membar_producer
81
82 #define LOADABLE_SYSCALL(a) 0 /* Not pertinent to Darwin. */
83 #define LOADED_SYSCALL(a) 1 /* Not pertinent to Darwin. */
84
85 static LCK_MTX_DECLARE_ATTR(dtrace_systrace_lock,
86 &dtrace_lck_grp, &dtrace_lck_attr); /* probe state lock */
87
88 systrace_sysent_t *systrace_sysent = NULL;
89 void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t);
90
91 static uint64_t systrace_getargval(void *, dtrace_id_t, void *, int, int);
92 static void systrace_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
93
94 void
systrace_stub(dtrace_id_t id,uint64_t arg0,uint64_t arg1,uint64_t arg2,uint64_t arg3,uint64_t arg4)95 systrace_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1,
96 uint64_t arg2, uint64_t arg3, uint64_t arg4)
97 {
98 #pragma unused(id,arg0,arg1,arg2,arg3,arg4)
99 }
100
101 int32_t
dtrace_systrace_syscall(struct proc * pp,void * uap,int * rv)102 dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv)
103 {
104 unsigned short code; /* The system call number */
105
106 systrace_sysent_t *sy;
107 dtrace_id_t id;
108 int32_t rval;
109 syscall_arg_t *ip = (syscall_arg_t *)uap;
110 uint64_t uargs[SYSTRACE_NARGS] = {0};
111
112 #if defined (__x86_64__)
113 {
114 pal_register_cache_state(current_thread(), VALID);
115 x86_saved_state_t *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread());
116
117 if (is_saved_state64(tagged_regs)) {
118 x86_saved_state64_t *regs = saved_state64(tagged_regs);
119 code = regs->rax & SYSCALL_NUMBER_MASK;
120 /*
121 * Check for indirect system call... system call number
122 * passed as 'arg0'
123 */
124 if (code == 0) {
125 code = regs->rdi;
126 }
127 } else {
128 code = saved_state32(tagged_regs)->eax & I386_SYSCALL_NUMBER_MASK;
129
130 if (code == 0) {
131 vm_offset_t params = (vm_offset_t) (saved_state32(tagged_regs)->uesp + sizeof(int));
132 code = fuword(params);
133 }
134 }
135 }
136 #elif defined(__arm__)
137 {
138 /*
139 * On arm, syscall numbers depend on a flavor (indirect or not)
140 * and can be in either r0 or r12 (always u32)
141 */
142
143 /* See bsd/dev/arm/systemcalls.c:arm_get_syscall_number */
144 arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread());
145
146 /* Check for indirect system call */
147 if (arm_regs->r[12] != 0) {
148 code = arm_regs->r[12];
149 } else {
150 code = arm_regs->r[0];
151 }
152 }
153 #elif defined(__arm64__)
154 {
155 /*
156 * On arm64, syscall numbers depend on a flavor (indirect or not)
157 * ... and for u32 can be in either r0 or r12
158 * ... and for u64 can be in either x0 or x16
159 */
160
161 /* see bsd/dev/arm/systemcalls.c:arm_get_syscall_number */
162 arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread());
163
164 if (is_saved_state32(arm_regs)) {
165 /* Check for indirect system call */
166 if (saved_state32(arm_regs)->r[12] != 0) {
167 code = saved_state32(arm_regs)->r[12];
168 } else {
169 code = saved_state32(arm_regs)->r[0];
170 }
171 } else {
172 /* Check for indirect system call */
173 if (saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM] != 0) {
174 code = saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM];
175 } else {
176 code = saved_state64(arm_regs)->x[0];
177 }
178 }
179 }
180 #else
181 #error Unknown Architecture
182 #endif
183
184 // Bounds "check" the value of code a la unix_syscall
185 sy = (code >= nsysent) ? &systrace_sysent[SYS_invalid] : &systrace_sysent[code];
186
187 systrace_args(code, ip, uargs);
188
189 if ((id = sy->stsy_entry) != DTRACE_IDNONE) {
190 uthread_t uthread = current_uthread();
191 if (uthread) {
192 uthread->t_dtrace_syscall_args = uargs;
193 }
194
195 static_assert(SYSTRACE_NARGS >= 5, "not enough system call arguments");
196 (*systrace_probe)(id, uargs[0], uargs[1], uargs[2], uargs[3], uargs[4]);
197
198 if (uthread) {
199 uthread->t_dtrace_syscall_args = NULL;
200 }
201 }
202
203
204
205 #if 0 /* XXX */
206 /*
207 * APPLE NOTE: Not implemented.
208 * We want to explicitly allow DTrace consumers to stop a process
209 * before it actually executes the meat of the syscall.
210 */
211 p = ttoproc(curthread);
212 mutex_enter(&p->p_lock);
213 if (curthread->t_dtrace_stop && !curthread->t_lwp->lwp_nostop) {
214 curthread->t_dtrace_stop = 0;
215 stop(PR_REQUESTED, 0);
216 }
217 mutex_exit(&p->p_lock);
218 #endif
219
220 rval = (*sy->stsy_underlying)(pp, uap, rv);
221
222 if ((id = sy->stsy_return) != DTRACE_IDNONE) {
223 uint64_t munged_rv0, munged_rv1;
224 uthread_t uthread = current_uthread();
225
226 if (uthread) {
227 uthread->t_dtrace_errno = rval; /* Establish t_dtrace_errno now in case this enabling refers to it. */
228 }
229 /*
230 * "Decode" rv for use in the call to dtrace_probe()
231 */
232 if (rval == ERESTART) {
233 munged_rv0 = -1LL; /* System call will be reissued in user mode. Make DTrace report a -1 return. */
234 munged_rv1 = -1LL;
235 } else if (rval != EJUSTRETURN) {
236 if (rval) {
237 munged_rv0 = -1LL; /* Mimic what libc will do. */
238 munged_rv1 = -1LL;
239 } else {
240 switch (sy->stsy_return_type) {
241 case _SYSCALL_RET_INT_T:
242 munged_rv0 = rv[0];
243 munged_rv1 = rv[1];
244 break;
245 case _SYSCALL_RET_UINT_T:
246 munged_rv0 = ((u_int)rv[0]);
247 munged_rv1 = ((u_int)rv[1]);
248 break;
249 case _SYSCALL_RET_OFF_T:
250 case _SYSCALL_RET_UINT64_T:
251 munged_rv0 = *(u_int64_t *)rv;
252 munged_rv1 = 0LL;
253 break;
254 case _SYSCALL_RET_ADDR_T:
255 case _SYSCALL_RET_SIZE_T:
256 case _SYSCALL_RET_SSIZE_T:
257 munged_rv0 = *(user_addr_t *)rv;
258 munged_rv1 = 0LL;
259 break;
260 case _SYSCALL_RET_NONE:
261 munged_rv0 = 0LL;
262 munged_rv1 = 0LL;
263 break;
264 default:
265 munged_rv0 = 0LL;
266 munged_rv1 = 0LL;
267 break;
268 }
269 }
270 } else {
271 munged_rv0 = 0LL;
272 munged_rv1 = 0LL;
273 }
274
275 /*
276 * <http://mail.opensolaris.org/pipermail/dtrace-discuss/2007-January/003276.html> says:
277 *
278 * "This is a bit of an historical artifact. At first, the syscall provider just
279 * had its return value in arg0, and the fbt and pid providers had their return
280 * values in arg1 (so that we could use arg0 for the offset of the return site).
281 *
282 * We inevitably started writing scripts where we wanted to see the return
283 * values from probes in all three providers, and we made this script easier
284 * to write by replicating the syscall return values in arg1 to match fbt and
285 * pid. We debated briefly about removing the return value from arg0, but
286 * decided that it would be less confusing to have the same data in two places
287 * than to have some non-helpful, non-intuitive value in arg0.
288 *
289 * This change was made 4/23/2003 according to the DTrace project's putback log."
290 */
291 (*systrace_probe)(id, munged_rv0, munged_rv0, munged_rv1, (uint64_t)rval, 0);
292 }
293
294 return rval;
295 }
296
297 void
dtrace_systrace_syscall_return(unsigned short code,int rval,int * rv)298 dtrace_systrace_syscall_return(unsigned short code, int rval, int *rv)
299 {
300 systrace_sysent_t *sy;
301 dtrace_id_t id;
302
303 // Bounds "check" the value of code a la unix_syscall_return
304 sy = (code >= nsysent) ? &systrace_sysent[SYS_invalid] : &systrace_sysent[code];
305
306 if ((id = sy->stsy_return) != DTRACE_IDNONE) {
307 uint64_t munged_rv0, munged_rv1;
308 uthread_t uthread = current_uthread();
309
310 if (uthread) {
311 uthread->t_dtrace_errno = rval; /* Establish t_dtrace_errno now in case this enabling refers to it. */
312 }
313 /*
314 * "Decode" rv for use in the call to dtrace_probe()
315 */
316 if (rval == ERESTART) {
317 munged_rv0 = -1LL; /* System call will be reissued in user mode. Make DTrace report a -1 return. */
318 munged_rv1 = -1LL;
319 } else if (rval != EJUSTRETURN) {
320 if (rval) {
321 munged_rv0 = -1LL; /* Mimic what libc will do. */
322 munged_rv1 = -1LL;
323 } else {
324 switch (sy->stsy_return_type) {
325 case _SYSCALL_RET_INT_T:
326 munged_rv0 = rv[0];
327 munged_rv1 = rv[1];
328 break;
329 case _SYSCALL_RET_UINT_T:
330 munged_rv0 = ((u_int)rv[0]);
331 munged_rv1 = ((u_int)rv[1]);
332 break;
333 case _SYSCALL_RET_OFF_T:
334 case _SYSCALL_RET_UINT64_T:
335 munged_rv0 = *(u_int64_t *)rv;
336 munged_rv1 = 0LL;
337 break;
338 case _SYSCALL_RET_ADDR_T:
339 case _SYSCALL_RET_SIZE_T:
340 case _SYSCALL_RET_SSIZE_T:
341 munged_rv0 = *(user_addr_t *)rv;
342 munged_rv1 = 0LL;
343 break;
344 case _SYSCALL_RET_NONE:
345 munged_rv0 = 0LL;
346 munged_rv1 = 0LL;
347 break;
348 default:
349 munged_rv0 = 0LL;
350 munged_rv1 = 0LL;
351 break;
352 }
353 }
354 } else {
355 munged_rv0 = 0LL;
356 munged_rv1 = 0LL;
357 }
358
359 (*systrace_probe)(id, munged_rv0, munged_rv0, munged_rv1, (uint64_t)rval, 0);
360 }
361 }
362
363 #define SYSTRACE_SHIFT 16
364 #define SYSTRACE_ISENTRY(x) ((int)(x) >> SYSTRACE_SHIFT)
365 #define SYSTRACE_SYSNUM(x) ((int)(x) & ((1 << SYSTRACE_SHIFT) - 1))
366 #define SYSTRACE_ENTRY(id) ((1 << SYSTRACE_SHIFT) | (id))
367 #define SYSTRACE_RETURN(id) (id)
368
369 #if ((1 << SYSTRACE_SHIFT) <= NSYSCALL)
370 #error 1 << SYSTRACE_SHIFT must exceed number of system calls
371 #endif
372
373 static dtrace_provider_id_t systrace_id;
374
375 /*
376 * APPLE NOTE: Avoid name clash with Darwin automagic conf symbol.
377 * See balanced undef below.
378 */
379 #define systrace_init _systrace_init
380
381 static void
systrace_init(const struct sysent * actual,systrace_sysent_t ** interposed)382 systrace_init(const struct sysent *actual, systrace_sysent_t **interposed)
383 {
384 systrace_sysent_t *ssysent = *interposed; /* Avoid sysent shadow warning
385 * from bsd/sys/sysent.h */
386 unsigned int i;
387
388 if (ssysent == NULL) {
389 *interposed = ssysent = kmem_zalloc(sizeof(systrace_sysent_t) *
390 NSYSCALL, KM_SLEEP);
391 }
392
393 for (i = 0; i < NSYSCALL; i++) {
394 /* Use of volatile protects the if statement below from being optimized away */
395 const volatile struct sysent *a = &actual[i];
396 systrace_sysent_t *s = &ssysent[i];
397
398 if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) {
399 continue;
400 }
401
402 if (a->sy_callc == dtrace_systrace_syscall) {
403 continue;
404 }
405
406 s->stsy_underlying = a->sy_callc;
407 s->stsy_return_type = a->sy_return_type;
408 }
409 }
410
411
412 /*ARGSUSED*/
413 static void
systrace_provide(void * arg,const dtrace_probedesc_t * desc)414 systrace_provide(void *arg, const dtrace_probedesc_t *desc)
415 {
416 #pragma unused(arg) /* __APPLE__ */
417 unsigned int i;
418
419 if (desc != NULL) {
420 return;
421 }
422
423 systrace_init(sysent, &systrace_sysent);
424
425 for (i = 0; i < NSYSCALL; i++) {
426 if (systrace_sysent[i].stsy_underlying == NULL) {
427 continue;
428 }
429
430 if (dtrace_probe_lookup(systrace_id, NULL,
431 syscallnames[i], "entry") != 0) {
432 continue;
433 }
434
435 (void) dtrace_probe_create(systrace_id, NULL, syscallnames[i],
436 "entry", SYSTRACE_ARTIFICIAL_FRAMES,
437 (void *)((uintptr_t)SYSTRACE_ENTRY(i)));
438 (void) dtrace_probe_create(systrace_id, NULL, syscallnames[i],
439 "return", SYSTRACE_ARTIFICIAL_FRAMES,
440 (void *)((uintptr_t)SYSTRACE_RETURN(i)));
441
442 systrace_sysent[i].stsy_entry = DTRACE_IDNONE;
443 systrace_sysent[i].stsy_return = DTRACE_IDNONE;
444 }
445 }
446 #undef systrace_init
447
448 /*ARGSUSED*/
449 static void
systrace_destroy(void * arg,dtrace_id_t id,void * parg)450 systrace_destroy(void *arg, dtrace_id_t id, void *parg)
451 {
452 #pragma unused(arg,id) /* __APPLE__ */
453
454 int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
455
456 #pragma unused(sysnum) /* __APPLE__ */
457 /*
458 * There's nothing to do here but assert that we have actually been
459 * disabled.
460 */
461 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
462 ASSERT(systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE);
463 } else {
464 ASSERT(systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
465 }
466 }
467
468 /*ARGSUSED*/
469 static int
systrace_enable(void * arg,dtrace_id_t id,void * parg)470 systrace_enable(void *arg, dtrace_id_t id, void *parg)
471 {
472 #pragma unused(arg) /* __APPLE__ */
473
474 int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
475 int enabled = (systrace_sysent[sysnum].stsy_entry != DTRACE_IDNONE ||
476 systrace_sysent[sysnum].stsy_return != DTRACE_IDNONE);
477
478 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
479 systrace_sysent[sysnum].stsy_entry = id;
480 } else {
481 systrace_sysent[sysnum].stsy_return = id;
482 }
483
484 if (enabled) {
485 ASSERT(sysent[sysnum].sy_callc == dtrace_systrace_syscall);
486 return 0;
487 }
488
489 lck_mtx_lock(&dtrace_systrace_lock);
490 if (sysent[sysnum].sy_callc == systrace_sysent[sysnum].stsy_underlying) {
491 /* It is not possible to write to sysent[] directly because it is const. */
492 vm_offset_t dss = ptrauth_nop_cast(vm_offset_t, &dtrace_systrace_syscall);
493 ml_nofault_copy((vm_offset_t)&dss, (vm_offset_t)&sysent[sysnum].sy_callc, sizeof(vm_offset_t));
494 }
495 lck_mtx_unlock(&dtrace_systrace_lock);
496
497 return 0;
498 }
499
500 /*ARGSUSED*/
501 static void
systrace_disable(void * arg,dtrace_id_t id,void * parg)502 systrace_disable(void *arg, dtrace_id_t id, void *parg)
503 {
504 #pragma unused(arg,id) /* __APPLE__ */
505
506 int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
507 int disable = (systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE ||
508 systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
509
510 if (disable) {
511 /*
512 * Usage of volatile protects the if statement below from being optimized away.
513 *
514 * Compilers are clever and know that const array values can't change in time
515 * and the if below is always false. That is because it can't see that DTrace
516 * injects dtrace_systrace_syscall dynamically and violates constness of the
517 * array.
518 */
519 volatile const struct sysent *syscallent = &sysent[sysnum];
520
521 lck_mtx_lock(&dtrace_systrace_lock);
522 if (syscallent->sy_callc == dtrace_systrace_syscall) {
523 ml_nofault_copy((vm_offset_t)&systrace_sysent[sysnum].stsy_underlying,
524 (vm_offset_t)&syscallent->sy_callc, sizeof(vm_offset_t));
525 }
526 lck_mtx_unlock(&dtrace_systrace_lock);
527 }
528
529 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
530 systrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE;
531 } else {
532 systrace_sysent[sysnum].stsy_return = DTRACE_IDNONE;
533 }
534 }
535
536 static dtrace_pattr_t systrace_attr = {
537 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
538 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
539 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
540 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
541 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
542 };
543
544 static dtrace_pops_t systrace_pops = {
545 .dtps_provide = systrace_provide,
546 .dtps_provide_module = NULL,
547 .dtps_enable = systrace_enable,
548 .dtps_disable = systrace_disable,
549 .dtps_suspend = NULL,
550 .dtps_resume = NULL,
551 .dtps_getargdesc = systrace_getargdesc,
552 .dtps_getargval = systrace_getargval,
553 .dtps_usermode = NULL,
554 .dtps_destroy = systrace_destroy
555 };
556
557 static int
systrace_attach(dev_info_t * devi)558 systrace_attach(dev_info_t *devi)
559 {
560 systrace_probe = (void*)&dtrace_probe;
561 membar_enter();
562
563 if (ddi_create_minor_node(devi, "systrace", S_IFCHR, 0,
564 DDI_PSEUDO, 0) == DDI_FAILURE ||
565 dtrace_register("syscall", &systrace_attr, DTRACE_PRIV_USER, NULL,
566 &systrace_pops, NULL, &systrace_id) != 0) {
567 systrace_probe = systrace_stub;
568 ddi_remove_minor_node(devi, NULL);
569 return DDI_FAILURE;
570 }
571
572 return DDI_SUCCESS;
573 }
574
575
576 /*
577 * APPLE NOTE: systrace_detach not implemented
578 */
579 #if !defined(__APPLE__)
580 static int
systrace_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)581 systrace_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
582 {
583 switch (cmd) {
584 case DDI_DETACH:
585 break;
586 case DDI_SUSPEND:
587 return DDI_SUCCESS;
588 default:
589 return DDI_FAILURE;
590 }
591
592 if (dtrace_unregister(systrace_id) != 0) {
593 return DDI_FAILURE;
594 }
595
596 ddi_remove_minor_node(devi, NULL);
597 systrace_probe = systrace_stub;
598 return DDI_SUCCESS;
599 }
600 #endif /* __APPLE__ */
601
602
603 typedef kern_return_t (*mach_call_t)(void *);
604
605 /* APPLE NOTE: From #include <kern/syscall_sw.h> which may be changed for 64 bit! */
606 typedef void mach_munge_t(void *);
607
608 typedef struct {
609 int mach_trap_arg_count;
610 kern_return_t (*mach_trap_function)(void *);
611 #if defined(__arm64__) || defined(__x86_64__)
612 mach_munge_t *mach_trap_arg_munge32; /* system call arguments for 32-bit */
613 #endif
614 int mach_trap_u32_words;
615 #if MACH_ASSERT
616 const char* mach_trap_name;
617 #endif /* MACH_ASSERT */
618 } mach_trap_t;
619
620 extern const mach_trap_t mach_trap_table[]; /* syscall_sw.h now declares this as const */
621 extern const int mach_trap_count;
622
623 extern const char *const mach_syscall_name_table[];
624
625 /* XXX From osfmk/i386/bsd_i386.c */
626 struct mach_call_args {
627 syscall_arg_t arg1;
628 syscall_arg_t arg2;
629 syscall_arg_t arg3;
630 syscall_arg_t arg4;
631 syscall_arg_t arg5;
632 syscall_arg_t arg6;
633 syscall_arg_t arg7;
634 syscall_arg_t arg8;
635 syscall_arg_t arg9;
636 };
637
638 #undef NSYSCALL
639 #define NSYSCALL mach_trap_count
640
641 #if ((1 << SYSTRACE_SHIFT) <= NSYSCALL)
642 #error 1 << SYSTRACE_SHIFT must exceed number of Mach traps
643 #endif
644
645 typedef struct machtrace_sysent {
646 dtrace_id_t stsy_entry;
647 dtrace_id_t stsy_return;
648 kern_return_t (*stsy_underlying)(void *);
649 int32_t stsy_return_type;
650 } machtrace_sysent_t;
651
652 static machtrace_sysent_t *machtrace_sysent = NULL;
653
654 void (*machtrace_probe)(dtrace_id_t, uint64_t, uint64_t,
655 uint64_t, uint64_t, uint64_t);
656
657 static uint64_t machtrace_getarg(void *, dtrace_id_t, void *, int, int);
658
659 static dtrace_provider_id_t machtrace_id;
660
661 static kern_return_t
dtrace_machtrace_syscall(struct mach_call_args * args)662 dtrace_machtrace_syscall(struct mach_call_args *args)
663 {
664 int code; /* The mach call number */
665
666 machtrace_sysent_t *sy;
667 dtrace_id_t id;
668 kern_return_t rval;
669 #if 0 /* XXX */
670 proc_t *p;
671 #endif
672 syscall_arg_t *ip = (syscall_arg_t *)args;
673 mach_call_t mach_call;
674
675 #if defined (__x86_64__)
676 {
677 pal_register_cache_state(current_thread(), VALID);
678 x86_saved_state_t *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread());
679
680 if (is_saved_state64(tagged_regs)) {
681 code = saved_state64(tagged_regs)->rax & SYSCALL_NUMBER_MASK;
682 } else {
683 code = -saved_state32(tagged_regs)->eax;
684 }
685 }
686 #elif defined(__arm__)
687 {
688 /* r12 has the machcall number, but it is -ve */
689 arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread());
690 code = (int)arm_regs->r[12];
691 ASSERT(code < 0); /* Otherwise it would be a Unix syscall */
692 code = -code;
693 }
694 #elif defined(__arm64__)
695 {
696 /* From arm/thread_status.h:get_saved_state_svc_number */
697 arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread());
698 if (is_saved_state32(arm_regs)) {
699 code = (int)saved_state32(arm_regs)->r[12];
700 } else {
701 code = (int)saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM];
702 }
703
704 /* From bsd/arm64.c:mach_syscall */
705 ASSERT(code < 0); /* Otherwise it would be a Unix syscall */
706 code = -code;
707 }
708 #else
709 #error Unknown Architecture
710 #endif
711
712 sy = &machtrace_sysent[code];
713
714 if ((id = sy->stsy_entry) != DTRACE_IDNONE) {
715 uthread_t uthread = current_uthread();
716
717 if (uthread) {
718 uthread->t_dtrace_syscall_args = (void *)ip;
719 }
720
721 (*machtrace_probe)(id, *ip, *(ip + 1), *(ip + 2), *(ip + 3), *(ip + 4));
722
723 if (uthread) {
724 uthread->t_dtrace_syscall_args = (void *)0;
725 }
726 }
727
728 #if 0 /* XXX */
729 /*
730 * APPLE NOTE: Not implemented.
731 * We want to explicitly allow DTrace consumers to stop a process
732 * before it actually executes the meat of the syscall.
733 */
734 p = ttoproc(curthread);
735 mutex_enter(&p->p_lock);
736 if (curthread->t_dtrace_stop && !curthread->t_lwp->lwp_nostop) {
737 curthread->t_dtrace_stop = 0;
738 stop(PR_REQUESTED, 0);
739 }
740 mutex_exit(&p->p_lock);
741 #endif
742
743 mach_call = (mach_call_t)(*sy->stsy_underlying);
744 rval = mach_call(args);
745
746 if ((id = sy->stsy_return) != DTRACE_IDNONE) {
747 (*machtrace_probe)(id, (uint64_t)rval, 0, 0, 0, 0);
748 }
749
750 return rval;
751 }
752
753 static void
machtrace_init(const mach_trap_t * actual,machtrace_sysent_t ** interposed)754 machtrace_init(const mach_trap_t *actual, machtrace_sysent_t **interposed)
755 {
756 machtrace_sysent_t *msysent = *interposed;
757 int i;
758
759 if (msysent == NULL) {
760 *interposed = msysent = kmem_zalloc(sizeof(machtrace_sysent_t) *
761 NSYSCALL, KM_SLEEP);
762 }
763
764 for (i = 0; i < NSYSCALL; i++) {
765 const volatile mach_trap_t *a = &actual[i];
766 machtrace_sysent_t *s = &msysent[i];
767
768 if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a)) {
769 continue;
770 }
771
772 if (a->mach_trap_function == (mach_call_t)(dtrace_machtrace_syscall)) {
773 continue;
774 }
775
776 s->stsy_underlying = a->mach_trap_function;
777 }
778 }
779
780 /*ARGSUSED*/
781 static void
machtrace_provide(void * arg,const dtrace_probedesc_t * desc)782 machtrace_provide(void *arg, const dtrace_probedesc_t *desc)
783 {
784 #pragma unused(arg) /* __APPLE__ */
785
786 int i;
787
788 if (desc != NULL) {
789 return;
790 }
791
792 machtrace_init(mach_trap_table, &machtrace_sysent);
793
794 for (i = 0; i < NSYSCALL; i++) {
795 if (machtrace_sysent[i].stsy_underlying == NULL) {
796 continue;
797 }
798
799 if (dtrace_probe_lookup(machtrace_id, NULL,
800 mach_syscall_name_table[i], "entry") != 0) {
801 continue;
802 }
803
804 (void) dtrace_probe_create(machtrace_id, NULL, mach_syscall_name_table[i],
805 "entry", MACHTRACE_ARTIFICIAL_FRAMES,
806 (void *)((uintptr_t)SYSTRACE_ENTRY(i)));
807 (void) dtrace_probe_create(machtrace_id, NULL, mach_syscall_name_table[i],
808 "return", MACHTRACE_ARTIFICIAL_FRAMES,
809 (void *)((uintptr_t)SYSTRACE_RETURN(i)));
810
811 machtrace_sysent[i].stsy_entry = DTRACE_IDNONE;
812 machtrace_sysent[i].stsy_return = DTRACE_IDNONE;
813 }
814 }
815
816 /*ARGSUSED*/
817 static void
machtrace_destroy(void * arg,dtrace_id_t id,void * parg)818 machtrace_destroy(void *arg, dtrace_id_t id, void *parg)
819 {
820 #pragma unused(arg,id) /* __APPLE__ */
821 int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
822
823 #pragma unused(sysnum) /* __APPLE__ */
824
825 /*
826 * There's nothing to do here but assert that we have actually been
827 * disabled.
828 */
829 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
830 ASSERT(machtrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE);
831 } else {
832 ASSERT(machtrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
833 }
834 }
835
836 /*ARGSUSED*/
837 static int
machtrace_enable(void * arg,dtrace_id_t id,void * parg)838 machtrace_enable(void *arg, dtrace_id_t id, void *parg)
839 {
840 #pragma unused(arg) /* __APPLE__ */
841
842 int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
843 int enabled = (machtrace_sysent[sysnum].stsy_entry != DTRACE_IDNONE ||
844 machtrace_sysent[sysnum].stsy_return != DTRACE_IDNONE);
845
846 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
847 machtrace_sysent[sysnum].stsy_entry = id;
848 } else {
849 machtrace_sysent[sysnum].stsy_return = id;
850 }
851
852 if (enabled) {
853 ASSERT(mach_trap_table[sysnum].mach_trap_function == (void *)dtrace_machtrace_syscall);
854 return 0;
855 }
856
857 lck_mtx_lock(&dtrace_systrace_lock);
858
859 if (mach_trap_table[sysnum].mach_trap_function == machtrace_sysent[sysnum].stsy_underlying) {
860 /* It is not possible to write to mach_trap_table[] directly because it is const. */
861 vm_offset_t dss = ptrauth_nop_cast(vm_offset_t, &dtrace_machtrace_syscall);
862 ml_nofault_copy((vm_offset_t)&dss, (vm_offset_t)&mach_trap_table[sysnum].mach_trap_function, sizeof(vm_offset_t));
863 }
864
865 lck_mtx_unlock(&dtrace_systrace_lock);
866
867 return 0;
868 }
869
870 /*ARGSUSED*/
871 static void
machtrace_disable(void * arg,dtrace_id_t id,void * parg)872 machtrace_disable(void *arg, dtrace_id_t id, void *parg)
873 {
874 #pragma unused(arg,id) /* __APPLE__ */
875
876 int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
877 int disable = (machtrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE ||
878 machtrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
879
880 if (disable) {
881 /*
882 * Usage of volatile protects the if statement below from being optimized away.
883 *
884 * Compilers are clever and know that const array values can't change in time
885 * and the if below is always false. That is because it can't see that DTrace
886 * injects dtrace_machtrace_syscall dynamically and violates constness of the
887 * array.
888 */
889 volatile const mach_trap_t *machtrap = &mach_trap_table[sysnum];
890
891 lck_mtx_lock(&dtrace_systrace_lock);
892 if (machtrap->mach_trap_function == (mach_call_t)dtrace_machtrace_syscall) {
893 ml_nofault_copy((vm_offset_t)&machtrace_sysent[sysnum].stsy_underlying,
894 (vm_offset_t)&machtrap->mach_trap_function, sizeof(vm_offset_t));
895 }
896 lck_mtx_unlock(&dtrace_systrace_lock);
897 }
898
899 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
900 machtrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE;
901 } else {
902 machtrace_sysent[sysnum].stsy_return = DTRACE_IDNONE;
903 }
904 }
905
906 static dtrace_pattr_t machtrace_attr = {
907 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
908 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
909 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
910 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
911 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
912 };
913
914 static dtrace_pops_t machtrace_pops = {
915 .dtps_provide = machtrace_provide,
916 .dtps_provide_module = NULL,
917 .dtps_enable = machtrace_enable,
918 .dtps_disable = machtrace_disable,
919 .dtps_suspend = NULL,
920 .dtps_resume = NULL,
921 .dtps_getargdesc = NULL,
922 .dtps_getargval = machtrace_getarg,
923 .dtps_usermode = NULL,
924 .dtps_destroy = machtrace_destroy
925 };
926
927 static int
machtrace_attach(dev_info_t * devi)928 machtrace_attach(dev_info_t *devi)
929 {
930 machtrace_probe = dtrace_probe;
931 membar_enter();
932
933 if (ddi_create_minor_node(devi, "machtrace", S_IFCHR, 0,
934 DDI_PSEUDO, 0) == DDI_FAILURE ||
935 dtrace_register("mach_trap", &machtrace_attr, DTRACE_PRIV_USER, NULL,
936 &machtrace_pops, NULL, &machtrace_id) != 0) {
937 machtrace_probe = (void*)&systrace_stub;
938 ddi_remove_minor_node(devi, NULL);
939 return DDI_FAILURE;
940 }
941
942 return DDI_SUCCESS;
943 }
944
945 d_open_t _systrace_open;
946
947 int
_systrace_open(dev_t dev,int flags,int devtype,struct proc * p)948 _systrace_open(dev_t dev, int flags, int devtype, struct proc *p)
949 {
950 #pragma unused(dev,flags,devtype,p)
951 return 0;
952 }
953
954 #define SYSTRACE_MAJOR -24 /* let the kernel pick the device number */
955
956 static struct cdevsw systrace_cdevsw =
957 {
958 .d_open = _systrace_open,
959 .d_close = eno_opcl,
960 .d_read = eno_rdwrt,
961 .d_write = eno_rdwrt,
962 .d_ioctl = eno_ioctl,
963 .d_stop = (stop_fcn_t *)nulldev,
964 .d_reset = (reset_fcn_t *)nulldev,
965 .d_select = eno_select,
966 .d_mmap = eno_mmap,
967 .d_strategy = eno_strat,
968 .d_reserved_1 = eno_getc,
969 .d_reserved_2 = eno_putc,
970 };
971
972 void systrace_init( void );
973
974 void
systrace_init(void)975 systrace_init( void )
976 {
977 if (dtrace_sdt_probes_restricted()) {
978 return;
979 }
980
981 int majdevno = cdevsw_add(SYSTRACE_MAJOR, &systrace_cdevsw);
982
983 if (majdevno < 0) {
984 printf("systrace_init: failed to allocate a major number!\n");
985 return;
986 }
987
988 systrace_attach((dev_info_t*)(uintptr_t)majdevno);
989 machtrace_attach((dev_info_t*)(uintptr_t)majdevno);
990 }
991 #undef SYSTRACE_MAJOR
992
993 static uint64_t
systrace_getargval(void * arg,dtrace_id_t id,void * parg,int argno,int aframes)994 systrace_getargval(void *arg, dtrace_id_t id, void *parg, int argno, int aframes)
995 {
996 #pragma unused(arg,id,parg,aframes) /* __APPLE__ */
997 uint64_t val = 0;
998 uint64_t *uargs = NULL;
999
1000 uthread_t uthread = current_uthread();
1001
1002 if (uthread) {
1003 uargs = uthread->t_dtrace_syscall_args;
1004 }
1005 if (!uargs) {
1006 return 0;
1007 }
1008 if (argno < 0 || argno >= SYSTRACE_NARGS) {
1009 return 0;
1010 }
1011
1012 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
1013 val = uargs[argno];
1014 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
1015 return val;
1016 }
1017
1018 static void
systrace_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)1019 systrace_getargdesc(void *arg, dtrace_id_t id, void *parg,
1020 dtrace_argdesc_t *desc)
1021 {
1022 #pragma unused(arg, id)
1023 int sysnum = SYSTRACE_SYSNUM(parg);
1024 uthread_t uthread = current_uthread();
1025 uint64_t *uargs = NULL;
1026
1027 if (!uthread) {
1028 desc->dtargd_ndx = DTRACE_ARGNONE;
1029 return;
1030 }
1031
1032 uargs = uthread->t_dtrace_syscall_args;
1033
1034 if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
1035 systrace_entry_setargdesc(sysnum, desc->dtargd_ndx,
1036 desc->dtargd_native, sizeof(desc->dtargd_native));
1037 } else {
1038 systrace_return_setargdesc(sysnum, desc->dtargd_ndx,
1039 desc->dtargd_native, sizeof(desc->dtargd_native));
1040 }
1041
1042 if (desc->dtargd_native[0] == '\0') {
1043 desc->dtargd_ndx = DTRACE_ARGNONE;
1044 }
1045 }
1046
1047 static uint64_t
machtrace_getarg(void * arg,dtrace_id_t id,void * parg,int argno,int aframes)1048 machtrace_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes)
1049 {
1050 #pragma unused(arg,id,parg,aframes) /* __APPLE__ */
1051 uint64_t val = 0;
1052 syscall_arg_t *stack = (syscall_arg_t *)NULL;
1053
1054 uthread_t uthread = current_uthread();
1055
1056 if (uthread) {
1057 stack = (syscall_arg_t *)uthread->t_dtrace_syscall_args;
1058 }
1059
1060 if (!stack) {
1061 return 0;
1062 }
1063
1064 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
1065 /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
1066 val = (uint64_t)*(stack + argno);
1067 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
1068 return val;
1069 }
1070