1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <stdbool.h>
31
32 #include <kern/assert.h>
33 #include <kern/cpu_data.h>
34 #include <kern/debug.h>
35 #include <kern/locks.h>
36 #include <kern/kalloc.h>
37 #include <kern/startup.h>
38 #include <kern/task.h>
39 #include <kern/thread.h>
40 #include <kern/zalloc.h>
41
42 #include <vm/vm_kern_xnu.h>
43 #include <vm/vm_protos.h>
44 #include <vm/pmap.h>
45 #include <vm/vm_memory_entry_xnu.h>
46
47 #include <mach/mach_vm.h>
48 #include <mach/mach_types.h>
49 #include <mach/mach_port.h>
50 #include <mach/vm_map.h>
51 #include <mach/vm_param.h>
52 #include <mach/machine/vm_param.h>
53
54 #include <sys/stat.h> /* dev_t */
55 #include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
56 #include <sys/conf.h> /* must come after sys/stat.h */
57 #include <sys/sysctl.h>
58
59 #include <pexpert/pexpert.h> /* PE_parse_boot_argn */
60
61 #include <libkern/libkern.h>
62 #include <libkern/OSKextLibPrivate.h>
63 #include <libkern/kernel_mach_header.h>
64 #include <os/atomic_private.h>
65 #include <os/log.h>
66 #include <os/overflow.h>
67
68 #include <san/kcov_data.h>
69 #include <san/kcov_ksancov.h>
70
71 /* header mess... */
72 struct uthread;
73 typedef struct uthread * uthread_t;
74
75 #include <sys/sysproto.h>
76 #include <sys/queue.h>
77 #include <sys/sysctl.h>
78 #include <sys/kdebug.h>
79
80 #define USE_PC_TABLE 0
81 #define KSANCOV_MAX_DEV 64
82 #define KSANCOV_MAX_PCS (1024U * 64) /* default to 256k buffer => 64k pcs */
83
84 extern boolean_t ml_at_interrupt_context(void);
85 extern boolean_t ml_get_interrupts_enabled(void);
86
87 static void ksancov_detach(ksancov_dev_t);
88
89 static int dev_major;
90 static size_t nedges = 0;
91 static uint32_t __unused npcs = 0;
92
93 static LCK_GRP_DECLARE(ksancov_lck_grp, "ksancov_lck_grp");
94 static LCK_RW_DECLARE(ksancov_devs_lck, &ksancov_lck_grp);
95
96 /* array of devices indexed by devnode minor */
97 static ksancov_dev_t ksancov_devs[KSANCOV_MAX_DEV];
98 static struct ksancov_edgemap *ksancov_edgemap;
99
100 /* Global flag that enables the sanitizer hook. */
101 static _Atomic unsigned int ksancov_enabled = 0;
102
103 /* Toggled after ksancov_init() */
104 static boolean_t ksancov_initialized = false;
105
106
107 /* Support for gated callbacks (referred to as "on demand", "od") */
108 static void kcov_ksancov_bookmark_on_demand_module(uint32_t *start, uint32_t *stop);
109
110 static LCK_MTX_DECLARE(ksancov_od_lck, &ksancov_lck_grp);
111
112 /* Bookkeeping structures for gated sancov instrumentation */
113 struct ksancov_od_module_entry {
114 char bundle[KMOD_MAX_NAME]; /* module bundle */
115 uint32_t idx; /* index into entries/handles arrays */
116 };
117
118 struct ksancov_od_module_handle {
119 uint32_t *start; /* guards boundaries */
120 uint32_t *stop;
121 uint64_t *gate; /* pointer to __DATA,__sancov_gate*/
122 };
123
124 static struct ksancov_od_module_entry *ksancov_od_module_entries = NULL;
125 static struct ksancov_od_module_handle *ksancov_od_module_handles = NULL;
126
127 /* number of entries/handles allocated */
128 static unsigned int ksancov_od_allocated_count = 0;
129
130 /* number of registered modules */
131 static unsigned int ksancov_od_modules_count = 0;
132 /* number of modules whose callbacks are currently enabled */
133 static unsigned int ksancov_od_enabled_count = 0;
134
135 /* Valid values for ksancov.on_demand= boot-arg */
136 #define KSANCOV_OD_SUPPORT 0x0010 // Enable runtime support
137 #define KSANCOV_OD_LOGGING 0x0020 // Enable logging (via os_log)
138
139 __options_decl(ksancov_od_config_t, uint32_t, {
140 KSANCOV_OD_NONE = 0,
141 KSANCOV_OD_ENABLE_SUPPORT = 0x0010,
142 KSANCOV_OD_ENABLE_LOGGING = 0x0020,
143 });
144
145 /* configurable at boot; enabled by default */
146 static ksancov_od_config_t ksancov_od_config = KSANCOV_OD_ENABLE_SUPPORT;
147
148 static unsigned ksancov_od_support_enabled = 1;
149 static unsigned ksancov_od_logging_enabled = 0;
150
151 SYSCTL_DECL(_kern_kcov);
152 SYSCTL_ULONG(_kern_kcov, OID_AUTO, nedges, CTLFLAG_RD, &nedges, "");
153
154 SYSCTL_NODE(_kern_kcov, OID_AUTO, od, CTLFLAG_RD, 0, "od");
155 SYSCTL_UINT(_kern_kcov_od, OID_AUTO, config, CTLFLAG_RD, &ksancov_od_config, 0, "");
156 SYSCTL_UINT(_kern_kcov_od, OID_AUTO, allocated_entries, CTLFLAG_RD, &ksancov_od_allocated_count, 0, "");
157 SYSCTL_UINT(_kern_kcov_od, OID_AUTO, modules_count, CTLFLAG_RD, &ksancov_od_modules_count, 0, "");
158 SYSCTL_UINT(_kern_kcov_od, OID_AUTO, enabled_count, CTLFLAG_RD, &ksancov_od_enabled_count, 0, "");
159 SYSCTL_UINT(_kern_kcov_od, OID_AUTO, support_enabled, CTLFLAG_RD, &ksancov_od_support_enabled, 0, "");
160 SYSCTL_UINT(_kern_kcov_od, OID_AUTO, logging_enabled, CTLFLAG_RW, &ksancov_od_logging_enabled, 0, "");
161
162 #define ksancov_od_log(...) do { \
163 if (ksancov_od_logging_enabled) { \
164 os_log_debug(OS_LOG_DEFAULT, __VA_ARGS__); \
165 } \
166 } while (0)
167
168 __startup_func
169 void
ksancov_init(void)170 ksancov_init(void)
171 {
172 unsigned arg;
173
174 /* handle ksancov boot-args */
175 if (PE_parse_boot_argn("ksancov.on_demand", &arg, sizeof(arg))) {
176 ksancov_od_config = (ksancov_od_config_t)arg;
177 }
178
179 if (ksancov_od_config & KSANCOV_OD_ENABLE_SUPPORT) {
180 /* enable the runtime support for on-demand instrumentation */
181 ksancov_od_support_enabled = 1;
182 ksancov_od_allocated_count = 64;
183 ksancov_od_module_entries = kalloc_type_tag(struct ksancov_od_module_entry,
184 ksancov_od_allocated_count, Z_WAITOK_ZERO_NOFAIL, VM_KERN_MEMORY_DIAG);
185 ksancov_od_module_handles = kalloc_type_tag(struct ksancov_od_module_handle,
186 ksancov_od_allocated_count, Z_WAITOK_ZERO_NOFAIL, VM_KERN_MEMORY_DIAG);
187 } else {
188 ksancov_od_support_enabled = 0;
189 }
190
191 if (ksancov_od_config & KSANCOV_OD_ENABLE_LOGGING) {
192 ksancov_od_logging_enabled = 1;
193 } else {
194 ksancov_od_logging_enabled = 0;
195 }
196
197 ksancov_initialized = true;
198 }
199
200 /*
201 * Coverage sanitizer per-thread routines.
202 */
203
204 /* Initialize per-thread sanitizer data for each new kernel thread. */
205 void
kcov_ksancov_init_thread(ksancov_dev_t * dev)206 kcov_ksancov_init_thread(ksancov_dev_t *dev)
207 {
208 *dev = NULL;
209 }
210
211
212 #define GUARD_SEEN (uint32_t)0x80000000
213 #define GUARD_IDX_MASK (uint32_t)0x0fffffff
214
215 static void
trace_pc_guard_pcs(struct ksancov_dev * dev,uintptr_t pc)216 trace_pc_guard_pcs(struct ksancov_dev *dev, uintptr_t pc)
217 {
218 if (os_atomic_load(&dev->trace->kt_head, relaxed) >= dev->maxpcs) {
219 return; /* overflow */
220 }
221
222 uint32_t idx = os_atomic_inc_orig(&dev->trace->kt_head, relaxed);
223 if (__improbable(idx >= dev->maxpcs)) {
224 return;
225 }
226
227 ksancov_trace_pc_ent_t *entries = (ksancov_trace_pc_ent_t *)dev->trace->kt_entries;
228 entries[idx] = pc;
229 }
230
231 #if CONFIG_STKSZ
232 static void
trace_pc_guard_pcs_stk(struct ksancov_dev * dev,uintptr_t pc,uint32_t stksize)233 trace_pc_guard_pcs_stk(struct ksancov_dev *dev, uintptr_t pc, uint32_t stksize)
234 {
235 if (os_atomic_load(&dev->trace->kt_head, relaxed) >= dev->maxpcs) {
236 return; /* overflow */
237 }
238
239 uint32_t idx = os_atomic_inc_orig(&dev->trace->kt_head, relaxed);
240 if (__improbable(idx >= dev->maxpcs)) {
241 return;
242 }
243
244 ksancov_trace_stksize_ent_t *entries = (ksancov_trace_stksize_ent_t *)dev->trace->kt_entries;
245 entries[idx] = {
246 .pc = pc,
247 .stksize = stksize
248 };
249 }
250 #endif
251
252 static void
trace_pc_guard_counter(struct ksancov_dev * dev,uint32_t * guardp)253 trace_pc_guard_counter(struct ksancov_dev *dev, uint32_t *guardp)
254 {
255 size_t idx = *guardp & GUARD_IDX_MASK;
256 ksancov_counters_t *counters = dev->counters;
257
258 /* saturating 8bit add */
259 if (counters->kc_hits[idx] < KSANCOV_MAX_HITS) {
260 counters->kc_hits[idx]++;
261 }
262 }
263
264 void
kcov_ksancov_trace_guard(uint32_t * guardp,void * caller)265 kcov_ksancov_trace_guard(uint32_t *guardp, void *caller)
266 {
267 /*
268 * Return as early as possible if we haven't had a chance to
269 * create the edge map yet.
270 *
271 * Note: this will also protect us from performing unnecessary
272 * operations (especially during early boot) which may result
273 * in increased maintenance burden for the instrumentation (see
274 * the comment about VM_KERNEL_UNSLIDE below).
275 */
276 if (__probable(ksancov_edgemap == NULL)) {
277 return;
278 }
279
280 if (guardp == NULL) {
281 return;
282 }
283
284 /*
285 * Since this code was originally introduced, VM_KERNEL_UNSLIDE
286 * evolved significantly, and it now expands to a series of
287 * function calls that check whether the address is slid, mask
288 * off tags and ultimately unslide the pointer.
289 *
290 * Therefore we need to make sure that we do not instrument any function
291 * in the closure of VM_KERNEL_UNSLIDE: this would cause a loop where the
292 * instrumentation callbacks end up calling into instrumented code.
293 *
294 */
295 uintptr_t pc = (uintptr_t)(VM_KERNEL_UNSLIDE(caller) - 1);
296
297 uint32_t gd = *guardp;
298 if (__improbable(gd && !(gd & GUARD_SEEN))) {
299 size_t idx = gd & GUARD_IDX_MASK;
300 if (idx < ksancov_edgemap->ke_nedges) {
301 ksancov_edgemap->ke_addrs[idx] = pc;
302 *guardp |= GUARD_SEEN;
303 }
304 }
305 }
306
307 void
kcov_ksancov_trace_pc(kcov_thread_data_t * data,uint32_t * guardp,void * caller,uintptr_t sp)308 kcov_ksancov_trace_pc(kcov_thread_data_t *data, uint32_t *guardp, void *caller, uintptr_t sp)
309 {
310 #pragma unused(sp)
311 uintptr_t pc = (uintptr_t)(VM_KERNEL_UNSLIDE(caller) - 1);
312 ksancov_dev_t dev = data->ktd_device;
313
314 /* Check that we have coverage recording enabled for a thread. */
315 if (__probable(dev == NULL)) {
316 return;
317 }
318
319 if (os_atomic_load(&dev->hdr->kh_enabled, relaxed) == 0) {
320 return;
321 }
322
323 /*
324 * Coverage sanitizer is disabled in the code called below. This allows calling back to the kernel without
325 * the risk of killing machine with recursive calls.
326 */
327 switch (dev->mode) {
328 case KS_MODE_TRACE:
329 trace_pc_guard_pcs(dev, pc);
330 break;
331 #if CONFIG_STKSZ
332 case KS_MODE_STKSIZE:
333 trace_pc_guard_pcs_stk(dev, pc, data->ktd_stksz.kst_stksz);
334 break;
335 #endif
336 case KS_MODE_COUNTERS:
337 trace_pc_guard_counter(dev, guardp);
338 break;
339 default:
340 /*
341 * Treat all unsupported tracing modes as no-op. It is not destructive for the kernel itself just
342 * coverage sanitiser will not record anything in such case.
343 */
344 ;
345 }
346 }
347
348 void
kcov_ksancov_trace_pc_guard_init(uint32_t * start,uint32_t * stop)349 kcov_ksancov_trace_pc_guard_init(uint32_t *start, uint32_t *stop)
350 {
351 const size_t orig_nedges = nedges;
352
353 /* assign a unique number to each guard */
354 for (uint32_t *cur = start; cur != stop; cur++) {
355 /* zero means that the guard has not been assigned */
356 if (*cur == 0) {
357 if (nedges < KSANCOV_MAX_EDGES) {
358 *cur = (uint32_t)++nedges;
359 }
360 }
361 }
362
363 /* only invoke kcov_ksancov_bookmark_on_demand_module if we assigned new guards */
364 if (nedges > orig_nedges) {
365 kcov_ksancov_bookmark_on_demand_module(start, stop);
366 }
367 }
368
369 void
kcov_ksancov_pcs_init(uintptr_t * start,uintptr_t * stop)370 kcov_ksancov_pcs_init(uintptr_t *start, uintptr_t *stop)
371 {
372 #if USE_PC_TABLE
373 static const uintptr_t pc_table_seen_flag = 0x100;
374
375 for (; start < stop; start += 2) {
376 uintptr_t pc = start[0];
377 uintptr_t flags = start[1];
378
379 /*
380 * This function gets called multiple times on the same range, so mark the
381 * ones we've seen using unused bits in the flags field.
382 */
383 if (flags & pc_table_seen_flag) {
384 continue;
385 }
386
387 start[1] |= pc_table_seen_flag;
388 assert(npcs < KSANCOV_MAX_EDGES - 1);
389 edge_addrs[++npcs] = pc;
390 }
391 #else
392 (void)start;
393 (void)stop;
394 #endif
395 }
396
397 static void
kcov_ksancov_bookmark_on_demand_module(uint32_t * start,uint32_t * stop)398 kcov_ksancov_bookmark_on_demand_module(uint32_t *start, uint32_t *stop)
399 {
400 OSKextLoadedKextSummary summary = {};
401 struct ksancov_od_module_entry *entry = NULL;
402 struct ksancov_od_module_handle *handle = NULL;
403 uint64_t *gate_section = NULL;
404 unsigned long gate_sz = 0;
405 uint32_t idx = 0;
406
407 if (!ksancov_od_support_enabled) {
408 return;
409 }
410
411 if (OSKextGetLoadedKextSummaryForAddress(start, &summary) != KERN_SUCCESS) {
412 return;
413 }
414
415 if (!ksancov_initialized) {
416 ksancov_od_log("ksancov: Dropping %s pre-initialization\n", summary.name);
417 return;
418 }
419
420 if (nedges >= KSANCOV_MAX_EDGES) {
421 ksancov_od_log("ksancov: Dropping %s: maximum number of edges reached\n",
422 summary.name);
423 return;
424 }
425
426 /*
427 * The __DATA,__sancov_gate section is where the compiler stores the 64-bit
428 * global variable that is used by the inline instrumentation to decide
429 * whether it should call into the runtime or not.
430 */
431 gate_section = getsectdatafromheader((kernel_mach_header_t *)summary.address,
432 "__DATA", "__sancov_gate", &gate_sz);
433 if (gate_sz == 0) {
434 ksancov_od_log("ksancov: Dropping %s: not instrumented with gated callbacks\n",
435 summary.name);
436 return;
437 }
438
439 lck_mtx_lock(&ksancov_od_lck);
440
441 /* reallocate the bookkeeping structures if needed */
442 if (ksancov_od_modules_count >= ksancov_od_allocated_count) {
443 unsigned int old_ksancov_od_allocated_count = ksancov_od_allocated_count;
444 ksancov_od_allocated_count += (ksancov_od_allocated_count / 2);
445
446 ksancov_od_log("ksancov: Reallocating entries: %u -> %u\n",
447 old_ksancov_od_allocated_count,
448 ksancov_od_allocated_count);
449
450 ksancov_od_module_entries = krealloc_type_tag(struct ksancov_od_module_entry,
451 old_ksancov_od_allocated_count,
452 ksancov_od_allocated_count,
453 ksancov_od_module_entries, Z_WAITOK_ZERO | Z_REALLOCF, VM_KERN_MEMORY_DIAG);
454
455 ksancov_od_module_handles = krealloc_type_tag(struct ksancov_od_module_handle,
456 old_ksancov_od_allocated_count,
457 ksancov_od_allocated_count,
458 ksancov_od_module_handles, Z_WAITOK_ZERO | Z_REALLOCF, VM_KERN_MEMORY_DIAG);
459 }
460
461 /* this is the index of the entry we're going to fill in both arrays */
462 idx = ksancov_od_modules_count++;
463
464 entry = &ksancov_od_module_entries[idx];
465 handle = &ksancov_od_module_handles[idx];
466
467 handle->start = start;
468 handle->stop = stop;
469 handle->gate = gate_section;
470
471 strlcpy(entry->bundle, summary.name, sizeof(entry->bundle));
472 entry->idx = (uint32_t)idx;
473
474 ksancov_od_log("ksancov: Bookmarked module %s (0x%lx - 0x%lx, %lu guards) [idx: %u]\n",
475 entry->bundle, (uintptr_t)handle->start, (uintptr_t)handle->stop,
476 handle->stop - handle->start, entry->idx);
477 lck_mtx_unlock(&ksancov_od_lck);
478 }
479
480 /*
481 * Coverage sanitizer pseudo-device code.
482 */
483
484 static ksancov_dev_t
create_dev(dev_t dev)485 create_dev(dev_t dev)
486 {
487 ksancov_dev_t d;
488
489 d = kalloc_type_tag(struct ksancov_dev, Z_WAITOK_ZERO_NOFAIL, VM_KERN_MEMORY_DIAG);
490 d->mode = KS_MODE_NONE;
491 d->maxpcs = KSANCOV_MAX_PCS;
492 d->dev = dev;
493 lck_mtx_init(&d->lock, &ksancov_lck_grp, LCK_ATTR_NULL);
494
495 return d;
496 }
497
498 static void
free_dev(ksancov_dev_t d)499 free_dev(ksancov_dev_t d)
500 {
501 if (d->mode == KS_MODE_TRACE && d->trace) {
502 kmem_free(kernel_map, (uintptr_t)d->trace, d->sz);
503 } else if (d->mode == KS_MODE_COUNTERS && d->counters) {
504 kmem_free(kernel_map, (uintptr_t)d->counters, d->sz);
505 }
506 lck_mtx_destroy(&d->lock, &ksancov_lck_grp);
507 kfree_type(struct ksancov_dev, d);
508 }
509
510 static void *
ksancov_do_map(uintptr_t base,size_t sz,vm_prot_t prot)511 ksancov_do_map(uintptr_t base, size_t sz, vm_prot_t prot)
512 {
513 kern_return_t kr;
514 mach_port_t mem_entry = MACH_PORT_NULL;
515 mach_vm_address_t user_addr = 0;
516 memory_object_size_t size = sz;
517
518 kr = mach_make_memory_entry_64(kernel_map,
519 &size,
520 (mach_vm_offset_t)base,
521 MAP_MEM_VM_SHARE | prot,
522 &mem_entry,
523 MACH_PORT_NULL);
524 if (kr != KERN_SUCCESS) {
525 return NULL;
526 }
527
528 kr = mach_vm_map_kernel(get_task_map(current_task()),
529 &user_addr,
530 size,
531 0,
532 VM_MAP_KERNEL_FLAGS_ANYWHERE(),
533 mem_entry,
534 0,
535 FALSE,
536 prot,
537 prot,
538 VM_INHERIT_SHARE);
539
540 /*
541 * At this point, either vm_map() has taken a reference on the memory entry
542 * and we can release our local reference, or the map failed and the entry
543 * needs to be freed.
544 */
545 mach_memory_entry_port_release(mem_entry);
546
547 if (kr != KERN_SUCCESS) {
548 return NULL;
549 }
550
551 return (void *)user_addr;
552 }
553
554 /*
555 * map the sancov buffer into the current process
556 */
557 static int
ksancov_map(ksancov_dev_t d,uintptr_t * bufp,size_t * sizep)558 ksancov_map(ksancov_dev_t d, uintptr_t *bufp, size_t *sizep)
559 {
560 uintptr_t addr;
561 size_t size = d->sz;
562
563 switch (d->mode) {
564 case KS_MODE_STKSIZE:
565 case KS_MODE_TRACE:
566 if (!d->trace) {
567 return EINVAL;
568 }
569 addr = (uintptr_t)d->trace;
570 break;
571 case KS_MODE_COUNTERS:
572 if (!d->counters) {
573 return EINVAL;
574 }
575 addr = (uintptr_t)d->counters;
576 break;
577 default:
578 return EINVAL; /* not configured */
579 }
580
581 void *buf = ksancov_do_map(addr, size, VM_PROT_READ | VM_PROT_WRITE);
582 if (buf == NULL) {
583 return ENOMEM;
584 }
585
586 *bufp = (uintptr_t)buf;
587 *sizep = size;
588
589 return 0;
590 }
591
592 /*
593 * map the edge -> pc mapping as read-only
594 */
595 static int
ksancov_map_edgemap(uintptr_t * bufp,size_t * sizep)596 ksancov_map_edgemap(uintptr_t *bufp, size_t *sizep)
597 {
598 uintptr_t addr;
599 size_t size;
600
601 if (ksancov_edgemap == NULL) {
602 return EINVAL;
603 }
604
605 addr = (uintptr_t)ksancov_edgemap;
606 size = sizeof(ksancov_edgemap_t) + ksancov_edgemap->ke_nedges * sizeof(uintptr_t);
607
608 void *buf = ksancov_do_map(addr, size, VM_PROT_READ);
609 if (buf == NULL) {
610 return ENOMEM;
611 }
612
613 *bufp = (uintptr_t)buf;
614 *sizep = size;
615 return 0;
616 }
617
618 /*
619 * Device node management
620 */
621
622 static int
ksancov_open(dev_t dev,int flags,int devtype,proc_t p)623 ksancov_open(dev_t dev, int flags, int devtype, proc_t p)
624 {
625 #pragma unused(flags,devtype,p)
626 const int minor_num = minor(dev);
627
628 if (minor_num < 0 || minor_num >= KSANCOV_MAX_DEV) {
629 return ENXIO;
630 }
631
632 lck_rw_lock_exclusive(&ksancov_devs_lck);
633
634 if (ksancov_devs[minor_num]) {
635 lck_rw_unlock_exclusive(&ksancov_devs_lck);
636 return EBUSY;
637 }
638
639 ksancov_dev_t d = create_dev(dev);
640 if (!d) {
641 lck_rw_unlock_exclusive(&ksancov_devs_lck);
642 return ENOMEM;
643 }
644 ksancov_devs[minor_num] = d;
645
646 if (ksancov_edgemap == NULL) {
647 uintptr_t buf;
648 size_t sz = sizeof(struct ksancov_edgemap) + nedges * sizeof(uintptr_t);
649
650 kern_return_t kr = kmem_alloc(kernel_map, &buf, sz,
651 KMA_DATA | KMA_ZERO | KMA_PERMANENT, VM_KERN_MEMORY_DIAG);
652 if (kr) {
653 printf("ksancov: failed to allocate edge addr map\n");
654 lck_rw_unlock_exclusive(&ksancov_devs_lck);
655 return ENOMEM;
656 }
657
658 ksancov_edgemap = (void *)buf;
659 ksancov_edgemap->ke_magic = KSANCOV_EDGEMAP_MAGIC;
660 ksancov_edgemap->ke_nedges = (uint32_t)nedges;
661 }
662
663 lck_rw_unlock_exclusive(&ksancov_devs_lck);
664
665 return 0;
666 }
667
668 static int
ksancov_trace_alloc(ksancov_dev_t d,ksancov_mode_t mode,size_t maxpcs)669 ksancov_trace_alloc(ksancov_dev_t d, ksancov_mode_t mode, size_t maxpcs)
670 {
671 if (d->mode != KS_MODE_NONE) {
672 return EBUSY; /* trace/counters already created */
673 }
674 assert(d->trace == NULL);
675
676 uintptr_t buf;
677 size_t sz;
678
679 if (mode == KS_MODE_TRACE) {
680 if (os_mul_and_add_overflow(maxpcs, sizeof(ksancov_trace_pc_ent_t),
681 sizeof(struct ksancov_trace), &sz)) {
682 return EINVAL;
683 }
684 } else if (mode == KS_MODE_STKSIZE) {
685 if (os_mul_and_add_overflow(maxpcs, sizeof(ksancov_trace_stksize_ent_t),
686 sizeof(struct ksancov_trace), &sz)) {
687 return EINVAL;
688 }
689 } else {
690 return EINVAL;
691 }
692
693 /* allocate the shared memory buffer */
694 kern_return_t kr = kmem_alloc(kernel_map, &buf, sz, KMA_DATA | KMA_ZERO,
695 VM_KERN_MEMORY_DIAG);
696 if (kr != KERN_SUCCESS) {
697 return ENOMEM;
698 }
699
700 struct ksancov_trace *trace = (struct ksancov_trace *)buf;
701 trace->kt_hdr.kh_magic = (mode == KS_MODE_TRACE) ? KSANCOV_TRACE_MAGIC : KSANCOV_STKSIZE_MAGIC;
702 os_atomic_init(&trace->kt_head, 0);
703 os_atomic_init(&trace->kt_hdr.kh_enabled, 0);
704 trace->kt_maxent = (uint32_t)maxpcs;
705
706 d->trace = trace;
707 d->sz = sz;
708 d->maxpcs = maxpcs;
709 d->mode = mode;
710
711 return 0;
712 }
713
714 static int
ksancov_counters_alloc(ksancov_dev_t d)715 ksancov_counters_alloc(ksancov_dev_t d)
716 {
717 if (d->mode != KS_MODE_NONE) {
718 return EBUSY; /* trace/counters already created */
719 }
720 assert(d->counters == NULL);
721
722 uintptr_t buf;
723 size_t sz = sizeof(struct ksancov_counters) + ksancov_edgemap->ke_nedges * sizeof(uint8_t);
724
725 /* allocate the shared memory buffer */
726 kern_return_t kr = kmem_alloc(kernel_map, &buf, sz, KMA_DATA | KMA_ZERO,
727 VM_KERN_MEMORY_DIAG);
728 if (kr != KERN_SUCCESS) {
729 return ENOMEM;
730 }
731
732 ksancov_counters_t *counters = (ksancov_counters_t *)buf;
733 counters->kc_hdr.kh_magic = KSANCOV_COUNTERS_MAGIC;
734 counters->kc_nedges = ksancov_edgemap->ke_nedges;
735 os_atomic_init(&counters->kc_hdr.kh_enabled, 0);
736
737 d->counters = counters;
738 d->sz = sz;
739 d->mode = KS_MODE_COUNTERS;
740
741 return 0;
742 }
743
744 /*
745 * attach a thread to a ksancov dev instance
746 */
747 static int
ksancov_attach(ksancov_dev_t d,thread_t th)748 ksancov_attach(ksancov_dev_t d, thread_t th)
749 {
750 if (d->mode == KS_MODE_NONE) {
751 return EINVAL; /* not configured */
752 }
753
754 if (th != current_thread()) {
755 /* can only attach to self presently */
756 return EINVAL;
757 }
758
759 kcov_thread_data_t *data = kcov_get_thread_data(th);
760 if (data->ktd_device) {
761 return EBUSY; /* one dev per thread */
762 }
763
764 if (d->thread != THREAD_NULL) {
765 ksancov_detach(d);
766 }
767
768 d->thread = th;
769 thread_reference(d->thread);
770
771 os_atomic_store(&data->ktd_device, d, relaxed);
772 os_atomic_add(&ksancov_enabled, 1, relaxed);
773 kcov_enable();
774
775 return 0;
776 }
777
778 extern void
779 thread_wait(
780 thread_t thread,
781 boolean_t until_not_runnable);
782
783
784 /*
785 * disconnect thread from ksancov dev
786 */
787 static void
ksancov_detach(ksancov_dev_t d)788 ksancov_detach(ksancov_dev_t d)
789 {
790 if (d->thread == THREAD_NULL) {
791 /* no thread attached */
792 return;
793 }
794
795 /* disconnect dev from thread */
796 kcov_thread_data_t *data = kcov_get_thread_data(d->thread);
797 if (data->ktd_device != NULL) {
798 assert(data->ktd_device == d);
799 os_atomic_store(&data->ktd_device, NULL, relaxed);
800 }
801
802 if (d->thread != current_thread()) {
803 /* wait until it's safe to yank */
804 thread_wait(d->thread, TRUE);
805 }
806
807 assert(ksancov_enabled >= 1);
808 os_atomic_sub(&ksancov_enabled, 1, relaxed);
809 kcov_disable();
810
811 /* drop our thread reference */
812 thread_deallocate(d->thread);
813 d->thread = THREAD_NULL;
814 }
815
816 static int
ksancov_close(dev_t dev,int flags,int devtype,proc_t p)817 ksancov_close(dev_t dev, int flags, int devtype, proc_t p)
818 {
819 #pragma unused(flags,devtype,p)
820 const int minor_num = minor(dev);
821
822 if (minor_num < 0 || minor_num >= KSANCOV_MAX_DEV) {
823 return ENXIO;
824 }
825
826 lck_rw_lock_exclusive(&ksancov_devs_lck);
827 ksancov_dev_t d = ksancov_devs[minor_num];
828 ksancov_devs[minor_num] = NULL; /* dev no longer discoverable */
829 lck_rw_unlock_exclusive(&ksancov_devs_lck);
830
831 /*
832 * No need to lock d here as there is and will be no one having its
833 * reference except for this thread and the one which is going to
834 * be detached below.
835 */
836
837 if (!d) {
838 return ENXIO;
839 }
840
841 if (d->mode != KS_MODE_NONE && d->hdr != NULL) {
842 os_atomic_store(&d->hdr->kh_enabled, 0, relaxed); /* stop tracing */
843 }
844
845 ksancov_detach(d);
846 free_dev(d);
847
848 return 0;
849 }
850
851 static void
ksancov_testpanic(volatile uint64_t guess)852 ksancov_testpanic(volatile uint64_t guess)
853 {
854 const uint64_t tgt = 0xf85de3b12891c817UL;
855
856 #define X(n) ((tgt & (0xfUL << (4*n))) == (guess & (0xfUL << (4*n))))
857
858 if (X(0)) {
859 if (X(1)) {
860 if (X(2)) {
861 if (X(3)) {
862 if (X(4)) {
863 if (X(5)) {
864 if (X(6)) {
865 if (X(7)) {
866 if (X(8)) {
867 if (X(9)) {
868 if (X(10)) {
869 if (X(11)) {
870 if (X(12)) {
871 if (X(13)) {
872 if (X(14)) {
873 if (X(15)) {
874 panic("ksancov: found test value");
875 }
876 }
877 }
878 }
879 }
880 }
881 }
882 }
883 }
884 }
885 }
886 }
887 }
888 }
889 }
890 }
891 }
892
893 static int
ksancov_handle_on_demand_cmd(struct ksancov_on_demand_msg * kmsg)894 ksancov_handle_on_demand_cmd(struct ksancov_on_demand_msg *kmsg)
895 {
896 struct ksancov_od_module_entry *entry = NULL;
897 struct ksancov_od_module_handle *handle = NULL;
898 ksancov_on_demand_operation_t op = kmsg->operation;
899 int ret = 0;
900
901 lck_mtx_lock(&ksancov_od_lck);
902
903 /* find the entry/handle to the module */
904 for (unsigned int idx = 0; idx < ksancov_od_modules_count; idx++) {
905 entry = &ksancov_od_module_entries[idx];
906 if (strncmp(entry->bundle, kmsg->bundle, sizeof(entry->bundle)) == 0) {
907 handle = &ksancov_od_module_handles[idx];
908 break;
909 }
910 }
911
912 if (handle == NULL) {
913 ksancov_od_log("ksancov: Could not find module '%s'\n", kmsg->bundle);
914 lck_mtx_unlock(&ksancov_od_lck);
915 return EINVAL;
916 }
917
918 switch (op) {
919 case KS_OD_GET_GATE:
920 /* Get whether on-demand instrumentation is enabled in a given module */
921 if (handle->gate) {
922 kmsg->gate = *handle->gate;
923 } else {
924 ret = EINVAL;
925 }
926 break;
927 case KS_OD_SET_GATE:
928 /* Toggle callback invocation for a given module */
929 if (handle->gate) {
930 ksancov_od_log("ksancov: Setting gate for '%s': %llu\n",
931 kmsg->bundle, kmsg->gate);
932 if (kmsg->gate != *handle->gate) {
933 if (kmsg->gate) {
934 ksancov_od_enabled_count++;
935 } else {
936 ksancov_od_enabled_count--;
937 }
938 *handle->gate = kmsg->gate;
939 }
940 } else {
941 ret = EINVAL;
942 }
943 break;
944 case KS_OD_GET_RANGE:
945 /* Get which range of the guards table covers the given module */
946 ksancov_od_log("ksancov: Range for '%s': %u, %u\n",
947 kmsg->bundle, *handle->start, *(handle->stop - 1));
948 kmsg->range.start = *handle->start;
949 kmsg->range.stop = *(handle->stop - 1);
950 break;
951 default:
952 ret = EINVAL;
953 break;
954 }
955
956 lck_mtx_unlock(&ksancov_od_lck);
957 return ret;
958 }
959
960 static int
ksancov_ioctl(dev_t dev,unsigned long cmd,caddr_t _data,int fflag,proc_t p)961 ksancov_ioctl(dev_t dev, unsigned long cmd, caddr_t _data, int fflag, proc_t p)
962 {
963 #pragma unused(fflag,p)
964 const int minor_num = minor(dev);
965
966 if (minor_num < 0 || minor_num >= KSANCOV_MAX_DEV) {
967 return ENXIO;
968 }
969
970 struct ksancov_buf_desc *mcmd;
971 void *data = (void *)_data;
972
973 lck_rw_lock_shared(&ksancov_devs_lck);
974 ksancov_dev_t d = ksancov_devs[minor_num];
975 if (!d) {
976 lck_rw_unlock_shared(&ksancov_devs_lck);
977 return EINVAL; /* dev not open */
978 }
979
980 int ret = 0;
981
982 switch (cmd) {
983 case KSANCOV_IOC_TRACE:
984 case KSANCOV_IOC_STKSIZE:
985 lck_mtx_lock(&d->lock);
986 ksancov_mode_t mode = (cmd == KSANCOV_IOC_TRACE) ? KS_MODE_TRACE : KS_MODE_STKSIZE;
987 ret = ksancov_trace_alloc(d, mode, *(size_t *)data);
988 lck_mtx_unlock(&d->lock);
989 break;
990 case KSANCOV_IOC_COUNTERS:
991 lck_mtx_lock(&d->lock);
992 ret = ksancov_counters_alloc(d);
993 lck_mtx_unlock(&d->lock);
994 break;
995 case KSANCOV_IOC_MAP:
996 mcmd = (struct ksancov_buf_desc *)data;
997 lck_mtx_lock(&d->lock);
998 ret = ksancov_map(d, &mcmd->ptr, &mcmd->sz);
999 lck_mtx_unlock(&d->lock);
1000 break;
1001 case KSANCOV_IOC_MAP_EDGEMAP:
1002 mcmd = (struct ksancov_buf_desc *)data;
1003 ret = ksancov_map_edgemap(&mcmd->ptr, &mcmd->sz);
1004 break;
1005 case KSANCOV_IOC_START:
1006 lck_mtx_lock(&d->lock);
1007 ret = ksancov_attach(d, current_thread());
1008 lck_mtx_unlock(&d->lock);
1009 break;
1010 case KSANCOV_IOC_NEDGES:
1011 *(size_t *)data = nedges;
1012 break;
1013 case KSANCOV_IOC_ON_DEMAND:
1014 ret = ksancov_handle_on_demand_cmd((struct ksancov_on_demand_msg *)data);
1015 break;
1016 case KSANCOV_IOC_TESTPANIC:
1017 ksancov_testpanic(*(uint64_t *)data);
1018 break;
1019 default:
1020 ret = EINVAL;
1021 break;
1022 }
1023
1024 lck_rw_unlock_shared(&ksancov_devs_lck);
1025
1026 return ret;
1027 }
1028
1029 static int
ksancov_dev_clone(dev_t dev,int action)1030 ksancov_dev_clone(dev_t dev, int action)
1031 {
1032 #pragma unused(dev)
1033 if (action == DEVFS_CLONE_ALLOC) {
1034 for (int i = 0; i < KSANCOV_MAX_DEV; i++) {
1035 if (ksancov_devs[i] == NULL) {
1036 return i;
1037 }
1038 }
1039 } else if (action == DEVFS_CLONE_FREE) {
1040 return 0;
1041 }
1042
1043 return -1;
1044 }
1045
1046 static const struct cdevsw
1047 ksancov_cdev = {
1048 .d_open = ksancov_open,
1049 .d_close = ksancov_close,
1050 .d_ioctl = ksancov_ioctl,
1051
1052 .d_read = eno_rdwrt,
1053 .d_write = eno_rdwrt,
1054 .d_stop = eno_stop,
1055 .d_reset = eno_reset,
1056 .d_select = eno_select,
1057 .d_mmap = eno_mmap,
1058 .d_strategy = eno_strat,
1059 .d_type = 0
1060 };
1061
1062 int
ksancov_init_dev(void)1063 ksancov_init_dev(void)
1064 {
1065 dev_major = cdevsw_add(-1, &ksancov_cdev);
1066 if (dev_major < 0) {
1067 printf("ksancov: failed to allocate major device node\n");
1068 return -1;
1069 }
1070
1071 dev_t dev = makedev(dev_major, 0);
1072 void *node = devfs_make_node_clone(dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
1073 ksancov_dev_clone, KSANCOV_DEVNODE);
1074 if (!node) {
1075 printf("ksancov: failed to create device node\n");
1076 return -1;
1077 }
1078
1079 return 0;
1080 }
1081