1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/monotonic.h>
30 #include <kern/perfmon.h>
31 #include <machine/machine_routines.h>
32 #include <machine/monotonic.h>
33 #include <pexpert/pexpert.h>
34 #include <sys/param.h> /* NULL */
35 #include <sys/stat.h> /* dev_t */
36 #include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
37 #include <sys/conf.h> /* must come after sys/stat.h */
38 #include <sys/resource_private.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysproto.h>
41 #include <sys/systm.h>
42 #include <sys/types.h>
43 #include <sys/monotonic.h>
44
45 static int mt_cdev_open(dev_t dev, int flags, int devtype, proc_t p);
46 static int mt_cdev_close(dev_t dev, int flags, int devtype, proc_t p);
47 static int mt_cdev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
48 proc_t p);
49
50 #define MT_NODE "monotonic"
51
52 static const struct cdevsw mt_cdevsw = {
53 .d_open = mt_cdev_open,
54 .d_close = mt_cdev_close,
55 .d_ioctl = mt_cdev_ioctl,
56
57 .d_read = eno_rdwrt, .d_write = eno_rdwrt, .d_stop = eno_stop,
58 .d_reset = eno_reset, .d_ttys = NULL, .d_select = eno_select,
59 .d_mmap = eno_mmap, .d_strategy = eno_strat, .d_type = 0
60 };
61
62 /*
63 * Written at initialization, read-only thereafter.
64 */
65 LCK_GRP_DECLARE(mt_lock_grp, MT_NODE);
66 static int mt_dev_major;
67
68 static mt_device_t
mt_get_device(dev_t devnum)69 mt_get_device(dev_t devnum)
70 {
71 return &mt_devices[minor(devnum)];
72 }
73
74 static void
mt_device_lock(mt_device_t dev)75 mt_device_lock(mt_device_t dev)
76 {
77 lck_mtx_lock(&dev->mtd_lock);
78 }
79
80 static void
mt_device_unlock(mt_device_t dev)81 mt_device_unlock(mt_device_t dev)
82 {
83 lck_mtx_unlock(&dev->mtd_lock);
84 }
85
86 static void
mt_device_assert_lock_held(__assert_only mt_device_t dev)87 mt_device_assert_lock_held(__assert_only mt_device_t dev)
88 {
89 LCK_MTX_ASSERT(&dev->mtd_lock, LCK_MTX_ASSERT_OWNED);
90 }
91
92 static void
mt_device_assert_inuse(__assert_only mt_device_t dev)93 mt_device_assert_inuse(__assert_only mt_device_t dev)
94 {
95 assert(dev->mtd_inuse == true);
96 }
97
98 int
mt_dev_init(void)99 mt_dev_init(void)
100 {
101 mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
102 if (mt_dev_major < 0) {
103 panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
104 __builtin_unreachable();
105 }
106
107 for (int i = 0; i < MT_NDEVS; i++) {
108 if (mt_devices[i].mtd_init(&mt_devices[i])) {
109 continue;
110 }
111
112 assert(mt_devices[i].mtd_ncounters > 0);
113
114 dev_t dev = makedev(mt_dev_major, i);
115 void *node = devfs_make_node(dev, DEVFS_CHAR, UID_ROOT,
116 GID_WINDOWSERVER, 0666, MT_NODE "/%s",
117 mt_devices[i].mtd_name);
118 if (!node) {
119 panic("monotonic: devfs_make_node failed for '%s'",
120 mt_devices[i].mtd_name);
121 __builtin_unreachable();
122 }
123
124 lck_mtx_init(&mt_devices[i].mtd_lock, &mt_lock_grp, LCK_ATTR_NULL);
125 }
126
127 return 0;
128 }
129
130 static int
mt_cdev_open(dev_t devnum,__unused int flags,__unused int devtype,__unused proc_t p)131 mt_cdev_open(dev_t devnum, __unused int flags, __unused int devtype,
132 __unused proc_t p)
133 {
134 int error = 0;
135
136 mt_device_t dev = mt_get_device(devnum);
137 if (!perfmon_acquire(perfmon_upmu, "monotonic")) {
138 return EBUSY;
139 }
140 mt_device_lock(dev);
141 if (dev->mtd_inuse) {
142 error = EALREADY;
143 } else if (!mt_acquire_counters()) {
144 error = ECONNREFUSED;
145 } else {
146 dev->mtd_reset();
147 dev->mtd_inuse = true;
148 }
149 mt_device_unlock(dev);
150
151 if (error != 0) {
152 perfmon_release(perfmon_upmu, "monotonic");
153 }
154 return error;
155 }
156
157 static int
mt_cdev_close(dev_t devnum,__unused int flags,__unused int devtype,__unused struct proc * p)158 mt_cdev_close(dev_t devnum, __unused int flags, __unused int devtype,
159 __unused struct proc *p)
160 {
161 mt_device_t dev = mt_get_device(devnum);
162
163 perfmon_release(perfmon_upmu, "monotonic");
164
165 mt_device_lock(dev);
166 mt_device_assert_inuse(dev);
167 dev->mtd_inuse = false;
168 dev->mtd_reset();
169 mt_release_counters();
170 mt_device_unlock(dev);
171
172 return 0;
173 }
174
175 static int
mt_ctl_add(mt_device_t dev,user_addr_t uptr)176 mt_ctl_add(mt_device_t dev, user_addr_t uptr)
177 {
178 int error;
179 uint32_t ctr;
180 union monotonic_ctl_add ctl;
181
182 mt_device_assert_lock_held(dev);
183
184 error = copyin(uptr, &ctl, sizeof(ctl.in));
185 if (error) {
186 return error;
187 }
188
189 error = dev->mtd_add(&ctl.in.config, &ctr);
190 if (error) {
191 return error;
192 }
193
194 ctl.out.ctr = ctr;
195
196 error = copyout(&ctl, uptr, sizeof(ctl.out));
197 if (error) {
198 return error;
199 }
200
201 return 0;
202 }
203
204 static int
mt_ctl_counts(mt_device_t dev,user_addr_t uptr)205 mt_ctl_counts(mt_device_t dev, user_addr_t uptr)
206 {
207 int error;
208 union monotonic_ctl_counts ctl;
209
210 mt_device_assert_lock_held(dev);
211
212 error = copyin(uptr, &ctl, sizeof(ctl.in));
213 if (error) {
214 return error;
215 }
216
217 if (ctl.in.ctr_mask == 0) {
218 return EINVAL;
219 }
220
221 {
222 uint64_t counts[dev->mtd_nmonitors][dev->mtd_ncounters];
223 memset(counts, 0,
224 dev->mtd_ncounters * dev->mtd_nmonitors * sizeof(counts[0][0]));
225 error = dev->mtd_read(ctl.in.ctr_mask, (uint64_t *)counts);
226 if (error) {
227 return error;
228 }
229
230 error = copyout(&counts, uptr, sizeof(counts));
231 if (error) {
232 return error;
233 }
234 }
235
236 return 0;
237 }
238
239 static int
mt_ctl_enable(mt_device_t dev,user_addr_t uptr)240 mt_ctl_enable(mt_device_t dev, user_addr_t uptr)
241 {
242 int error;
243 union monotonic_ctl_enable ctl;
244
245 mt_device_assert_lock_held(dev);
246
247 error = copyin(uptr, &ctl, sizeof(ctl));
248 if (error) {
249 return error;
250 }
251
252 dev->mtd_enable(ctl.in.enable);
253
254 return 0;
255 }
256
257 static int
mt_ctl_reset(mt_device_t dev)258 mt_ctl_reset(mt_device_t dev)
259 {
260 mt_device_assert_lock_held(dev);
261 dev->mtd_reset();
262 return 0;
263 }
264
265 static int
mt_cdev_ioctl(dev_t devnum,unsigned long cmd,char * arg,__unused int flags,__unused proc_t p)266 mt_cdev_ioctl(dev_t devnum, unsigned long cmd, char *arg, __unused int flags,
267 __unused proc_t p)
268 {
269 int error = ENODEV;
270 user_addr_t uptr = *(user_addr_t *)(void *)arg;
271
272 mt_device_t dev = mt_get_device(devnum);
273 mt_device_lock(dev);
274
275 switch (cmd) {
276 case MT_IOC_RESET:
277 error = mt_ctl_reset(dev);
278 break;
279
280 case MT_IOC_ADD:
281 error = mt_ctl_add(dev, uptr);
282 break;
283
284 case MT_IOC_ENABLE:
285 error = mt_ctl_enable(dev, uptr);
286 break;
287
288 case MT_IOC_COUNTS:
289 error = mt_ctl_counts(dev, uptr);
290 break;
291
292 case MT_IOC_GET_INFO: {
293 union monotonic_ctl_info info = {
294 .out = {
295 .nmonitors = dev->mtd_nmonitors,
296 .ncounters = dev->mtd_ncounters,
297 },
298 };
299 error = copyout(&info, uptr, sizeof(info));
300 break;
301 }
302
303 default:
304 error = ENODEV;
305 break;
306 }
307
308 mt_device_unlock(dev);
309
310 return error;
311 }
312
313 int
thread_selfcounts(__unused struct proc * p,struct thread_selfcounts_args * uap,__unused int * ret_out)314 thread_selfcounts(__unused struct proc *p,
315 struct thread_selfcounts_args *uap, __unused int *ret_out)
316 {
317 switch (uap->kind) {
318 case THSC_CPI: {
319 struct thsc_cpi counts = { 0 };
320 uint64_t thread_counts[MT_CORE_NFIXED] = { 0 };
321
322 mt_cur_thread_fixed_counts(thread_counts);
323
324 #ifdef MT_CORE_INSTRS
325 counts.tcpi_instructions = thread_counts[MT_CORE_INSTRS];
326 #endif /* defined(MT_CORE_INSTRS) */
327 counts.tcpi_cycles = thread_counts[MT_CORE_CYCLES];
328
329 return copyout(&counts, uap->buf, MIN(sizeof(counts), uap->nbytes));
330 }
331 default:
332 return EINVAL;
333 }
334 }
335
336 enum mt_sysctl {
337 MT_SUPPORTED,
338 MT_PMIS,
339 MT_RETROGRADE,
340 MT_TASK_THREAD,
341 MT_DEBUG,
342 MT_KDBG_TEST,
343 MT_FIX_CPU_PERF,
344 MT_FIX_THREAD_PERF,
345 MT_FIX_TASK_PERF,
346 };
347
348 static int
349 mt_sysctl SYSCTL_HANDLER_ARGS
350 {
351 #pragma unused(oidp, arg2)
352 uint64_t start[MT_CORE_NFIXED] = { 0 }, end[MT_CORE_NFIXED] = { 0 };
353 uint64_t counts[2] = { 0 };
354
355 switch ((enum mt_sysctl)arg1) {
356 case MT_SUPPORTED:
357 return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
358 case MT_PMIS:
359 return sysctl_io_number(req, mt_count_pmis(), sizeof(uint64_t), NULL, NULL);
360 case MT_RETROGRADE: {
361 uint64_t value = os_atomic_load_wide(&mt_retrograde, relaxed);
362 return sysctl_io_number(req, value, sizeof(mt_retrograde), NULL, NULL);
363 }
364 case MT_TASK_THREAD:
365 return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
366 case MT_DEBUG: {
367 int value = mt_debug;
368
369 int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
370 if (r) {
371 return r;
372 }
373 mt_debug = value;
374
375 return 0;
376 }
377 case MT_KDBG_TEST: {
378 if (req->newptr == USER_ADDR_NULL) {
379 return EINVAL;
380 }
381
382 int intrs_en = ml_set_interrupts_enabled(FALSE);
383 MT_KDBG_TMPCPU_START(0x3fff);
384 MT_KDBG_TMPCPU_END(0x3fff);
385
386 MT_KDBG_TMPTH_START(0x3fff);
387 MT_KDBG_TMPTH_END(0x3fff);
388 ml_set_interrupts_enabled(intrs_en);
389
390 return 0;
391 }
392 case MT_FIX_CPU_PERF: {
393 int intrs_en = ml_set_interrupts_enabled(FALSE);
394 mt_fixed_counts(start);
395 mt_fixed_counts(end);
396 ml_set_interrupts_enabled(intrs_en);
397
398 goto copyout_counts;
399 }
400 case MT_FIX_THREAD_PERF: {
401 int intrs_en = ml_set_interrupts_enabled(FALSE);
402 mt_cur_thread_fixed_counts(start);
403 mt_cur_thread_fixed_counts(end);
404 ml_set_interrupts_enabled(intrs_en);
405
406 goto copyout_counts;
407 }
408 case MT_FIX_TASK_PERF: {
409 int intrs_en = ml_set_interrupts_enabled(FALSE);
410 mt_cur_task_fixed_counts(start);
411 mt_cur_task_fixed_counts(end);
412 ml_set_interrupts_enabled(intrs_en);
413
414 goto copyout_counts;
415 }
416 default:
417 return ENOENT;
418 }
419
420 copyout_counts:
421
422 #ifdef MT_CORE_INSTRS
423 counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
424 #endif /* defined(MT_CORE_INSTRS) */
425 counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];
426
427 return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
428 }
429
430 SYSCTL_DECL(_kern_monotonic);
431 SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
432 "monotonic");
433
434 #define MT_SYSCTL(NAME, ARG, FLAGS, SIZE, SIZESTR, DESC) \
435 SYSCTL_PROC(_kern_monotonic, OID_AUTO, NAME, \
436 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | (FLAGS), \
437 (void *)(ARG), SIZE, mt_sysctl, SIZESTR, DESC)
438
439 MT_SYSCTL(supported, MT_SUPPORTED, 0, sizeof(int), "I",
440 "whether monotonic is supported");
441 MT_SYSCTL(debug, MT_DEBUG, CTLFLAG_MASKED, sizeof(int), "I",
442 "whether monotonic is printing debug messages");
443 MT_SYSCTL(pmis, MT_PMIS, 0, sizeof(uint64_t), "Q",
444 "number of PMIs seen");
445 MT_SYSCTL(retrograde_updates, MT_RETROGRADE, 0, sizeof(uint64_t), "Q",
446 "number of times a counter appeared to go backwards");
447 MT_SYSCTL(task_thread_counting, MT_TASK_THREAD, 0, sizeof(int), "I",
448 "whether task and thread counting is enabled");
449 MT_SYSCTL(kdebug_test, MT_KDBG_TEST, CTLFLAG_MASKED, sizeof(int), "O",
450 "whether task and thread counting is enabled");
451 MT_SYSCTL(fixed_cpu_perf, MT_FIX_CPU_PERF, CTLFLAG_MASKED,
452 sizeof(uint64_t) * 2, "O",
453 "overhead of accessing the current CPU's counters");
454 MT_SYSCTL(fixed_thread_perf, MT_FIX_THREAD_PERF, CTLFLAG_MASKED,
455 sizeof(uint64_t) * 2, "O",
456 "overhead of accessing the current thread's counters");
457 MT_SYSCTL(fixed_task_perf, MT_FIX_TASK_PERF, CTLFLAG_MASKED,
458 sizeof(uint64_t) * 2, "O",
459 "overhead of accessing the current task's counters");
460