1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/monotonic.h>
30 #include <kern/perfmon.h>
31 #include <machine/machine_routines.h>
32 #include <machine/monotonic.h>
33 #include <pexpert/pexpert.h>
34 #include <sys/param.h> /* NULL */
35 #include <sys/stat.h> /* dev_t */
36 #include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */
37 #include <sys/conf.h> /* must come after sys/stat.h */
38 #include <sys/resource_private.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysproto.h>
41 #include <sys/systm.h>
42 #include <sys/types.h>
43 #include <sys/monotonic.h>
44
45 static int mt_cdev_open(dev_t dev, int flags, int devtype, proc_t p);
46 static int mt_cdev_close(dev_t dev, int flags, int devtype, proc_t p);
47 static int mt_cdev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag,
48 proc_t p);
49
50 #define MT_NODE "monotonic"
51
52 static const struct cdevsw mt_cdevsw = {
53 .d_open = mt_cdev_open,
54 .d_close = mt_cdev_close,
55 .d_ioctl = mt_cdev_ioctl,
56
57 .d_read = eno_rdwrt, .d_write = eno_rdwrt, .d_stop = eno_stop,
58 .d_reset = eno_reset, .d_ttys = NULL, .d_select = eno_select,
59 .d_mmap = eno_mmap, .d_strategy = eno_strat, .d_type = 0
60 };
61
62 /*
63 * Written at initialization, read-only thereafter.
64 */
65 LCK_GRP_DECLARE(mt_lock_grp, MT_NODE);
66 static int mt_dev_major;
67
68 static mt_device_t
mt_get_device(dev_t devnum)69 mt_get_device(dev_t devnum)
70 {
71 return &mt_devices[minor(devnum)];
72 }
73
74 static void
mt_device_lock(mt_device_t dev)75 mt_device_lock(mt_device_t dev)
76 {
77 lck_mtx_lock(&dev->mtd_lock);
78 }
79
80 static void
mt_device_unlock(mt_device_t dev)81 mt_device_unlock(mt_device_t dev)
82 {
83 lck_mtx_unlock(&dev->mtd_lock);
84 }
85
86 static void
mt_device_assert_lock_held(__assert_only mt_device_t dev)87 mt_device_assert_lock_held(__assert_only mt_device_t dev)
88 {
89 LCK_MTX_ASSERT(&dev->mtd_lock, LCK_MTX_ASSERT_OWNED);
90 }
91
92 static void
mt_device_assert_inuse(__assert_only mt_device_t dev)93 mt_device_assert_inuse(__assert_only mt_device_t dev)
94 {
95 assert(dev->mtd_inuse == true);
96 }
97
98 int
mt_dev_init(void)99 mt_dev_init(void)
100 {
101 mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw);
102 if (mt_dev_major < 0) {
103 panic("monotonic: cdevsw_add failed: %d", mt_dev_major);
104 __builtin_unreachable();
105 }
106
107 for (int i = 0; i < MT_NDEVS; i++) {
108 if (mt_devices[i].mtd_init(&mt_devices[i])) {
109 continue;
110 }
111
112 assert(mt_devices[i].mtd_ncounters > 0);
113
114 dev_t dev = makedev(mt_dev_major, i);
115 void *node = devfs_make_node(dev, DEVFS_CHAR, UID_ROOT,
116 GID_WINDOWSERVER, 0666, MT_NODE "/%s",
117 mt_devices[i].mtd_name);
118 if (!node) {
119 panic("monotonic: devfs_make_node failed for '%s'",
120 mt_devices[i].mtd_name);
121 __builtin_unreachable();
122 }
123
124 lck_mtx_init(&mt_devices[i].mtd_lock, &mt_lock_grp, LCK_ATTR_NULL);
125 }
126
127 return 0;
128 }
129
130 static int
mt_cdev_open(dev_t devnum,__unused int flags,__unused int devtype,__unused proc_t p)131 mt_cdev_open(dev_t devnum, __unused int flags, __unused int devtype,
132 __unused proc_t p)
133 {
134 int error = 0;
135
136 mt_device_t dev = mt_get_device(devnum);
137 if (!perfmon_acquire(perfmon_upmu, "monotonic")) {
138 return EBUSY;
139 }
140 mt_device_lock(dev);
141 if (dev->mtd_inuse) {
142 error = EBUSY;
143 } else if (!mt_acquire_counters()) {
144 error = EBUSY;
145 } else {
146 dev->mtd_reset();
147 dev->mtd_inuse = true;
148 }
149 mt_device_unlock(dev);
150
151 return error;
152 }
153
154 static int
mt_cdev_close(dev_t devnum,__unused int flags,__unused int devtype,__unused struct proc * p)155 mt_cdev_close(dev_t devnum, __unused int flags, __unused int devtype,
156 __unused struct proc *p)
157 {
158 mt_device_t dev = mt_get_device(devnum);
159
160 perfmon_release(perfmon_upmu, "monotonic");
161
162 mt_device_lock(dev);
163 mt_device_assert_inuse(dev);
164 dev->mtd_inuse = false;
165 dev->mtd_reset();
166 mt_release_counters();
167 mt_device_unlock(dev);
168
169 return 0;
170 }
171
172 static int
mt_ctl_add(mt_device_t dev,user_addr_t uptr)173 mt_ctl_add(mt_device_t dev, user_addr_t uptr)
174 {
175 int error;
176 uint32_t ctr;
177 union monotonic_ctl_add ctl;
178
179 mt_device_assert_lock_held(dev);
180
181 error = copyin(uptr, &ctl, sizeof(ctl.in));
182 if (error) {
183 return error;
184 }
185
186 error = dev->mtd_add(&ctl.in.config, &ctr);
187 if (error) {
188 return error;
189 }
190
191 ctl.out.ctr = ctr;
192
193 error = copyout(&ctl, uptr, sizeof(ctl.out));
194 if (error) {
195 return error;
196 }
197
198 return 0;
199 }
200
201 static int
mt_ctl_counts(mt_device_t dev,user_addr_t uptr)202 mt_ctl_counts(mt_device_t dev, user_addr_t uptr)
203 {
204 int error;
205 union monotonic_ctl_counts ctl;
206
207 mt_device_assert_lock_held(dev);
208
209 error = copyin(uptr, &ctl, sizeof(ctl.in));
210 if (error) {
211 return error;
212 }
213
214 if (ctl.in.ctr_mask == 0) {
215 return EINVAL;
216 }
217
218 {
219 uint64_t counts[dev->mtd_nmonitors][dev->mtd_ncounters];
220 memset(counts, 0,
221 dev->mtd_ncounters * dev->mtd_nmonitors * sizeof(counts[0][0]));
222 error = dev->mtd_read(ctl.in.ctr_mask, (uint64_t *)counts);
223 if (error) {
224 return error;
225 }
226
227 error = copyout(&counts, uptr, sizeof(counts));
228 if (error) {
229 return error;
230 }
231 }
232
233 return 0;
234 }
235
236 static int
mt_ctl_enable(mt_device_t dev,user_addr_t uptr)237 mt_ctl_enable(mt_device_t dev, user_addr_t uptr)
238 {
239 int error;
240 union monotonic_ctl_enable ctl;
241
242 mt_device_assert_lock_held(dev);
243
244 error = copyin(uptr, &ctl, sizeof(ctl));
245 if (error) {
246 return error;
247 }
248
249 dev->mtd_enable(ctl.in.enable);
250
251 return 0;
252 }
253
254 static int
mt_ctl_reset(mt_device_t dev)255 mt_ctl_reset(mt_device_t dev)
256 {
257 mt_device_assert_lock_held(dev);
258 dev->mtd_reset();
259 return 0;
260 }
261
262 static int
mt_cdev_ioctl(dev_t devnum,unsigned long cmd,char * arg,__unused int flags,__unused proc_t p)263 mt_cdev_ioctl(dev_t devnum, unsigned long cmd, char *arg, __unused int flags,
264 __unused proc_t p)
265 {
266 int error = ENODEV;
267 user_addr_t uptr = *(user_addr_t *)(void *)arg;
268
269 mt_device_t dev = mt_get_device(devnum);
270 mt_device_lock(dev);
271
272 switch (cmd) {
273 case MT_IOC_RESET:
274 error = mt_ctl_reset(dev);
275 break;
276
277 case MT_IOC_ADD:
278 error = mt_ctl_add(dev, uptr);
279 break;
280
281 case MT_IOC_ENABLE:
282 error = mt_ctl_enable(dev, uptr);
283 break;
284
285 case MT_IOC_COUNTS:
286 error = mt_ctl_counts(dev, uptr);
287 break;
288
289 case MT_IOC_GET_INFO: {
290 union monotonic_ctl_info info = {
291 .out = {
292 .nmonitors = dev->mtd_nmonitors,
293 .ncounters = dev->mtd_ncounters,
294 },
295 };
296 error = copyout(&info, uptr, sizeof(info));
297 break;
298 }
299
300 default:
301 error = ENODEV;
302 break;
303 }
304
305 mt_device_unlock(dev);
306
307 return error;
308 }
309
310 int
thread_selfcounts(__unused struct proc * p,struct thread_selfcounts_args * uap,__unused int * ret_out)311 thread_selfcounts(__unused struct proc *p,
312 struct thread_selfcounts_args *uap, __unused int *ret_out)
313 {
314 switch (uap->kind) {
315 case THSC_CPI: {
316 struct thsc_cpi counts = { 0 };
317 uint64_t thread_counts[MT_CORE_NFIXED] = { 0 };
318
319 mt_cur_thread_fixed_counts(thread_counts);
320
321 #ifdef MT_CORE_INSTRS
322 counts.tcpi_instructions = thread_counts[MT_CORE_INSTRS];
323 #endif /* defined(MT_CORE_INSTRS) */
324 counts.tcpi_cycles = thread_counts[MT_CORE_CYCLES];
325
326 return copyout(&counts, uap->buf, MIN(sizeof(counts), uap->nbytes));
327 }
328 default:
329 return EINVAL;
330 }
331 }
332
333 enum mt_sysctl {
334 MT_SUPPORTED,
335 MT_PMIS,
336 MT_RETROGRADE,
337 MT_TASK_THREAD,
338 MT_DEBUG,
339 MT_KDBG_TEST,
340 MT_FIX_CPU_PERF,
341 MT_FIX_THREAD_PERF,
342 MT_FIX_TASK_PERF,
343 };
344
345 static int
346 mt_sysctl SYSCTL_HANDLER_ARGS
347 {
348 #pragma unused(oidp, arg2)
349 uint64_t start[MT_CORE_NFIXED] = { 0 }, end[MT_CORE_NFIXED] = { 0 };
350 uint64_t counts[2] = { 0 };
351
352 switch ((enum mt_sysctl)arg1) {
353 case MT_SUPPORTED:
354 return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
355 case MT_PMIS:
356 return sysctl_io_number(req, mt_count_pmis(), sizeof(uint64_t), NULL, NULL);
357 case MT_RETROGRADE: {
358 uint64_t value = os_atomic_load_wide(&mt_retrograde, relaxed);
359 return sysctl_io_number(req, value, sizeof(mt_retrograde), NULL, NULL);
360 }
361 case MT_TASK_THREAD:
362 return sysctl_io_number(req, (int)mt_core_supported, sizeof(int), NULL, NULL);
363 case MT_DEBUG: {
364 int value = mt_debug;
365
366 int r = sysctl_io_number(req, value, sizeof(value), &value, NULL);
367 if (r) {
368 return r;
369 }
370 mt_debug = value;
371
372 return 0;
373 }
374 case MT_KDBG_TEST: {
375 if (req->newptr == USER_ADDR_NULL) {
376 return EINVAL;
377 }
378
379 int intrs_en = ml_set_interrupts_enabled(FALSE);
380 MT_KDBG_TMPCPU_START(0x3fff);
381 MT_KDBG_TMPCPU_END(0x3fff);
382
383 MT_KDBG_TMPTH_START(0x3fff);
384 MT_KDBG_TMPTH_END(0x3fff);
385 ml_set_interrupts_enabled(intrs_en);
386
387 return 0;
388 }
389 case MT_FIX_CPU_PERF: {
390 int intrs_en = ml_set_interrupts_enabled(FALSE);
391 mt_fixed_counts(start);
392 mt_fixed_counts(end);
393 ml_set_interrupts_enabled(intrs_en);
394
395 goto copyout_counts;
396 }
397 case MT_FIX_THREAD_PERF: {
398 int intrs_en = ml_set_interrupts_enabled(FALSE);
399 mt_cur_thread_fixed_counts(start);
400 mt_cur_thread_fixed_counts(end);
401 ml_set_interrupts_enabled(intrs_en);
402
403 goto copyout_counts;
404 }
405 case MT_FIX_TASK_PERF: {
406 int intrs_en = ml_set_interrupts_enabled(FALSE);
407 mt_cur_task_fixed_counts(start);
408 mt_cur_task_fixed_counts(end);
409 ml_set_interrupts_enabled(intrs_en);
410
411 goto copyout_counts;
412 }
413 default:
414 return ENOENT;
415 }
416
417 copyout_counts:
418
419 #ifdef MT_CORE_INSTRS
420 counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS];
421 #endif /* defined(MT_CORE_INSTRS) */
422 counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES];
423
424 return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts)));
425 }
426
427 SYSCTL_DECL(_kern_monotonic);
428 SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
429 "monotonic");
430
431 #define MT_SYSCTL(NAME, ARG, FLAGS, SIZE, SIZESTR, DESC) \
432 SYSCTL_PROC(_kern_monotonic, OID_AUTO, NAME, \
433 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | (FLAGS), \
434 (void *)(ARG), SIZE, mt_sysctl, SIZESTR, DESC)
435
436 MT_SYSCTL(supported, MT_SUPPORTED, 0, sizeof(int), "I",
437 "whether monotonic is supported");
438 MT_SYSCTL(debug, MT_DEBUG, CTLFLAG_MASKED, sizeof(int), "I",
439 "whether monotonic is printing debug messages");
440 MT_SYSCTL(pmis, MT_PMIS, 0, sizeof(uint64_t), "Q",
441 "number of PMIs seen");
442 MT_SYSCTL(retrograde_updates, MT_RETROGRADE, 0, sizeof(uint64_t), "Q",
443 "number of times a counter appeared to go backwards");
444 MT_SYSCTL(task_thread_counting, MT_TASK_THREAD, 0, sizeof(int), "I",
445 "whether task and thread counting is enabled");
446 MT_SYSCTL(kdebug_test, MT_KDBG_TEST, CTLFLAG_MASKED, sizeof(int), "O",
447 "whether task and thread counting is enabled");
448 MT_SYSCTL(fixed_cpu_perf, MT_FIX_CPU_PERF, CTLFLAG_MASKED,
449 sizeof(uint64_t) * 2, "O",
450 "overhead of accessing the current CPU's counters");
451 MT_SYSCTL(fixed_thread_perf, MT_FIX_THREAD_PERF, CTLFLAG_MASKED,
452 sizeof(uint64_t) * 2, "O",
453 "overhead of accessing the current thread's counters");
454 MT_SYSCTL(fixed_task_perf, MT_FIX_TASK_PERF, CTLFLAG_MASKED,
455 sizeof(uint64_t) * 2, "O",
456 "overhead of accessing the current task's counters");
457