1 /*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/debug.h>
30 #include <kern/kalloc.h>
31 #include <kern/perfmon.h>
32 #include <sys/param.h>
33 #include <sys/mman.h>
34 #include <sys/stat.h>
35 #include <sys/sysctl.h>
36 #include <libkern/libkern.h>
37 #include <kern/assert.h>
38
39 #include <kern/kpc.h>
40 #include <sys/ktrace.h>
41
42 #include <pexpert/pexpert.h>
43 #include <kperf/kperf.h>
44
45 /* Various sysctl requests */
46 #define REQ_CLASSES (1)
47 #define REQ_COUNTING (2)
48 #define REQ_THREAD_COUNTING (3)
49 #define REQ_CONFIG_COUNT (4)
50 #define REQ_COUNTER_COUNT (5)
51 #define REQ_THREAD_COUNTERS (6)
52 #define REQ_COUNTERS (7)
53 #define REQ_SHADOW_COUNTERS (8)
54 #define REQ_CONFIG (9)
55 #define REQ_PERIOD (10)
56 #define REQ_ACTIONID (11)
57 #define REQ_SW_INC (14)
58 #define REQ_PMU_VERSION (15)
59
60 /* Type-munging casts */
61 typedef int (*getint_t)(void);
62 typedef int (*setint_t)(int);
63
64 static int kpc_initted = 0;
65
66 static LCK_GRP_DECLARE(sysctl_lckgrp, "kpc");
67 static LCK_MTX_DECLARE(sysctl_lock, &sysctl_lckgrp);
68
69 /*
70 * Another element is needed to hold the CPU number when getting counter values.
71 */
72 #define KPC_MAX_BUF_LEN (KPC_MAX_COUNTERS_COPIED + 1)
73
74 typedef int (*setget_func_t)(int);
75
76 void
kpc_init(void)77 kpc_init(void)
78 {
79 kpc_arch_init();
80
81 kpc_initted = 1;
82 }
83
84 static uint64_t *
kpc_get_bigarray(uint32_t * size_out)85 kpc_get_bigarray(uint32_t *size_out)
86 {
87 static uint64_t *bigarray = NULL;
88
89 LCK_MTX_ASSERT(&sysctl_lock, LCK_MTX_ASSERT_OWNED);
90
91 uint32_t size = kpc_get_counterbuf_size() + sizeof(uint64_t);
92 *size_out = size;
93
94 if (bigarray) {
95 return bigarray;
96 }
97
98 /*
99 * Another element is needed to hold the CPU number when getting counter
100 * values.
101 */
102 bigarray = kalloc_data_tag(size, Z_WAITOK, VM_KERN_MEMORY_DIAG);
103 assert(bigarray != NULL);
104 return bigarray;
105 }
106
107 /* abstract sysctl handlers */
108 static int
sysctl_get_int(struct sysctl_oid * oidp,struct sysctl_req * req,uint32_t value)109 sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req,
110 uint32_t value )
111 {
112 int error = 0;
113
114 /* copy out the old value */
115 error = sysctl_handle_int(oidp, &value, 0, req);
116
117 return error;
118 }
119
120 static int
sysctl_set_int(struct sysctl_req * req,int (* set_func)(int))121 sysctl_set_int( struct sysctl_req *req, int (*set_func)(int))
122 {
123 int error = 0;
124 int value = 0;
125
126 error = SYSCTL_IN( req, &value, sizeof(value));
127 if (error) {
128 return error;
129 }
130
131 error = set_func( value );
132
133 return error;
134 }
135
136 static int
sysctl_getset_int(struct sysctl_oid * oidp,struct sysctl_req * req,int (* get_func)(void),int (* set_func)(int))137 sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req,
138 int (*get_func)(void), int (*set_func)(int))
139 {
140 int error = 0;
141 uint32_t value = 0;
142
143 /* get the old value and process it */
144 value = get_func();
145
146 /* copy out the old value, get the new value */
147 error = sysctl_handle_int(oidp, &value, 0, req);
148 if (error || !req->newptr) {
149 return error;
150 }
151
152 /* if that worked, and we're writing... */
153 error = set_func( value );
154
155 return error;
156 }
157
158
159 static int
sysctl_setget_int(struct sysctl_req * req,int (* setget_func)(int))160 sysctl_setget_int( struct sysctl_req *req,
161 int (*setget_func)(int))
162 {
163 int error = 0;
164 int value = 0;
165
166 error = SYSCTL_IN( req, &value, sizeof(value));
167 if (error) {
168 return error;
169 }
170
171 value = setget_func(value);
172
173 error = SYSCTL_OUT( req, &value, sizeof(value));
174
175 return error;
176 }
177
178 static int
sysctl_kpc_get_counters(uint32_t counters,uint32_t * size,void * buf)179 sysctl_kpc_get_counters(uint32_t counters,
180 uint32_t *size, void *buf)
181 {
182 uint64_t *ctr_buf = (uint64_t*)buf;
183 int curcpu;
184 uint32_t count;
185
186 count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS,
187 counters,
188 &curcpu, &ctr_buf[1]);
189 if (!count) {
190 return EINVAL;
191 }
192
193 ctr_buf[0] = curcpu;
194
195 *size = (count + 1) * sizeof(uint64_t);
196
197 return 0;
198 }
199
200 static int
sysctl_kpc_get_shadow_counters(uint32_t counters,uint32_t * size,void * buf)201 sysctl_kpc_get_shadow_counters(uint32_t counters,
202 uint32_t *size, void *buf)
203 {
204 uint64_t *ctr_buf = (uint64_t*)buf;
205 int curcpu;
206 uint32_t count;
207
208 count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS,
209 counters,
210 &curcpu, &ctr_buf[1]);
211
212 if (!count) {
213 return EINVAL;
214 }
215
216 ctr_buf[0] = curcpu;
217
218 *size = (count + 1) * sizeof(uint64_t);
219
220 return 0;
221 }
222
223 static int
sysctl_kpc_get_thread_counters(uint32_t tid,uint32_t * size,void * buf)224 sysctl_kpc_get_thread_counters(uint32_t tid,
225 uint32_t *size, void *buf)
226 {
227 uint32_t count = *size / sizeof(uint64_t);
228 int r;
229
230 if (tid != 0) {
231 return EINVAL;
232 }
233
234 r = kpc_get_curthread_counters(&count, buf);
235 if (!r) {
236 *size = count * sizeof(uint64_t);
237 }
238
239 return r;
240 }
241
242 static int
sysctl_kpc_get_config(uint32_t classes,void * buf)243 sysctl_kpc_get_config(uint32_t classes, void* buf)
244 {
245 return kpc_get_config( classes, buf );
246 }
247
248 static int
sysctl_kpc_set_config(uint32_t classes,void * buf)249 sysctl_kpc_set_config(uint32_t classes, void* buf)
250 {
251 /* userspace cannot reconfigure the power class */
252 if (classes & KPC_CLASS_POWER_MASK) {
253 return EPERM;
254 }
255 return kpc_set_config_kernel(classes, buf);
256 }
257
258 static int
sysctl_kpc_get_period(uint32_t classes,void * buf)259 sysctl_kpc_get_period(uint32_t classes, void* buf)
260 {
261 return kpc_get_period( classes, buf );
262 }
263
264 static int
sysctl_kpc_set_period(uint32_t classes,void * buf)265 sysctl_kpc_set_period(uint32_t classes, void* buf)
266 {
267 /* userspace cannot reconfigure the power class */
268 if (classes & KPC_CLASS_POWER_MASK) {
269 return EPERM;
270 }
271 return kpc_set_period( classes, buf);
272 }
273
274 static int
sysctl_kpc_get_actionid(uint32_t classes,void * buf)275 sysctl_kpc_get_actionid(uint32_t classes, void* buf)
276 {
277 return kpc_get_actionid( classes, buf );
278 }
279
280 static int
sysctl_kpc_set_actionid(uint32_t classes,void * buf)281 sysctl_kpc_set_actionid(uint32_t classes, void* buf)
282 {
283 return kpc_set_actionid( classes, buf);
284 }
285
286
287 static int
sysctl_get_bigarray(struct sysctl_req * req,int (* get_fn)(uint32_t,uint32_t *,void *))288 sysctl_get_bigarray(struct sysctl_req *req,
289 int (*get_fn)(uint32_t, uint32_t*, void*))
290 {
291 uint32_t bufsize = 0;
292 uint64_t *buf = kpc_get_bigarray(&bufsize);
293 uint32_t arg = 0;
294
295 /* get the argument */
296 int error = SYSCTL_IN(req, &arg, sizeof(arg));
297 if (error) {
298 return error;
299 }
300
301 error = get_fn(arg, &bufsize, buf);
302 if (!error) {
303 error = SYSCTL_OUT(req, buf, bufsize);
304 }
305
306 return error;
307 }
308
309 /* given a config word, how many bytes does it take? */
310 static int
sysctl_config_size(uint32_t config)311 sysctl_config_size( uint32_t config )
312 {
313 return kpc_get_config_count(config) * sizeof(kpc_config_t);
314 }
315
316 static int
sysctl_counter_size(uint32_t classes)317 sysctl_counter_size( uint32_t classes )
318 {
319 return kpc_get_counter_count(classes) * sizeof(uint64_t);
320 }
321
322 static int
sysctl_actionid_size(uint32_t classes)323 sysctl_actionid_size( uint32_t classes )
324 {
325 return kpc_get_counter_count(classes) * sizeof(int32_t);
326 }
327
328 static int
sysctl_getset_bigarray(struct sysctl_req * req,int (* size_fn)(uint32_t arg),int (* get_fn)(uint32_t,void *),int (* set_fn)(uint32_t,void *))329 sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg),
330 int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*))
331 {
332 int error = 0;
333 uint64_t arg;
334
335 uint32_t bufsize = 0;
336 uint64_t *buf = kpc_get_bigarray(&bufsize);
337
338 /* get the config word */
339 error = SYSCTL_IN(req, &arg, sizeof(arg));
340 if (error) {
341 return error;
342 }
343
344 /* Determine the size of registers to modify. */
345 uint32_t regsize = size_fn((uint32_t)arg);
346 if (regsize == 0 || regsize > bufsize) {
347 return EINVAL;
348 }
349
350 /* if writing */
351 if (req->newptr) {
352 /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */
353 error = SYSCTL_IN(req, buf, regsize);
354
355 /* SYSCTL_IN failure means only need to read */
356 if (!error) {
357 error = set_fn((uint32_t)arg, buf);
358 if (error) {
359 return error;
360 }
361 }
362 }
363
364 /* if reading */
365 if (req->oldptr) {
366 error = get_fn((uint32_t)arg, buf);
367 if (error) {
368 return error;
369 }
370
371 error = SYSCTL_OUT(req, buf, regsize);
372 }
373
374 return error;
375 }
376
377 static int
378 kpc_sysctl SYSCTL_HANDLER_ARGS
379 {
380 int ret;
381
382 // __unused struct sysctl_oid *unused_oidp = oidp;
383 (void)arg2;
384
385 if (!kpc_initted) {
386 panic("kpc_init not called");
387 }
388
389 if (!kpc_supported) {
390 return ENOTSUP;
391 }
392
393 ktrace_lock();
394
395 // Most sysctls require an access check, but a few are public.
396 switch ((uintptr_t) arg1) {
397 case REQ_CLASSES:
398 case REQ_CONFIG_COUNT:
399 case REQ_COUNTER_COUNT:
400 // These read-only sysctls are public.
401 break;
402
403 default:
404 // Require kperf access to read or write anything else.
405 // This is either root or the blessed pid.
406 if ((ret = ktrace_read_check())) {
407 ktrace_unlock();
408 return ret;
409 }
410 break;
411 }
412
413 ktrace_unlock();
414
415 if (perfmon_in_use(perfmon_cpmu)) {
416 return EBUSY;
417 }
418
419 lck_mtx_lock(&sysctl_lock);
420
421 /* which request */
422 switch ((uintptr_t) arg1) {
423 case REQ_CLASSES:
424 ret = sysctl_get_int( oidp, req,
425 kpc_get_classes());
426 break;
427 case REQ_COUNTING:
428 ret = sysctl_getset_int( oidp, req,
429 (getint_t)kpc_get_running,
430 (setint_t)kpc_set_running );
431 break;
432 case REQ_THREAD_COUNTING:
433 ret = sysctl_getset_int( oidp, req,
434 (getint_t)kpc_get_thread_counting,
435 (setint_t)kpc_set_thread_counting );
436 break;
437
438 case REQ_CONFIG_COUNT:
439 ret = sysctl_setget_int( req,
440 (setget_func_t)kpc_get_config_count );
441 break;
442
443 case REQ_COUNTER_COUNT:
444 ret = sysctl_setget_int( req,
445 (setget_func_t)kpc_get_counter_count );
446 break;
447
448 case REQ_THREAD_COUNTERS:
449 ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters );
450 break;
451
452 case REQ_COUNTERS:
453 ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters );
454 break;
455
456 case REQ_SHADOW_COUNTERS:
457 ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters );
458 break;
459
460 case REQ_CONFIG:
461 ret = sysctl_getset_bigarray( req,
462 sysctl_config_size,
463 sysctl_kpc_get_config,
464 sysctl_kpc_set_config );
465 break;
466
467 case REQ_PERIOD:
468 ret = sysctl_getset_bigarray( req,
469 sysctl_counter_size,
470 sysctl_kpc_get_period,
471 sysctl_kpc_set_period );
472 break;
473
474 case REQ_ACTIONID:
475 ret = sysctl_getset_bigarray( req,
476 sysctl_actionid_size,
477 sysctl_kpc_get_actionid,
478 sysctl_kpc_set_actionid );
479 break;
480
481
482 case REQ_SW_INC:
483 ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc );
484 break;
485
486 case REQ_PMU_VERSION:
487 ret = sysctl_get_int(oidp, req, kpc_get_pmu_version());
488 break;
489
490 default:
491 ret = ENOENT;
492 break;
493 }
494
495 lck_mtx_unlock(&sysctl_lock);
496
497 return ret;
498 }
499
500
501 /*** sysctl definitions ***/
502
503 /* root kperf node */
504 SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
505 "kpc");
506
507 /* values */
508 SYSCTL_PROC(_kpc, OID_AUTO, classes,
509 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
510 (void*)REQ_CLASSES,
511 sizeof(int), kpc_sysctl, "I", "Available classes");
512
513 SYSCTL_PROC(_kpc, OID_AUTO, counting,
514 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
515 (void*)REQ_COUNTING,
516 sizeof(int), kpc_sysctl, "I", "PMCs counting");
517
518 SYSCTL_PROC(_kpc, OID_AUTO, thread_counting,
519 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
520 (void*)REQ_THREAD_COUNTING,
521 sizeof(int), kpc_sysctl, "I", "Thread accumulation");
522
523 SYSCTL_PROC(_kpc, OID_AUTO, pmu_version,
524 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
525 (void *)REQ_PMU_VERSION,
526 sizeof(int), kpc_sysctl, "I", "PMU version for hardware");
527
528 /* faux values */
529 SYSCTL_PROC(_kpc, OID_AUTO, config_count,
530 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
531 (void*)REQ_CONFIG_COUNT,
532 sizeof(int), kpc_sysctl, "S", "Config count");
533
534 SYSCTL_PROC(_kpc, OID_AUTO, counter_count,
535 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
536 (void*)REQ_COUNTER_COUNT,
537 sizeof(int), kpc_sysctl, "S", "Counter count");
538
539 SYSCTL_PROC(_kpc, OID_AUTO, sw_inc,
540 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
541 (void*)REQ_SW_INC,
542 sizeof(int), kpc_sysctl, "S", "Software increment");
543
544 /* arrays */
545 SYSCTL_PROC(_kpc, OID_AUTO, thread_counters,
546 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
547 (void*)REQ_THREAD_COUNTERS,
548 sizeof(uint64_t), kpc_sysctl,
549 "QU", "Current thread counters");
550
551 SYSCTL_PROC(_kpc, OID_AUTO, counters,
552 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
553 (void*)REQ_COUNTERS,
554 sizeof(uint64_t), kpc_sysctl,
555 "QU", "Current counters");
556
557 SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters,
558 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
559 (void*)REQ_SHADOW_COUNTERS,
560 sizeof(uint64_t), kpc_sysctl,
561 "QU", "Current shadow counters");
562
563 SYSCTL_PROC(_kpc, OID_AUTO, config,
564 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
565 (void*)REQ_CONFIG,
566 sizeof(uint64_t), kpc_sysctl,
567 "QU", "Set counter configs");
568
569 SYSCTL_PROC(_kpc, OID_AUTO, period,
570 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
571 (void*)REQ_PERIOD,
572 sizeof(uint64_t), kpc_sysctl,
573 "QU", "Set counter periods");
574
575 SYSCTL_PROC(_kpc, OID_AUTO, actionid,
576 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
577 (void*)REQ_ACTIONID,
578 sizeof(uint32_t), kpc_sysctl,
579 "QU", "Set counter actionids");
580
581
582
583 #ifdef __arm64__
584
585 extern int kpc_pc_capture;
586 SYSCTL_INT(_kpc, OID_AUTO, pc_capture_supported,
587 CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, &kpc_pc_capture, 0,
588 "whether PC capture is supported by the hardware");
589
590 #endif /* __arm64__ */
591