1 /*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/debug.h>
30 #include <kern/kalloc.h>
31 #include <sys/param.h>
32 #include <sys/mman.h>
33 #include <sys/stat.h>
34 #include <sys/sysctl.h>
35 #include <libkern/libkern.h>
36 #include <kern/assert.h>
37 #include <kern/cpc.h>
38
39 #include <kern/kpc.h>
40 #include <sys/ktrace.h>
41
42 #include <pexpert/pexpert.h>
43 #include <kperf/kperf.h>
44
45 #if CONFIG_CPU_COUNTERS
46
47 /* Various sysctl requests */
48 #define REQ_CLASSES (1)
49 #define REQ_COUNTING (2)
50 #define REQ_THREAD_COUNTING (3)
51 #define REQ_CONFIG_COUNT (4)
52 #define REQ_COUNTER_COUNT (5)
53 #define REQ_THREAD_COUNTERS (6)
54 #define REQ_COUNTERS (7)
55 #define REQ_SHADOW_COUNTERS (8)
56 #define REQ_CONFIG (9)
57 #define REQ_PERIOD (10)
58 #define REQ_ACTIONID (11)
59 #define REQ_SW_INC (14)
60 #define REQ_PMU_VERSION (15)
61
62 /* Type-munging casts */
63 typedef int (*getint_t)(void);
64 typedef int (*setint_t)(int);
65
66 static int kpc_initted = 0;
67
68 static LCK_GRP_DECLARE(sysctl_lckgrp, "kpc");
69 static LCK_MTX_DECLARE(sysctl_lock, &sysctl_lckgrp);
70
71 /*
72 * Another element is needed to hold the CPU number when getting counter values.
73 */
74 #define KPC_MAX_BUF_LEN (KPC_MAX_COUNTERS_COPIED + 1)
75
76 typedef int (*setget_func_t)(int);
77
78 void
kpc_init(void)79 kpc_init(void)
80 {
81 kpc_arch_init();
82
83 kpc_initted = 1;
84 }
85
86 static uint64_t *
kpc_get_bigarray(uint32_t * size_out)87 kpc_get_bigarray(uint32_t *size_out)
88 {
89 static uint64_t *bigarray = NULL;
90
91 LCK_MTX_ASSERT(&sysctl_lock, LCK_MTX_ASSERT_OWNED);
92
93 uint32_t size = kpc_get_counterbuf_size() + sizeof(uint64_t);
94 *size_out = size;
95
96 if (bigarray) {
97 return bigarray;
98 }
99
100 /*
101 * Another element is needed to hold the CPU number when getting counter
102 * values.
103 */
104 bigarray = kalloc_data_tag(size, Z_WAITOK, VM_KERN_MEMORY_DIAG);
105 assert(bigarray != NULL);
106 return bigarray;
107 }
108
109 /* abstract sysctl handlers */
110 static int
sysctl_get_int(struct sysctl_oid * oidp,struct sysctl_req * req,uint32_t value)111 sysctl_get_int( struct sysctl_oid *oidp, struct sysctl_req *req,
112 uint32_t value )
113 {
114 int error = 0;
115
116 /* copy out the old value */
117 error = sysctl_handle_int(oidp, &value, 0, req);
118
119 return error;
120 }
121
122 static int
sysctl_set_int(struct sysctl_req * req,int (* set_func)(int))123 sysctl_set_int( struct sysctl_req *req, int (*set_func)(int))
124 {
125 int error = 0;
126 int value = 0;
127
128 error = SYSCTL_IN( req, &value, sizeof(value));
129 if (error) {
130 return error;
131 }
132
133 error = set_func( value );
134
135 return error;
136 }
137
138 static int
sysctl_getset_int(struct sysctl_oid * oidp,struct sysctl_req * req,int (* get_func)(void),int (* set_func)(int))139 sysctl_getset_int( struct sysctl_oid *oidp, struct sysctl_req *req,
140 int (*get_func)(void), int (*set_func)(int))
141 {
142 int error = 0;
143 uint32_t value = 0;
144
145 /* get the old value and process it */
146 value = get_func();
147
148 /* copy out the old value, get the new value */
149 error = sysctl_handle_int(oidp, &value, 0, req);
150 if (error || !req->newptr) {
151 return error;
152 }
153
154 /* if that worked, and we're writing... */
155 error = set_func( value );
156
157 return error;
158 }
159
160
161 static int
sysctl_setget_int(struct sysctl_req * req,int (* setget_func)(int))162 sysctl_setget_int( struct sysctl_req *req,
163 int (*setget_func)(int))
164 {
165 int error = 0;
166 int value = 0;
167
168 error = SYSCTL_IN( req, &value, sizeof(value));
169 if (error) {
170 return error;
171 }
172
173 value = setget_func(value);
174
175 error = SYSCTL_OUT( req, &value, sizeof(value));
176
177 return error;
178 }
179
180 static int
sysctl_kpc_get_counters(uint32_t counters,uint32_t * size,void * buf)181 sysctl_kpc_get_counters(uint32_t counters,
182 uint32_t *size, void *buf)
183 {
184 uint64_t *ctr_buf = (uint64_t*)buf;
185 int curcpu;
186 uint32_t count;
187
188 count = kpc_get_cpu_counters(counters & KPC_ALL_CPUS,
189 counters,
190 &curcpu, &ctr_buf[1]);
191 if (!count) {
192 return EINVAL;
193 }
194
195 ctr_buf[0] = curcpu;
196
197 *size = (count + 1) * sizeof(uint64_t);
198
199 return 0;
200 }
201
202 static int
sysctl_kpc_get_shadow_counters(uint32_t counters,uint32_t * size,void * buf)203 sysctl_kpc_get_shadow_counters(uint32_t counters,
204 uint32_t *size, void *buf)
205 {
206 uint64_t *ctr_buf = (uint64_t*)buf;
207 int curcpu;
208 uint32_t count;
209
210 count = kpc_get_shadow_counters(counters & KPC_ALL_CPUS,
211 counters,
212 &curcpu, &ctr_buf[1]);
213
214 if (!count) {
215 return EINVAL;
216 }
217
218 ctr_buf[0] = curcpu;
219
220 *size = (count + 1) * sizeof(uint64_t);
221
222 return 0;
223 }
224
225 static int
sysctl_kpc_get_thread_counters(uint32_t tid,uint32_t * size,void * buf)226 sysctl_kpc_get_thread_counters(uint32_t tid,
227 uint32_t *size, void *buf)
228 {
229 uint32_t count = *size / sizeof(uint64_t);
230 int r;
231
232 if (tid != 0) {
233 return EINVAL;
234 }
235
236 r = kpc_get_curthread_counters(&count, buf);
237 if (!r) {
238 *size = count * sizeof(uint64_t);
239 }
240
241 return r;
242 }
243
244 static int
sysctl_kpc_get_config(uint32_t classes,void * buf)245 sysctl_kpc_get_config(uint32_t classes, void* buf)
246 {
247 return kpc_get_config( classes, buf );
248 }
249
250 static int
sysctl_kpc_set_config(uint32_t classes,void * buf)251 sysctl_kpc_set_config(uint32_t classes, void* buf)
252 {
253 /* userspace cannot reconfigure the power class */
254 if (classes & KPC_CLASS_POWER_MASK) {
255 return EPERM;
256 }
257 return kpc_set_config_kernel(classes, buf);
258 }
259
260 static int
sysctl_kpc_get_period(uint32_t classes,void * buf)261 sysctl_kpc_get_period(uint32_t classes, void* buf)
262 {
263 return kpc_get_period( classes, buf );
264 }
265
266 static int
sysctl_kpc_set_period(uint32_t classes,void * buf)267 sysctl_kpc_set_period(uint32_t classes, void* buf)
268 {
269 /* userspace cannot reconfigure the power class */
270 if (classes & KPC_CLASS_POWER_MASK) {
271 return EPERM;
272 }
273 return kpc_set_period( classes, buf);
274 }
275
276 static int
sysctl_kpc_get_actionid(uint32_t classes,void * buf)277 sysctl_kpc_get_actionid(uint32_t classes, void* buf)
278 {
279 return kpc_get_actionid( classes, buf );
280 }
281
282 static int
sysctl_kpc_set_actionid(uint32_t classes,void * buf)283 sysctl_kpc_set_actionid(uint32_t classes, void* buf)
284 {
285 return kpc_set_actionid( classes, buf);
286 }
287
288
289 static int
sysctl_get_bigarray(struct sysctl_req * req,int (* get_fn)(uint32_t,uint32_t *,void *))290 sysctl_get_bigarray(struct sysctl_req *req,
291 int (*get_fn)(uint32_t, uint32_t*, void*))
292 {
293 uint32_t bufsize = 0;
294 uint64_t *buf = kpc_get_bigarray(&bufsize);
295 uint32_t arg = 0;
296
297 /* get the argument */
298 int error = SYSCTL_IN(req, &arg, sizeof(arg));
299 if (error) {
300 return error;
301 }
302
303 error = get_fn(arg, &bufsize, buf);
304 if (!error) {
305 error = SYSCTL_OUT(req, buf, bufsize);
306 }
307
308 return error;
309 }
310
311 /* given a config word, how many bytes does it take? */
312 static int
sysctl_config_size(uint32_t config)313 sysctl_config_size( uint32_t config )
314 {
315 return kpc_get_config_count(config) * sizeof(kpc_config_t);
316 }
317
318 static int
sysctl_counter_size(uint32_t classes)319 sysctl_counter_size( uint32_t classes )
320 {
321 return kpc_get_counter_count(classes) * sizeof(uint64_t);
322 }
323
324 static int
sysctl_actionid_size(uint32_t classes)325 sysctl_actionid_size( uint32_t classes )
326 {
327 return kpc_get_counter_count(classes) * sizeof(int32_t);
328 }
329
330 static int
sysctl_getset_bigarray(struct sysctl_req * req,int (* size_fn)(uint32_t arg),int (* get_fn)(uint32_t,void *),int (* set_fn)(uint32_t,void *))331 sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg),
332 int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*))
333 {
334 int error = 0;
335 uint64_t arg;
336
337 uint32_t bufsize = 0;
338 uint64_t *buf = kpc_get_bigarray(&bufsize);
339
340 /* get the config word */
341 error = SYSCTL_IN(req, &arg, sizeof(arg));
342 if (error) {
343 return error;
344 }
345
346 /* Determine the size of registers to modify. */
347 uint32_t regsize = size_fn((uint32_t)arg);
348 if (regsize == 0 || regsize > bufsize) {
349 return EINVAL;
350 }
351
352 /* if writing */
353 if (req->newptr) {
354 /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */
355 error = SYSCTL_IN(req, buf, regsize);
356
357 /* SYSCTL_IN failure means only need to read */
358 if (!error) {
359 error = set_fn((uint32_t)arg, buf);
360 if (error) {
361 return error;
362 }
363 }
364 }
365
366 /* if reading */
367 if (req->oldptr) {
368 error = get_fn((uint32_t)arg, buf);
369 if (error) {
370 return error;
371 }
372
373 error = SYSCTL_OUT(req, buf, regsize);
374 }
375
376 return error;
377 }
378
379 static int
380 kpc_sysctl SYSCTL_HANDLER_ARGS
381 {
382 int ret;
383
384 // __unused struct sysctl_oid *unused_oidp = oidp;
385 (void)arg2;
386
387 if (!kpc_initted) {
388 panic("kpc_init not called");
389 }
390
391 if (!kpc_supported) {
392 return ENOTSUP;
393 }
394
395 ktrace_lock();
396
397 // Most sysctls require an access check, but a few are public.
398 switch ((uintptr_t) arg1) {
399 case REQ_CLASSES:
400 case REQ_CONFIG_COUNT:
401 case REQ_COUNTER_COUNT:
402 // These read-only sysctls are public.
403 break;
404
405 default:
406 // Require kperf access to read or write anything else.
407 // This is either root or the blessed pid.
408 if ((ret = ktrace_read_check())) {
409 ktrace_unlock();
410 return ret;
411 }
412 break;
413 }
414
415 ktrace_unlock();
416
417 if (cpc_hw_in_use(CPC_HW_CPMU)) {
418 return EBUSY;
419 }
420
421 lck_mtx_lock(&sysctl_lock);
422
423 /* which request */
424 switch ((uintptr_t) arg1) {
425 case REQ_CLASSES:
426 ret = sysctl_get_int( oidp, req,
427 kpc_get_classes());
428 break;
429 case REQ_COUNTING:
430 ret = sysctl_getset_int( oidp, req,
431 (getint_t)kpc_get_running,
432 (setint_t)kpc_set_running );
433 break;
434 case REQ_THREAD_COUNTING:
435 ret = sysctl_getset_int( oidp, req,
436 (getint_t)kpc_get_thread_counting,
437 (setint_t)kpc_set_thread_counting );
438 break;
439
440 case REQ_CONFIG_COUNT:
441 ret = sysctl_setget_int( req,
442 (setget_func_t)kpc_get_config_count );
443 break;
444
445 case REQ_COUNTER_COUNT:
446 ret = sysctl_setget_int( req,
447 (setget_func_t)kpc_get_counter_count );
448 break;
449
450 case REQ_THREAD_COUNTERS:
451 ret = sysctl_get_bigarray( req, sysctl_kpc_get_thread_counters );
452 break;
453
454 case REQ_COUNTERS:
455 ret = sysctl_get_bigarray( req, sysctl_kpc_get_counters );
456 break;
457
458 case REQ_SHADOW_COUNTERS:
459 ret = sysctl_get_bigarray( req, sysctl_kpc_get_shadow_counters );
460 break;
461
462 case REQ_CONFIG:
463 ret = sysctl_getset_bigarray( req,
464 sysctl_config_size,
465 sysctl_kpc_get_config,
466 sysctl_kpc_set_config );
467 break;
468
469 case REQ_PERIOD:
470 ret = sysctl_getset_bigarray( req,
471 sysctl_counter_size,
472 sysctl_kpc_get_period,
473 sysctl_kpc_set_period );
474 break;
475
476 case REQ_ACTIONID:
477 ret = sysctl_getset_bigarray( req,
478 sysctl_actionid_size,
479 sysctl_kpc_get_actionid,
480 sysctl_kpc_set_actionid );
481 break;
482
483
484 case REQ_SW_INC:
485 ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc );
486 break;
487
488 case REQ_PMU_VERSION:
489 ret = sysctl_get_int(oidp, req, kpc_get_pmu_version());
490 break;
491
492 default:
493 ret = ENOENT;
494 break;
495 }
496
497 lck_mtx_unlock(&sysctl_lock);
498
499 return ret;
500 }
501
502
503 /*** sysctl definitions ***/
504
505 /* root kperf node */
506 SYSCTL_NODE(, OID_AUTO, kpc, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
507 "kpc");
508
509 /* values */
510 SYSCTL_PROC(_kpc, OID_AUTO, classes,
511 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
512 (void*)REQ_CLASSES,
513 sizeof(int), kpc_sysctl, "I", "Available classes");
514
515 SYSCTL_PROC(_kpc, OID_AUTO, counting,
516 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
517 (void*)REQ_COUNTING,
518 sizeof(int), kpc_sysctl, "I", "PMCs counting");
519
520 SYSCTL_PROC(_kpc, OID_AUTO, thread_counting,
521 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
522 (void*)REQ_THREAD_COUNTING,
523 sizeof(int), kpc_sysctl, "I", "Thread accumulation");
524
525 SYSCTL_PROC(_kpc, OID_AUTO, pmu_version,
526 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
527 (void *)REQ_PMU_VERSION,
528 sizeof(int), kpc_sysctl, "I", "PMU version for hardware");
529
530 /* faux values */
531 SYSCTL_PROC(_kpc, OID_AUTO, config_count,
532 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
533 (void*)REQ_CONFIG_COUNT,
534 sizeof(int), kpc_sysctl, "S", "Config count");
535
536 SYSCTL_PROC(_kpc, OID_AUTO, counter_count,
537 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
538 (void*)REQ_COUNTER_COUNT,
539 sizeof(int), kpc_sysctl, "S", "Counter count");
540
541 SYSCTL_PROC(_kpc, OID_AUTO, sw_inc,
542 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
543 (void*)REQ_SW_INC,
544 sizeof(int), kpc_sysctl, "S", "Software increment");
545
546 /* arrays */
547 SYSCTL_PROC(_kpc, OID_AUTO, thread_counters,
548 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
549 (void*)REQ_THREAD_COUNTERS,
550 sizeof(uint64_t), kpc_sysctl,
551 "QU", "Current thread counters");
552
553 SYSCTL_PROC(_kpc, OID_AUTO, counters,
554 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
555 (void*)REQ_COUNTERS,
556 sizeof(uint64_t), kpc_sysctl,
557 "QU", "Current counters");
558
559 SYSCTL_PROC(_kpc, OID_AUTO, shadow_counters,
560 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
561 (void*)REQ_SHADOW_COUNTERS,
562 sizeof(uint64_t), kpc_sysctl,
563 "QU", "Current shadow counters");
564
565 SYSCTL_PROC(_kpc, OID_AUTO, config,
566 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
567 (void*)REQ_CONFIG,
568 sizeof(uint64_t), kpc_sysctl,
569 "QU", "Set counter configs");
570
571 SYSCTL_PROC(_kpc, OID_AUTO, period,
572 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
573 (void*)REQ_PERIOD,
574 sizeof(uint64_t), kpc_sysctl,
575 "QU", "Set counter periods");
576
577 SYSCTL_PROC(_kpc, OID_AUTO, actionid,
578 CTLFLAG_RD | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_MASKED | CTLFLAG_LOCKED,
579 (void*)REQ_ACTIONID,
580 sizeof(uint32_t), kpc_sysctl,
581 "QU", "Set counter actionids");
582
583
584
585 #ifdef __arm64__
586
587 extern int kpc_pc_capture;
588 SYSCTL_INT(_kpc, OID_AUTO, pc_capture_supported,
589 CTLFLAG_RD | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, &kpc_pc_capture, 0,
590 "whether PC capture is supported by the hardware");
591
592 #endif /* __arm64__ */
593
594 #endif // CONFIG_CPU_COUNTERS
595