1 /*
2 * Copyright (c) 2012-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/cpu_data_internal.h>
30 #include <arm/cpu_internal.h>
31 #include <kern/cpu_number.h>
32 #include <kern/kpc.h>
33 #include <kern/thread.h>
34 #include <kern/processor.h>
35 #include <kern/monotonic.h>
36 #include <mach/mach_types.h>
37 #include <machine/machine_routines.h>
38 #include <kern/cpc.h>
39 #include <stdint.h>
40 #include <sys/errno.h>
41
42 #if HAS_CPMU_PC_CAPTURE
43 int kpc_pc_capture = 1;
44 #else /* HAS_CPMU_PC_CAPTURE */
45 int kpc_pc_capture = 0;
46 #endif /* !HAS_CPMU_PC_CAPTURE */
47
48 #if DEVELOPMENT || DEBUG
49 bool kpc_allows_counting_system = true;
50 #else // DEVELOPMENT || DEBUG
51 __security_const_late bool kpc_allows_counting_system = false;
52 #endif // !(DEVELOPMENT || DEBUG)
53
54 #if APPLE_ARM64_ARCH_FAMILY
55
56 void kpc_pmi_handler(unsigned int ctr);
57
58 /*
59 * PMCs 8 and 9 were added to Hurricane and to maintain the existing bit
60 * positions of the other PMCs, their configuration bits start at position 32.
61 */
62 #define PMCR_PMC_8_9_OFFSET (32)
63 #define PMCR_PMC_8_9_SHIFT(PMC) (((PMC) - 8) + PMCR_PMC_8_9_OFFSET)
64 #define PMCR_PMC_SHIFT(PMC) (((PMC) <= 7) ? (PMC) : \
65 PMCR_PMC_8_9_SHIFT(PMC))
66
67 /*
68 * PMCR0 controls enabling, interrupts, and overflow of performance counters.
69 */
70
71 /* PMC is enabled */
72 #define PMCR0_PMC_ENABLE_MASK(PMC) (UINT64_C(0x1) << PMCR_PMC_SHIFT(PMC))
73 #define PMCR0_PMC_DISABLE_MASK(PMC) (~PMCR0_PMC_ENABLE_MASK(PMC))
74
75 /* overflow on a PMC generates an interrupt */
76 #define PMCR0_PMI_OFFSET (12)
77 #define PMCR0_PMI_SHIFT(PMC) (PMCR0_PMI_OFFSET + PMCR_PMC_SHIFT(PMC))
78 #define PMCR0_PMI_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR0_PMI_SHIFT(PMC))
79 #define PMCR0_PMI_DISABLE_MASK(PMC) (~PMCR0_PMI_ENABLE_MASK(PMC))
80
81 /* disable counting when a PMI is signaled (except for AIC interrupts) */
82 #define PMCR0_DISCNT_SHIFT (20)
83 #define PMCR0_DISCNT_ENABLE_MASK (UINT64_C(1) << PMCR0_DISCNT_SHIFT)
84 #define PMCR0_DISCNT_DISABLE_MASK (~PMCR0_DISCNT_ENABLE_MASK)
85
86 /* 21 unused */
87
88 /* block PMIs until ERET retires */
89 #define PMCR0_WFRFE_SHIFT (22)
90 #define PMCR0_WFRFE_ENABLE_MASK (UINT64_C(1) << PMCR0_WFRE_SHIFT)
91 #define PMCR0_WFRFE_DISABLE_MASK (~PMCR0_WFRFE_ENABLE_MASK)
92
93 /* count global L2C events */
94 #define PMCR0_L2CGLOBAL_SHIFT (23)
95 #define PMCR0_L2CGLOBAL_ENABLE_MASK (UINT64_C(1) << PMCR0_L2CGLOBAL_SHIFT)
96 #define PMCR0_L2CGLOBAL_DISABLE_MASK (~PMCR0_L2CGLOBAL_ENABLE_MASK)
97
98 /* allow user mode access to configuration registers */
99 #define PMCR0_USEREN_SHIFT (30)
100 #define PMCR0_USEREN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT)
101 #define PMCR0_USEREN_DISABLE_MASK (~PMCR0_USEREN_ENABLE_MASK)
102
103 /* force the CPMU clocks in case of a clocking bug */
104 #define PMCR0_CLKEN_SHIFT (31)
105 #define PMCR0_CLKEN_ENABLE_MASK (UINT64_C(1) << PMCR0_CLKEN_SHIFT)
106 #define PMCR0_CLKEN_DISABLE_MASK (~PMCR0_CLKEN_ENABLE_MASK)
107
108 /* 32 - 44 mirror the low bits for PMCs 8 and 9 */
109
110 /* PMCR1 enables counters in different processor modes */
111
112 #define PMCR1_EL0_A32_OFFSET (0)
113 #define PMCR1_EL0_A64_OFFSET (8)
114 #define PMCR1_EL1_A64_OFFSET (16)
115 #define PMCR1_EL3_A64_OFFSET (24)
116
117 #define PMCR1_EL0_A32_SHIFT(PMC) (PMCR1_EL0_A32_OFFSET + PMCR_PMC_SHIFT(PMC))
118 #define PMCR1_EL0_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC))
119 #define PMCR1_EL1_A64_SHIFT(PMC) (PMCR1_EL1_A64_OFFSET + PMCR_PMC_SHIFT(PMC))
120 #define PMCR1_EL3_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC))
121
122 #define PMCR1_EL0_A32_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A32_SHIFT(PMC))
123 #define PMCR1_EL0_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A64_SHIFT(PMC))
124 #define PMCR1_EL1_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL1_A64_SHIFT(PMC))
125 /* PMCR1_EL3_A64 is not supported on PMCs 8 and 9 */
126 #if NO_MONITOR
127 #define PMCR1_EL3_A64_ENABLE_MASK(PMC) UINT64_C(0)
128 #else
129 #define PMCR1_EL3_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL3_A64_SHIFT(PMC))
130 #endif
131
132 #define PMCR1_EL_ALL_ENABLE_MASK(PMC) (PMCR1_EL0_A32_ENABLE_MASK(PMC) | \
133 PMCR1_EL0_A64_ENABLE_MASK(PMC) | \
134 PMCR1_EL1_A64_ENABLE_MASK(PMC) | \
135 PMCR1_EL3_A64_ENABLE_MASK(PMC))
136 #define PMCR1_EL_ALL_DISABLE_MASK(PMC) (~PMCR1_EL_ALL_ENABLE_MASK(PMC))
137
138 /* PMESR0 and PMESR1 are event selection registers */
139
140 /* PMESR0 selects which event is counted on PMCs 2, 3, 4, and 5 */
141 /* PMESR1 selects which event is counted on PMCs 6, 7, 8, and 9 */
142
143 #if CPMU_16BIT_EVENTS
144 #define PMESR_PMC_WIDTH UINT64_C(16)
145 #define PMESR_PMC_MASK ((uint64_t)UINT16_MAX)
146 #else // CPMU_16BIT_EVENTS
147 #define PMESR_PMC_WIDTH UINT64_C(8)
148 #define PMESR_PMC_MASK ((uint64_t)UINT8_MAX)
149 #endif // !CPMU_16BIT_EVENTS
150
151 #define PMESR_SHIFT(PMC, OFF) ((PMESR_PMC_WIDTH) * ((PMC) - (OFF)))
152 #define PMESR_EVT_MASK(PMC, OFF) (PMESR_PMC_MASK << PMESR_SHIFT(PMC, OFF))
153 #define PMESR_EVT_CLEAR(PMC, OFF) (~PMESR_EVT_MASK(PMC, OFF))
154
155 #define PMESR_EVT_DECODE(PMESR, PMC, OFF) \
156 (((PMESR) >> PMESR_SHIFT(PMC, OFF)) & PMESR_PMC_MASK)
157 #define PMESR_EVT_ENCODE(EVT, PMC, OFF) \
158 (((EVT) & PMESR_PMC_MASK) << PMESR_SHIFT(PMC, OFF))
159
160 /*
161 * The low 8 bits of a configuration words select the event to program on
162 * PMESR{0,1}. Bits 16-19 are mapped to PMCR1 bits.
163 */
164 #define CFGWORD_EL0A32EN_MASK (0x10000)
165 #define CFGWORD_EL0A64EN_MASK (0x20000)
166 #define CFGWORD_EL1EN_MASK (0x40000)
167 #define CFGWORD_EL3EN_MASK (0x80000)
168 #define CFGWORD_ALLMODES_MASK (0xf0000)
169
170 /* ACC offsets for PIO */
171 #define ACC_CPMU_PMC0_OFFSET (0x200)
172 #define ACC_CPMU_PMC8_OFFSET (0x280)
173
174 /*
175 * Macros for reading and writing system registers.
176 *
177 * SR must be one of the SREG_* defines above.
178 */
179 #define SREG_WRITE(SR, V) __asm__ volatile("msr " SR ", %0 ; isb" : : "r"(V))
180 #define SREG_READ(SR) ({ uint64_t VAL; \
181 __asm__ volatile("mrs %0, " SR : "=r"(VAL)); \
182 VAL; })
183
184 /*
185 * Configuration registers that can be controlled by RAWPMU:
186 *
187 * All: PMCR2-4, OPMAT0-1, OPMSK0-1.
188 * Typhoon/Twister/Hurricane: PMMMAP, PMTRHLD2/4/6.
189 */
190 #if HAS_EARLY_APPLE_CPMU
191 #define RAWPMU_CONFIG_COUNT 7
192 #else /* HAS_EARLY_APPLE_CPMU */
193 #define RAWPMU_CONFIG_COUNT 11
194 #endif /* !HAS_EARLY_APPLE_CPMU */
195
196 #if HAS_CPMU_PC_CAPTURE
197 #define PMC_SUPPORTS_PC_CAPTURE(CTR) (((CTR) >= 5) && ((CTR) <= 7))
198 #define PC_CAPTURE_PMC(PCC_VAL) (((PCC_VAL) >> 56) & 0x7)
199 #define PC_CAPTURE_PC(PCC_VAL) ((PCC_VAL) & ((UINT64_C(1) << 48) - 1))
200 #endif /* HAS_CPMU_PC_CAPTURE */
201
202 /* TODO: allocate dynamically */
203 static uint64_t saved_PMCR[2];
204 static uint64_t saved_PMESR[2];
205 static uint64_t saved_RAWPMU[RAWPMU_CONFIG_COUNT];
206 static uint64_t saved_counter[MAX_CPUS][KPC_MAX_COUNTERS];
207 static uint64_t kpc_running_cfg_pmc_mask = 0;
208 static uint32_t kpc_running_classes = 0;
209 static uint32_t kpc_configured = 0;
210
211 #ifdef KPC_DEBUG
212 static void
dump_regs(void)213 dump_regs(void)
214 {
215 uint64_t val;
216 kprintf("PMCR0 = 0x%" PRIx64 "\n", SREG_READ("PMCR0_EL1"));
217 kprintf("PMCR1 = 0x%" PRIx64 "\n", SREG_READ("PMCR1_EL1"));
218 kprintf("PMCR2 = 0x%" PRIx64 "\n", SREG_READ("PMCR2_EL1"));
219 kprintf("PMCR3 = 0x%" PRIx64 "\n", SREG_READ("PMCR3_EL1"));
220 kprintf("PMCR4 = 0x%" PRIx64 "\n", SREG_READ("PMCR4_EL1"));
221 kprintf("PMESR0 = 0x%" PRIx64 "\n", SREG_READ("PMESR0_EL1"));
222 kprintf("PMESR1 = 0x%" PRIx64 "\n", SREG_READ("PMESR1_EL1"));
223
224 kprintf("PMC0 = 0x%" PRIx64 "\n", SREG_READ("PMC0"));
225 kprintf("PMC1 = 0x%" PRIx64 "\n", SREG_READ("PMC1"));
226 kprintf("PMC2 = 0x%" PRIx64 "\n", SREG_READ("PMC2"));
227 kprintf("PMC3 = 0x%" PRIx64 "\n", SREG_READ("PMC3"));
228 kprintf("PMC4 = 0x%" PRIx64 "\n", SREG_READ("PMC4"));
229 kprintf("PMC5 = 0x%" PRIx64 "\n", SREG_READ("PMC5"));
230 kprintf("PMC6 = 0x%" PRIx64 "\n", SREG_READ("PMC6"));
231 kprintf("PMC7 = 0x%" PRIx64 "\n", SREG_READ("PMC7"));
232
233 #if (KPC_ARM64_CONFIGURABLE_COUNT > 6)
234 kprintf("PMC8 = 0x%" PRIx64 "\n", SREG_READ("PMC8"));
235 kprintf("PMC9 = 0x%" PRIx64 "\n", SREG_READ("PMC9"));
236 #endif
237 }
238 #endif
239
240 static boolean_t
enable_counter(uint32_t counter)241 enable_counter(uint32_t counter)
242 {
243 uint64_t pmcr0 = 0;
244 boolean_t counter_running, pmi_enabled, enabled;
245
246 pmcr0 = SREG_READ("PMCR0_EL1") | 0x3 /* leave the fixed counters enabled for monotonic */;
247
248 counter_running = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0;
249 pmi_enabled = (pmcr0 & PMCR0_PMI_ENABLE_MASK(counter)) != 0;
250
251 enabled = counter_running && pmi_enabled;
252
253 if (!enabled) {
254 pmcr0 |= PMCR0_PMC_ENABLE_MASK(counter);
255 pmcr0 |= PMCR0_PMI_ENABLE_MASK(counter);
256 SREG_WRITE("PMCR0_EL1", pmcr0);
257 }
258
259 return enabled;
260 }
261
262 static boolean_t
disable_counter(uint32_t counter)263 disable_counter(uint32_t counter)
264 {
265 uint64_t pmcr0;
266 boolean_t enabled;
267
268 if (counter < 2) {
269 return true;
270 }
271
272 pmcr0 = SREG_READ("PMCR0_EL1") | 0x3;
273 enabled = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0;
274
275 if (enabled) {
276 pmcr0 &= PMCR0_PMC_DISABLE_MASK(counter);
277 SREG_WRITE("PMCR0_EL1", pmcr0);
278 }
279
280 return enabled;
281 }
282
283 /*
284 * Enable counter in processor modes determined by configuration word.
285 */
286 static void
set_modes(uint32_t counter,kpc_config_t cfgword,bool secure)287 set_modes(uint32_t counter, kpc_config_t cfgword, bool secure)
288 {
289 bool const allow_kernel = !secure || kpc_allows_counting_system;
290 uint64_t bits = 0;
291
292 if (cfgword & CFGWORD_EL0A32EN_MASK) {
293 bits |= PMCR1_EL0_A32_ENABLE_MASK(counter);
294 }
295 if (cfgword & CFGWORD_EL0A64EN_MASK) {
296 bits |= PMCR1_EL0_A64_ENABLE_MASK(counter);
297 }
298 if (allow_kernel && (cfgword & CFGWORD_EL1EN_MASK)) {
299 bits |= PMCR1_EL1_A64_ENABLE_MASK(counter);
300 }
301
302 /*
303 * Backwards compatibility: Writing a non-zero configuration word with
304 * all zeros in bits 16-19 is interpreted as enabling in all modes.
305 * This matches the behavior when the PMCR1 bits weren't exposed.
306 */
307 if (bits == 0 && cfgword != 0) {
308 bits = allow_kernel ?
309 PMCR1_EL_ALL_ENABLE_MASK(counter)
310 : PMCR1_EL0_A64_ENABLE_MASK(counter);
311 }
312
313 uint64_t pmcr1 = saved_PMCR[1];
314 pmcr1 &= PMCR1_EL_ALL_DISABLE_MASK(counter);
315 pmcr1 |= bits;
316 pmcr1 |= 0x30303; /* monotonic compatibility */
317 saved_PMCR[1] = pmcr1;
318 }
319
320 static uint64_t
read_counter(uint32_t counter)321 read_counter(uint32_t counter)
322 {
323 switch (counter) {
324 // case 0: return SREG_READ("PMC0");
325 // case 1: return SREG_READ("PMC1");
326 case 2: return SREG_READ("PMC2");
327 case 3: return SREG_READ("PMC3");
328 case 4: return SREG_READ("PMC4");
329 case 5: return SREG_READ("PMC5");
330 case 6: return SREG_READ("PMC6");
331 case 7: return SREG_READ("PMC7");
332 #if KPC_ARM64_CONFIGURABLE_COUNT > 6
333 case 8: return SREG_READ("PMC8");
334 case 9: return SREG_READ("PMC9");
335 #endif // KPC_ARM64_CONFIGURABLE_COUNT > 6
336 default: return 0;
337 }
338 }
339
340 static void
write_counter(uint32_t counter,uint64_t value)341 write_counter(uint32_t counter, uint64_t value)
342 {
343 switch (counter) {
344 case 2: SREG_WRITE("PMC2", value); break;
345 case 3: SREG_WRITE("PMC3", value); break;
346 case 4: SREG_WRITE("PMC4", value); break;
347 case 5: SREG_WRITE("PMC5", value); break;
348 case 6: SREG_WRITE("PMC6", value); break;
349 case 7: SREG_WRITE("PMC7", value); break;
350 #if KPC_ARM64_CONFIGURABLE_COUNT > 6
351 case 8: SREG_WRITE("PMC8", value); break;
352 case 9: SREG_WRITE("PMC9", value); break;
353 #endif // KPC_ARM64_CONFIGURABLE_COUNT > 6
354 default: break;
355 }
356 }
357
358 uint32_t
kpc_rawpmu_config_count(void)359 kpc_rawpmu_config_count(void)
360 {
361 return RAWPMU_CONFIG_COUNT;
362 }
363
364 int
kpc_get_rawpmu_config(kpc_config_t * configv)365 kpc_get_rawpmu_config(kpc_config_t *configv)
366 {
367 configv[0] = SREG_READ("PMCR2_EL1");
368 configv[1] = SREG_READ("PMCR3_EL1");
369 configv[2] = SREG_READ("PMCR4_EL1");
370 configv[3] = SREG_READ("OPMAT0_EL1");
371 configv[4] = SREG_READ("OPMAT1_EL1");
372 configv[5] = SREG_READ("OPMSK0_EL1");
373 configv[6] = SREG_READ("OPMSK1_EL1");
374 #if RAWPMU_CONFIG_COUNT > 7
375 configv[7] = SREG_READ("PMMMAP_EL1");
376 configv[8] = SREG_READ("PMTRHLD2_EL1");
377 configv[9] = SREG_READ("PMTRHLD4_EL1");
378 configv[10] = SREG_READ("PMTRHLD6_EL1");
379 #endif
380 return 0;
381 }
382
383 static void
save_regs(void)384 save_regs(void)
385 {
386 int cpuid = cpu_number();
387 __builtin_arm_dmb(DMB_ISH);
388 assert(ml_get_interrupts_enabled() == FALSE);
389 for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) {
390 saved_counter[cpuid][i] = read_counter(i);
391 }
392 }
393
394 static void
restore_control_regs(uint32_t classes)395 restore_control_regs(uint32_t classes)
396 {
397 SREG_WRITE("PMCR1_EL1", saved_PMCR[1] | 0x30303);
398 #if CONFIG_EXCLAVES
399 SREG_WRITE("PMCR1_EL12", saved_PMCR[1] | 0x30303);
400 #endif
401 SREG_WRITE("PMESR0_EL1", saved_PMESR[0]);
402 SREG_WRITE("PMESR1_EL1", saved_PMESR[1]);
403
404 if (classes & KPC_CLASS_RAWPMU_MASK) {
405 SREG_WRITE("PMCR2_EL1", saved_RAWPMU[0]);
406 SREG_WRITE("PMCR3_EL1", saved_RAWPMU[1]);
407 SREG_WRITE("PMCR4_EL1", saved_RAWPMU[2]);
408 SREG_WRITE("OPMAT0_EL1", saved_RAWPMU[3]);
409 SREG_WRITE("OPMAT1_EL1", saved_RAWPMU[4]);
410 SREG_WRITE("OPMSK0_EL1", saved_RAWPMU[5]);
411 SREG_WRITE("OPMSK1_EL1", saved_RAWPMU[6]);
412 #if RAWPMU_CONFIG_COUNT > 7
413 SREG_WRITE("PMMMAP_EL1", saved_RAWPMU[7]);
414 SREG_WRITE("PMTRHLD2_EL1", saved_RAWPMU[8]);
415 SREG_WRITE("PMTRHLD4_EL1", saved_RAWPMU[9]);
416 SREG_WRITE("PMTRHLD6_EL1", saved_RAWPMU[10]);
417 #endif // RAWPMU_CONFIG_COUNT > 7
418 }
419 }
420
421 static void
restore_regs(void)422 restore_regs(void)
423 {
424 int cpuid = cpu_number();
425 for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) {
426 write_counter(i, saved_counter[cpuid][i]);
427 }
428 restore_control_regs(kpc_running_classes);
429 }
430
431 static uint64_t
get_counter_config(uint32_t counter)432 get_counter_config(uint32_t counter)
433 {
434 uint64_t pmesr;
435
436 switch (counter) {
437 case 2:
438 case 3:
439 case 4:
440 case 5:
441 pmesr = PMESR_EVT_DECODE(SREG_READ("PMESR0_EL1"), counter, 2);
442 break;
443 case 6:
444 case 7:
445 #if KPC_ARM64_CONFIGURABLE_COUNT > 6
446 case 8:
447 case 9:
448 #endif // KPC_ARM64_CONFIGURABLE_COUNT > 6
449 pmesr = PMESR_EVT_DECODE(SREG_READ("PMESR1_EL1"), counter, 6);
450 break;
451 default:
452 pmesr = 0;
453 break;
454 }
455
456 kpc_config_t config = pmesr;
457
458 uint64_t pmcr1 = SREG_READ("PMCR1_EL1");
459
460 if (pmcr1 & PMCR1_EL0_A32_ENABLE_MASK(counter)) {
461 config |= CFGWORD_EL0A32EN_MASK;
462 }
463 if (pmcr1 & PMCR1_EL0_A64_ENABLE_MASK(counter)) {
464 config |= CFGWORD_EL0A64EN_MASK;
465 }
466 if (pmcr1 & PMCR1_EL1_A64_ENABLE_MASK(counter)) {
467 config |= CFGWORD_EL1EN_MASK;
468 #if NO_MONITOR
469 config |= CFGWORD_EL3EN_MASK;
470 #endif
471 }
472 #if !NO_MONITOR
473 if (pmcr1 & PMCR1_EL3_A64_ENABLE_MASK(counter)) {
474 config |= CFGWORD_EL3EN_MASK;
475 }
476 #endif
477
478 return config;
479 }
480
481 /* internal functions */
482
483 static bool
kpc_cpu_callback(void * __unused param,enum cpu_event event,unsigned int __unused cpu_or_cluster)484 kpc_cpu_callback(void * __unused param, enum cpu_event event,
485 unsigned int __unused cpu_or_cluster)
486 {
487 if (!kpc_configured) {
488 return true;
489 }
490
491 switch (event) {
492 case CPU_BOOTED:
493 restore_regs();
494 break;
495
496 case CPU_DOWN:
497 save_regs();
498 break;
499
500 default:
501 break;
502 }
503 return true;
504 }
505
506 void
kpc_arch_init(void)507 kpc_arch_init(void)
508 {
509 cpu_event_register_callback(kpc_cpu_callback, NULL);
510 kpc_allows_counting_system = PE_i_can_has_debugger(NULL);
511 }
512
513 boolean_t
kpc_is_running_fixed(void)514 kpc_is_running_fixed(void)
515 {
516 return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK;
517 }
518
519 boolean_t
kpc_is_running_configurable(uint64_t pmc_mask)520 kpc_is_running_configurable(uint64_t pmc_mask)
521 {
522 assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
523 return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) &&
524 ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask);
525 }
526
527 uint32_t
kpc_fixed_count(void)528 kpc_fixed_count(void)
529 {
530 return KPC_ARM64_FIXED_COUNT;
531 }
532
533 uint32_t
kpc_configurable_count(void)534 kpc_configurable_count(void)
535 {
536 return KPC_ARM64_CONFIGURABLE_COUNT;
537 }
538
539 uint32_t
kpc_fixed_config_count(void)540 kpc_fixed_config_count(void)
541 {
542 return 0;
543 }
544
545 uint32_t
kpc_configurable_config_count(uint64_t pmc_mask)546 kpc_configurable_config_count(uint64_t pmc_mask)
547 {
548 assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
549 return kpc_popcount(pmc_mask);
550 }
551
552 int
kpc_get_fixed_config(kpc_config_t * configv __unused)553 kpc_get_fixed_config(kpc_config_t *configv __unused)
554 {
555 return 0;
556 }
557
558 uint64_t
kpc_fixed_max(void)559 kpc_fixed_max(void)
560 {
561 return (1ULL << KPC_ARM64_COUNTER_WIDTH) - 1;
562 }
563
564 uint64_t
kpc_configurable_max(void)565 kpc_configurable_max(void)
566 {
567 return (1ULL << KPC_ARM64_COUNTER_WIDTH) - 1;
568 }
569
570 static void
set_running_configurable(uint64_t target_mask,uint64_t state_mask)571 set_running_configurable(uint64_t target_mask, uint64_t state_mask)
572 {
573 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
574 boolean_t enabled;
575
576 enabled = ml_set_interrupts_enabled(FALSE);
577
578 for (uint32_t i = 0; i < cfg_count; ++i) {
579 if (((1ULL << i) & target_mask) == 0) {
580 continue;
581 }
582 assert(kpc_controls_counter(offset + i));
583
584 if ((1ULL << i) & state_mask) {
585 enable_counter(offset + i);
586 } else {
587 disable_counter(offset + i);
588 }
589 }
590
591 ml_set_interrupts_enabled(enabled);
592 }
593
594 static uint32_t kpc_xcall_sync;
595 static void
kpc_set_running_xcall(void * vstate)596 kpc_set_running_xcall( void *vstate )
597 {
598 struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate;
599 assert(mp_config);
600
601 set_running_configurable(mp_config->cfg_target_mask,
602 mp_config->cfg_state_mask);
603
604 if (os_atomic_dec(&kpc_xcall_sync, relaxed) == 0) {
605 thread_wakeup((event_t) &kpc_xcall_sync);
606 }
607 }
608
609 static uint32_t kpc_xread_sync;
610 static void
kpc_get_curcpu_counters_xcall(void * args)611 kpc_get_curcpu_counters_xcall(void *args)
612 {
613 struct kpc_get_counters_remote *handler = args;
614
615 assert(handler != NULL);
616 assert(handler->buf != NULL);
617
618 int offset = cpu_number() * handler->buf_stride;
619 int r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]);
620
621 /* number of counters added by this CPU, needs to be atomic */
622 os_atomic_add(&(handler->nb_counters), r, relaxed);
623
624 if (os_atomic_dec(&kpc_xread_sync, relaxed) == 0) {
625 thread_wakeup((event_t) &kpc_xread_sync);
626 }
627 }
628
629 int
kpc_get_all_cpus_counters(uint32_t classes,int * curcpu,uint64_t * buf)630 kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
631 {
632 assert(buf != NULL);
633
634 int enabled = ml_set_interrupts_enabled(FALSE);
635
636 /* grab counters and CPU number as close as possible */
637 if (curcpu) {
638 *curcpu = cpu_number();
639 }
640
641 struct kpc_get_counters_remote hdl = {
642 .classes = classes,
643 .nb_counters = 0,
644 .buf = buf,
645 .buf_stride = kpc_get_counter_count(classes)
646 };
647
648 cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl);
649 int offset = hdl.nb_counters;
650
651 (void)ml_set_interrupts_enabled(enabled);
652
653 return offset;
654 }
655
656 int
kpc_get_fixed_counters(uint64_t * counterv)657 kpc_get_fixed_counters(uint64_t *counterv)
658 {
659 #if CONFIG_CPU_COUNTERS
660 mt_fixed_counts(counterv);
661 return 0;
662 #else /* CONFIG_CPU_COUNTERS */
663 #pragma unused(counterv)
664 return ENOTSUP;
665 #endif /* !CONFIG_CPU_COUNTERS */
666 }
667
668 int
kpc_get_configurable_counters(uint64_t * counterv,uint64_t pmc_mask)669 kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask)
670 {
671 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
672 uint64_t ctr = 0ULL;
673
674 assert(counterv);
675
676 for (uint32_t i = 0; i < cfg_count; ++i) {
677 if (((1ULL << i) & pmc_mask) == 0) {
678 continue;
679 }
680 ctr = read_counter(i + offset);
681
682 if (ctr & KPC_ARM64_COUNTER_OVF_MASK) {
683 ctr = CONFIGURABLE_SHADOW(i) +
684 (kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 /* Wrap */) +
685 (ctr & KPC_ARM64_COUNTER_MASK);
686 } else {
687 ctr = CONFIGURABLE_SHADOW(i) +
688 (ctr - CONFIGURABLE_RELOAD(i));
689 }
690
691 *counterv++ = ctr;
692 }
693
694 return 0;
695 }
696
697 int
kpc_get_configurable_config(kpc_config_t * configv,uint64_t pmc_mask)698 kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
699 {
700 uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
701
702 assert(configv);
703
704 for (uint32_t i = 0; i < cfg_count; ++i) {
705 if ((1ULL << i) & pmc_mask) {
706 *configv++ = get_counter_config(i + offset);
707 }
708 }
709 return 0;
710 }
711
712 static uint32_t kpc_config_sync;
713 static void
kpc_set_config_xcall(void * vmp_config)714 kpc_set_config_xcall(void *vmp_config)
715 {
716 struct kpc_config_remote *mp_config = vmp_config;
717 uint32_t classes = 0ULL;
718
719 assert(mp_config);
720 classes = mp_config->classes;
721 boolean_t enabled = ml_set_interrupts_enabled(FALSE);
722 restore_control_regs(classes);
723 ml_set_interrupts_enabled(enabled);
724
725 if (os_atomic_dec(&kpc_config_sync, relaxed) == 0) {
726 thread_wakeup((event_t) &kpc_config_sync);
727 }
728 }
729
730 static uint64_t
kpc_reload_counter(uint32_t ctr)731 kpc_reload_counter(uint32_t ctr)
732 {
733 assert(ctr < (kpc_configurable_count() + kpc_fixed_count()));
734
735 uint64_t old = read_counter(ctr);
736
737 if (kpc_controls_counter(ctr)) {
738 write_counter(ctr, FIXED_RELOAD(ctr));
739 return old & KPC_ARM64_COUNTER_MASK;
740 } else {
741 /*
742 * Unset the overflow bit to clear the condition that drives
743 * PMIs. The power manager is not interested in handling PMIs.
744 */
745 write_counter(ctr, old & KPC_ARM64_COUNTER_MASK);
746 return 0;
747 }
748 }
749
750 static uint32_t kpc_reload_sync;
751 static void
kpc_set_reload_xcall(void * vmp_config)752 kpc_set_reload_xcall(void *vmp_config)
753 {
754 struct kpc_config_remote *mp_config = vmp_config;
755 uint32_t classes = 0, count = 0, offset = kpc_fixed_count();
756 uint64_t *new_period = NULL, max = kpc_configurable_max();
757 boolean_t enabled;
758
759 assert(mp_config);
760 assert(mp_config->configv);
761 classes = mp_config->classes;
762 new_period = mp_config->configv;
763
764 enabled = ml_set_interrupts_enabled(FALSE);
765
766 if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
767 /*
768 * Update _all_ shadow counters, this cannot be done for only
769 * selected PMCs. Otherwise, we would corrupt the configurable
770 * shadow buffer since the PMCs are muxed according to the pmc
771 * mask.
772 */
773 uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1;
774 kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask);
775
776 /* set the new period */
777 count = kpc_configurable_count();
778 for (uint32_t i = 0; i < count; ++i) {
779 /* ignore the counter */
780 if (((1ULL << i) & mp_config->pmc_mask) == 0) {
781 continue;
782 }
783 if (*new_period == 0) {
784 *new_period = kpc_configurable_max();
785 }
786 CONFIGURABLE_RELOAD(i) = max - *new_period;
787 /* reload the counter */
788 kpc_reload_counter(offset + i);
789 /* next period value */
790 new_period++;
791 }
792 }
793
794 ml_set_interrupts_enabled(enabled);
795
796 if (os_atomic_dec(&kpc_reload_sync, relaxed) == 0) {
797 thread_wakeup((event_t) &kpc_reload_sync);
798 }
799 }
800
801 void
kpc_pmi_handler(unsigned int ctr)802 kpc_pmi_handler(unsigned int ctr)
803 {
804 uintptr_t pc = 0;
805 bool captured = false;
806
807 #if HAS_CPMU_PC_CAPTURE
808 if (FIXED_ACTIONID(ctr) && PMC_SUPPORTS_PC_CAPTURE(ctr)) {
809 uintptr_t pc_capture = SREG_READ("PM_PMI_PC");
810 captured = PC_CAPTURE_PMC(pc_capture) == ctr;
811 if (captured) {
812 pc = PC_CAPTURE_PC(pc_capture);
813 }
814 }
815 #endif // HAS_CPMU_PC_CAPTURE
816
817 uint64_t extra = kpc_reload_counter(ctr);
818
819 FIXED_SHADOW(ctr) += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* Wrap */) + extra;
820
821 if (FIXED_ACTIONID(ctr)) {
822 bool kernel = true;
823 struct arm_saved_state *state;
824 state = getCpuDatap()->cpu_int_state;
825 if (state) {
826 kernel = !PSR64_IS_USER(get_saved_state_cpsr(state));
827 if (!captured) {
828 pc = get_saved_state_pc(state);
829 }
830 if (kernel) {
831 pc = VM_KERNEL_UNSLIDE(pc);
832 }
833 } else {
834 /*
835 * Don't know where the PC came from and may be a kernel address, so
836 * clear it to prevent leaking the slide.
837 */
838 pc = 0;
839 }
840
841 uint64_t config = get_counter_config(ctr);
842 kperf_kpc_flags_t flags = kernel ? KPC_KERNEL_PC : 0;
843 flags |= captured ? KPC_CAPTURED_PC : 0;
844 bool custom_mode = false;
845 if ((config & CFGWORD_EL0A32EN_MASK) || (config & CFGWORD_EL0A64EN_MASK)) {
846 flags |= KPC_USER_COUNTING;
847 custom_mode = true;
848 }
849 if ((config & CFGWORD_EL1EN_MASK)) {
850 flags |= KPC_KERNEL_COUNTING;
851 custom_mode = true;
852 }
853 /*
854 * For backwards-compatibility.
855 */
856 if (!custom_mode) {
857 flags |= KPC_USER_COUNTING | KPC_KERNEL_COUNTING;
858 }
859 kpc_sample_kperf(FIXED_ACTIONID(ctr), ctr, config & 0xffff, FIXED_SHADOW(ctr),
860 pc, flags);
861 }
862 }
863
864 uint32_t
kpc_get_classes(void)865 kpc_get_classes(void)
866 {
867 return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_RAWPMU_MASK;
868 }
869
870 int
kpc_set_running_arch(struct kpc_running_remote * mp_config)871 kpc_set_running_arch(struct kpc_running_remote *mp_config)
872 {
873 assert(mp_config != NULL);
874
875 /* dispatch to all CPUs */
876 cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall, mp_config);
877
878 kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask;
879 kpc_running_classes = mp_config->classes;
880 kpc_configured = 1;
881
882 return 0;
883 }
884
885 int
kpc_set_period_arch(struct kpc_config_remote * mp_config)886 kpc_set_period_arch(struct kpc_config_remote *mp_config)
887 {
888 assert(mp_config);
889
890 /* dispatch to all CPUs */
891 cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config);
892
893 kpc_configured = 1;
894
895 return 0;
896 }
897
898 int
kpc_set_config_arch(struct kpc_config_remote * mp_config)899 kpc_set_config_arch(struct kpc_config_remote *mp_config)
900 {
901 assert(mp_config);
902 assert(mp_config->configv);
903
904 uint64_t cfg_pmc_mask = mp_config->pmc_mask;
905 unsigned int cfg_count = kpc_configurable_count();
906 unsigned int offset = kpc_fixed_count();
907 unsigned int config_index = 0;
908
909 if (mp_config->secure) {
910 /* Do a pass to find any disallowed events to avoid partial configuration. */
911 for (uint32_t i = 0; i < cfg_count; ++i) {
912 if (((1ULL << i) & cfg_pmc_mask) == 0) {
913 continue;
914 }
915 uint64_t config_value = mp_config->configv[config_index];
916 if (!cpc_event_allowed(CPC_HW_CPMU, config_value & PMESR_PMC_MASK)) {
917 return EPERM;
918 }
919 config_index++;
920 }
921 }
922
923 config_index = 0;
924 for (uint32_t i = 0; i < cfg_count; ++i) {
925 if (((1ULL << i) & cfg_pmc_mask) == 0) {
926 continue;
927 }
928 unsigned int counter = i + offset;
929 assert(kpc_controls_counter(counter));
930 uint64_t config_value = mp_config->configv[config_index];
931
932 const int pmesr_idx = counter < 6 ? 0 : 1;
933 const int pmesr_off = counter < 6 ? 2 : 6;
934 saved_PMESR[pmesr_idx] &= PMESR_EVT_CLEAR(counter, pmesr_off);
935 saved_PMESR[pmesr_idx] |= PMESR_EVT_ENCODE(config_value, counter,
936 pmesr_off);
937 set_modes(counter, config_value, mp_config->secure);
938 config_index++;
939 }
940
941 if (mp_config->classes & KPC_CLASS_RAWPMU_MASK) {
942 unsigned int rawpmu_start = kpc_popcount(mp_config->pmc_mask);
943 memcpy(saved_RAWPMU, &mp_config->configv[rawpmu_start],
944 sizeof(saved_RAWPMU));
945 }
946
947 cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config);
948 kpc_configured = 1;
949
950 return 0;
951 }
952
953 void
kpc_idle(void)954 kpc_idle(void)
955 {
956 if (kpc_configured) {
957 save_regs();
958 }
959 }
960
961 void
kpc_idle_exit(void)962 kpc_idle_exit(void)
963 {
964 if (kpc_configured) {
965 restore_regs();
966 }
967 }
968
969 int
kpc_set_sw_inc(uint32_t mask __unused)970 kpc_set_sw_inc( uint32_t mask __unused )
971 {
972 return ENOTSUP;
973 }
974
975 int
kpc_get_pmu_version(void)976 kpc_get_pmu_version(void)
977 {
978 return KPC_PMU_ARM_APPLE;
979 }
980
981 #else /* APPLE_ARM64_ARCH_FAMILY */
982
983 /* We don't currently support non-Apple arm64 PMU configurations like PMUv3 */
984
985 void
kpc_arch_init(void)986 kpc_arch_init(void)
987 {
988 /* No-op */
989 }
990
991 uint32_t
kpc_get_classes(void)992 kpc_get_classes(void)
993 {
994 return 0;
995 }
996
997 uint32_t
kpc_fixed_count(void)998 kpc_fixed_count(void)
999 {
1000 return 0;
1001 }
1002
1003 uint32_t
kpc_configurable_count(void)1004 kpc_configurable_count(void)
1005 {
1006 return 0;
1007 }
1008
1009 uint32_t
kpc_fixed_config_count(void)1010 kpc_fixed_config_count(void)
1011 {
1012 return 0;
1013 }
1014
1015 uint32_t
kpc_configurable_config_count(uint64_t pmc_mask __unused)1016 kpc_configurable_config_count(uint64_t pmc_mask __unused)
1017 {
1018 return 0;
1019 }
1020
1021 int
kpc_get_fixed_config(kpc_config_t * configv __unused)1022 kpc_get_fixed_config(kpc_config_t *configv __unused)
1023 {
1024 return 0;
1025 }
1026
1027 uint64_t
kpc_fixed_max(void)1028 kpc_fixed_max(void)
1029 {
1030 return 0;
1031 }
1032
1033 uint64_t
kpc_configurable_max(void)1034 kpc_configurable_max(void)
1035 {
1036 return 0;
1037 }
1038
1039 int
kpc_get_configurable_config(kpc_config_t * configv __unused,uint64_t pmc_mask __unused)1040 kpc_get_configurable_config(kpc_config_t *configv __unused, uint64_t pmc_mask __unused)
1041 {
1042 return ENOTSUP;
1043 }
1044
1045 int
kpc_get_configurable_counters(uint64_t * counterv __unused,uint64_t pmc_mask __unused)1046 kpc_get_configurable_counters(uint64_t *counterv __unused, uint64_t pmc_mask __unused)
1047 {
1048 return ENOTSUP;
1049 }
1050
1051 int
kpc_get_fixed_counters(uint64_t * counterv __unused)1052 kpc_get_fixed_counters(uint64_t *counterv __unused)
1053 {
1054 return 0;
1055 }
1056
1057 boolean_t
kpc_is_running_fixed(void)1058 kpc_is_running_fixed(void)
1059 {
1060 return FALSE;
1061 }
1062
1063 boolean_t
kpc_is_running_configurable(uint64_t pmc_mask __unused)1064 kpc_is_running_configurable(uint64_t pmc_mask __unused)
1065 {
1066 return FALSE;
1067 }
1068
1069 int
kpc_set_running_arch(struct kpc_running_remote * mp_config __unused)1070 kpc_set_running_arch(struct kpc_running_remote *mp_config __unused)
1071 {
1072 return ENOTSUP;
1073 }
1074
1075 int
kpc_set_period_arch(struct kpc_config_remote * mp_config __unused)1076 kpc_set_period_arch(struct kpc_config_remote *mp_config __unused)
1077 {
1078 return ENOTSUP;
1079 }
1080
1081 int
kpc_set_config_arch(struct kpc_config_remote * mp_config __unused)1082 kpc_set_config_arch(struct kpc_config_remote *mp_config __unused)
1083 {
1084 return ENOTSUP;
1085 }
1086
1087 void
kpc_idle(void)1088 kpc_idle(void)
1089 {
1090 // do nothing
1091 }
1092
1093 void
kpc_idle_exit(void)1094 kpc_idle_exit(void)
1095 {
1096 // do nothing
1097 }
1098
1099 int
kpc_get_all_cpus_counters(uint32_t classes __unused,int * curcpu __unused,uint64_t * buf __unused)1100 kpc_get_all_cpus_counters(uint32_t classes __unused, int *curcpu __unused, uint64_t *buf __unused)
1101 {
1102 return 0;
1103 }
1104
1105 int
kpc_set_sw_inc(uint32_t mask __unused)1106 kpc_set_sw_inc( uint32_t mask __unused )
1107 {
1108 return ENOTSUP;
1109 }
1110
1111 int
kpc_get_pmu_version(void)1112 kpc_get_pmu_version(void)
1113 {
1114 return KPC_PMU_ERROR;
1115 }
1116
1117 uint32_t
kpc_rawpmu_config_count(void)1118 kpc_rawpmu_config_count(void)
1119 {
1120 return 0;
1121 }
1122
1123 int
kpc_get_rawpmu_config(__unused kpc_config_t * configv)1124 kpc_get_rawpmu_config(__unused kpc_config_t *configv)
1125 {
1126 return 0;
1127 }
1128
1129 #endif /* !APPLE_ARM64_ARCH_FAMILY */
1130