1 /*
2 * Copyright (c) 2017-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/cpu_data_internal.h>
30 #include <arm/machine_routines.h>
31 #include <arm64/monotonic.h>
32 #include <kern/assert.h>
33 #include <kern/cpc.h>
34 #include <kern/debug.h> /* panic */
35 #include <kern/kpc.h>
36 #include <kern/monotonic.h>
37 #include <machine/atomic.h>
38 #include <machine/limits.h> /* CHAR_BIT */
39 #include <os/overflow.h>
40 #include <pexpert/arm64/board_config.h>
41 #include <pexpert/device_tree.h> /* SecureDTFindEntry */
42 #include <pexpert/pexpert.h>
43 #include <stdatomic.h>
44 #include <stdint.h>
45 #include <string.h>
46 #include <sys/errno.h>
47 #include <sys/monotonic.h>
48
49 /*
50 * Ensure that control registers read back what was written under MACH_ASSERT
51 * kernels.
52 *
53 * A static inline function cannot be used due to passing the register through
54 * the builtin -- it requires a constant string as its first argument, since
55 * MSRs registers are encoded as an immediate in the instruction.
56 */
57 #if MACH_ASSERT
58 #define CTRL_REG_SET(reg, val) do { \
59 __builtin_arm_wsr64((reg), (val)); \
60 uint64_t __check_reg = __builtin_arm_rsr64((reg)); \
61 if (__check_reg != (val)) { \
62 panic("value written to %s was not read back (wrote %llx, read %llx)", \
63 #reg, (val), __check_reg); \
64 } \
65 } while (0)
66 #else /* MACH_ASSERT */
67 #define CTRL_REG_SET(reg, val) __builtin_arm_wsr64((reg), (val))
68 #endif /* MACH_ASSERT */
69
70 #pragma mark core counters
71
72 const bool mt_core_supported = true;
73
74 static const ml_topology_info_t *topology_info;
75
76 /*
77 * PMC[0-1] are the 48/64-bit fixed counters -- PMC0 is cycles and PMC1 is
78 * instructions (see arm64/monotonic.h).
79 *
80 * PMC2+ are currently handled by kpc.
81 */
82 #define PMC_0_7(X, A) X(0, A); X(1, A); X(2, A); X(3, A); X(4, A); X(5, A); \
83 X(6, A); X(7, A)
84
85 #if CORE_NCTRS > 8
86 #define PMC_8_9(X, A) X(8, A); X(9, A)
87 #else // CORE_NCTRS > 8
88 #define PMC_8_9(X, A)
89 #endif // CORE_NCTRS > 8
90
91 #define PMC_ALL(X, A) PMC_0_7(X, A); PMC_8_9(X, A)
92
93 #if CPMU_64BIT_PMCS
94 #define PMC_WIDTH (63)
95 #else // UPMU_64BIT_PMCS
96 #define PMC_WIDTH (47)
97 #endif // !UPMU_64BIT_PMCS
98
99 #define CTR_MAX ((UINT64_C(1) << PMC_WIDTH) - 1)
100
101 #define CYCLES 0
102 #define INSTRS 1
103
104 /*
105 * PMC0's offset into a core's PIO range.
106 *
107 * This allows cores to remotely query another core's counters.
108 */
109
110 #define PIO_PMC0_OFFSET (0x200)
111
112 /*
113 * The offset of the counter in the configuration registers. Post-Hurricane
114 * devices have additional counters that need a larger shift than the original
115 * counters.
116 *
117 * XXX For now, just support the lower-numbered counters.
118 */
119 #define CTR_POS(CTR) (CTR)
120
121 /*
122 * PMCR0 is the main control register for the performance monitor. It
123 * controls whether the counters are enabled, how they deliver interrupts, and
124 * other features.
125 */
126
127 #define PMCR0_CTR_EN(CTR) (UINT64_C(1) << CTR_POS(CTR))
128 #define PMCR0_FIXED_EN (PMCR0_CTR_EN(CYCLES) | PMCR0_CTR_EN(INSTRS))
129 /* how interrupts are delivered on a PMI */
130 enum {
131 PMCR0_INTGEN_OFF = 0,
132 PMCR0_INTGEN_PMI = 1,
133 PMCR0_INTGEN_AIC = 2,
134 PMCR0_INTGEN_HALT = 3,
135 PMCR0_INTGEN_FIQ = 4,
136 };
137 #define PMCR0_INTGEN_SET(X) ((uint64_t)(X) << 8)
138
139 #if CPMU_AIC_PMI
140 #define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_AIC)
141 #else /* CPMU_AIC_PMI */
142 #define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_FIQ)
143 #endif /* !CPMU_AIC_PMI */
144
145 #define PMCR0_PMI_SHIFT (12)
146 #define PMCR0_CTR_GE8_PMI_SHIFT (44)
147 #define PMCR0_PMI_EN(CTR) (UINT64_C(1) << (PMCR0_PMI_SHIFT + CTR_POS(CTR)))
148 /* fixed counters are always counting */
149 #define PMCR0_PMI_INIT (PMCR0_PMI_EN(CYCLES) | PMCR0_PMI_EN(INSTRS))
150 /* disable counting on a PMI */
151 #define PMCR0_DISCNT_EN (UINT64_C(1) << 20)
152 /* block PMIs until ERET retires */
153 #define PMCR0_WFRFE_EN (UINT64_C(1) << 22)
154 /* count global (not just core-local) L2C events */
155 #define PMCR0_L2CGLOBAL_EN (UINT64_C(1) << 23)
156 /* user mode access to configuration registers */
157 #define PMCR0_USEREN_EN (UINT64_C(1) << 30)
158 #define PMCR0_CTR_GE8_EN_SHIFT (32)
159
160 #if HAS_CPMU_PC_CAPTURE
161 #define PMCR0_PCC_INIT (UINT64_C(0x7) << 24)
162 #else /* HAS_CPMU_PC_CAPTURE */
163 #define PMCR0_PCC_INIT (0)
164 #endif /* !HAS_CPMU_PC_CAPTURE */
165
166 #define PMCR0_INIT (PMCR0_INTGEN_INIT | PMCR0_PMI_INIT | PMCR0_PCC_INIT)
167
168 /*
169 * PMCR1 controls which execution modes count events.
170 */
171 #define PMCR1_EL0A32_EN(CTR) (UINT64_C(1) << (0 + CTR_POS(CTR)))
172 #define PMCR1_EL0A64_EN(CTR) (UINT64_C(1) << (8 + CTR_POS(CTR)))
173 #define PMCR1_EL1A64_EN(CTR) (UINT64_C(1) << (16 + CTR_POS(CTR)))
174 /* PMCR1_EL3A64 is not supported on systems with no monitor */
175 #if defined(APPLEHURRICANE)
176 #define PMCR1_EL3A64_EN(CTR) UINT64_C(0)
177 #else
178 #define PMCR1_EL3A64_EN(CTR) (UINT64_C(1) << (24 + CTR_POS(CTR)))
179 #endif
180 #define PMCR1_ALL_EN(CTR) (PMCR1_EL0A32_EN(CTR) | PMCR1_EL0A64_EN(CTR) | \
181 PMCR1_EL1A64_EN(CTR) | PMCR1_EL3A64_EN(CTR))
182
183 /* fixed counters always count in all modes */
184 #define PMCR1_INIT (PMCR1_ALL_EN(CYCLES) | PMCR1_ALL_EN(INSTRS))
185
186 static inline void
core_init_execution_modes(void)187 core_init_execution_modes(void)
188 {
189 uint64_t pmcr1;
190
191 pmcr1 = __builtin_arm_rsr64("PMCR1_EL1");
192 pmcr1 |= PMCR1_INIT;
193 __builtin_arm_wsr64("PMCR1_EL1", pmcr1);
194 #if CONFIG_EXCLAVES
195 __builtin_arm_wsr64("PMCR1_EL12", pmcr1);
196 #endif
197 }
198
199 #define PMSR_OVF(CTR) (1ULL << (CTR))
200
201 static int
core_init(__unused mt_device_t dev)202 core_init(__unused mt_device_t dev)
203 {
204 /* the dev node interface to the core counters is still unsupported */
205 return ENOTSUP;
206 }
207
208 struct mt_cpu *
mt_cur_cpu(void)209 mt_cur_cpu(void)
210 {
211 return &getCpuDatap()->cpu_monotonic;
212 }
213
214 uint64_t
mt_core_snap(unsigned int ctr)215 mt_core_snap(unsigned int ctr)
216 {
217 switch (ctr) {
218 #define PMC_RD(CTR, UNUSED) case (CTR): return __builtin_arm_rsr64(__MSR_STR(PMC ## CTR))
219 PMC_ALL(PMC_RD, 0);
220 #undef PMC_RD
221 default:
222 panic("monotonic: invalid core counter read: %u", ctr);
223 __builtin_unreachable();
224 }
225 }
226
227 void
mt_core_set_snap(unsigned int ctr,uint64_t count)228 mt_core_set_snap(unsigned int ctr, uint64_t count)
229 {
230 switch (ctr) {
231 case 0:
232 __builtin_arm_wsr64("PMC0", count);
233 break;
234 case 1:
235 __builtin_arm_wsr64("PMC1", count);
236 break;
237 default:
238 panic("monotonic: invalid core counter %u write %llu", ctr, count);
239 __builtin_unreachable();
240 }
241 }
242
243 static void
core_set_enabled(void)244 core_set_enabled(void)
245 {
246 uint32_t kpc_mask = kpc_get_running() &
247 (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK);
248 uint64_t pmcr0 = __builtin_arm_rsr64("PMCR0_EL1");
249 pmcr0 |= PMCR0_INIT | PMCR0_FIXED_EN;
250
251 if (kpc_mask != 0) {
252 uint64_t kpc_ctrs = kpc_get_configurable_pmc_mask(kpc_mask) <<
253 MT_CORE_NFIXED;
254 #if KPC_ARM64_CONFIGURABLE_COUNT > 6
255 uint64_t ctrs_ge8 = kpc_ctrs >> 8;
256 pmcr0 |= ctrs_ge8 << PMCR0_CTR_GE8_EN_SHIFT;
257 pmcr0 |= ctrs_ge8 << PMCR0_CTR_GE8_PMI_SHIFT;
258 kpc_ctrs &= (1ULL << 8) - 1;
259 #endif /* KPC_ARM64_CONFIGURABLE_COUNT > 6 */
260 kpc_ctrs |= kpc_ctrs << PMCR0_PMI_SHIFT;
261 pmcr0 |= kpc_ctrs;
262 }
263
264 __builtin_arm_wsr64("PMCR0_EL1", pmcr0);
265 #if MACH_ASSERT
266 /*
267 * Only check for the values that were ORed in.
268 */
269 uint64_t pmcr0_check = __builtin_arm_rsr64("PMCR0_EL1");
270 if ((pmcr0_check & (PMCR0_INIT | PMCR0_FIXED_EN)) != (PMCR0_INIT | PMCR0_FIXED_EN)) {
271 panic("monotonic: hardware ignored enable (read %llx, wrote %llx)",
272 pmcr0_check, pmcr0);
273 }
274 #endif /* MACH_ASSERT */
275 }
276
277 static void
core_idle(__unused cpu_data_t * cpu)278 core_idle(__unused cpu_data_t *cpu)
279 {
280 assert(cpu != NULL);
281 assert(ml_get_interrupts_enabled() == FALSE);
282
283 #if DEBUG
284 uint64_t pmcr0 = __builtin_arm_rsr64("PMCR0_EL1");
285 if ((pmcr0 & PMCR0_FIXED_EN) == 0) {
286 panic("monotonic: counters disabled before idling, pmcr0 = 0x%llx", pmcr0);
287 }
288 uint64_t pmcr1 = __builtin_arm_rsr64("PMCR1_EL1");
289 if ((pmcr1 & PMCR1_INIT) == 0) {
290 panic("monotonic: counter modes disabled before idling, pmcr1 = 0x%llx", pmcr1);
291 }
292 #endif /* DEBUG */
293
294 /* disable counters before updating */
295 __builtin_arm_wsr64("PMCR0_EL1", PMCR0_INIT);
296
297 mt_update_fixed_counts();
298 }
299
300 #pragma mark uncore performance monitor
301
302 #if HAS_UNCORE_CTRS
303
304 static bool mt_uncore_initted = false;
305
306 static bool mt_uncore_suspended_cpd = false;
307
308 /*
309 * Uncore Performance Monitor
310 *
311 * Uncore performance monitors provide event-counting for the last-level caches
312 * (LLCs). Each LLC has its own uncore performance monitor, which can only be
313 * accessed by cores that use that LLC. Like the core performance monitoring
314 * unit, uncore counters are configured globally. If there is more than one
315 * LLC on the system, PIO reads must be used to satisfy uncore requests (using
316 * the `_r` remote variants of the access functions). Otherwise, local MSRs
317 * suffice (using the `_l` local variants of the access functions).
318 */
319
320 #if UNCORE_PER_CLUSTER
321 #define MAX_NMONITORS MAX_CPU_CLUSTERS
322 static uintptr_t cpm_impl[MAX_NMONITORS] = {};
323 #else
324 #define MAX_NMONITORS (1)
325 #endif /* UNCORE_PER_CLUSTER */
326
327 #if UNCORE_VERSION >= 2
328 /*
329 * V2 uncore monitors feature a CTI mechanism -- the second bit of UPMSR is
330 * used to track if a CTI has been triggered due to an overflow.
331 */
332 #define UPMSR_OVF_POS 2
333 #else /* UNCORE_VERSION >= 2 */
334 #define UPMSR_OVF_POS 1
335 #endif /* UNCORE_VERSION < 2 */
336 #define UPMSR_OVF(R, CTR) ((R) >> ((CTR) + UPMSR_OVF_POS) & 0x1)
337 #define UPMSR_OVF_MASK (((UINT64_C(1) << UNCORE_NCTRS) - 1) << UPMSR_OVF_POS)
338
339 #define UPMPCM_CORE(ID) (UINT64_C(1) << (ID))
340
341 #if UPMU_64BIT_PMCS
342 #define UPMC_WIDTH (63)
343 #else // UPMU_64BIT_PMCS
344 #define UPMC_WIDTH (47)
345 #endif // !UPMU_64BIT_PMCS
346
347 /*
348 * The uncore_pmi_mask is a bitmask of CPUs that receive uncore PMIs. It's
349 * initialized by uncore_init and controllable by the uncore_pmi_mask boot-arg.
350 */
351 static int32_t uncore_pmi_mask = 0;
352
353 /*
354 * The uncore_active_ctrs is a bitmask of uncore counters that are currently
355 * requested.
356 */
357 static uint16_t uncore_active_ctrs = 0;
358 static_assert(sizeof(uncore_active_ctrs) * CHAR_BIT >= UNCORE_NCTRS,
359 "counter mask should fit the full range of counters");
360
361 /*
362 * mt_uncore_enabled is true when any uncore counters are active.
363 */
364 bool mt_uncore_enabled = false;
365
366 /*
367 * The uncore_events are the event configurations for each uncore counter -- as
368 * a union to make it easy to program the hardware registers.
369 */
370 static struct uncore_config {
371 union {
372 uint8_t uce_ctrs[UNCORE_NCTRS];
373 uint64_t uce_regs[UNCORE_NCTRS / 8];
374 } uc_events;
375 union {
376 uint16_t uccm_masks[UNCORE_NCTRS];
377 uint64_t uccm_regs[UNCORE_NCTRS / 4];
378 } uc_cpu_masks[MAX_NMONITORS];
379 } uncore_config;
380
381 static struct uncore_monitor {
382 /*
383 * The last snapshot of each of the hardware counter values.
384 */
385 uint64_t um_snaps[UNCORE_NCTRS];
386
387 /*
388 * The accumulated counts for each counter.
389 */
390 uint64_t um_counts[UNCORE_NCTRS];
391
392 /*
393 * Protects accessing the hardware registers and fields in this structure.
394 */
395 lck_spin_t um_lock;
396
397 /*
398 * Whether this monitor needs its registers restored after wake.
399 */
400 bool um_sleeping;
401
402 #if MACH_ASSERT
403 /*
404 * Save the last ID that read from this monitor.
405 */
406 uint8_t um_last_read_id;
407
408 /*
409 * Save whether this monitor has been read since sleeping.
410 */
411 bool um_read_since_sleep;
412 #endif /* MACH_ASSERT */
413 } uncore_monitors[MAX_NMONITORS];
414
415 /*
416 * Each uncore unit has its own monitor, corresponding to the memory hierarchy
417 * of the LLCs.
418 */
419 static unsigned int
uncore_nmonitors(void)420 uncore_nmonitors(void)
421 {
422 #if UNCORE_PER_CLUSTER
423 return topology_info->num_clusters;
424 #else /* UNCORE_PER_CLUSTER */
425 return 1;
426 #endif /* !UNCORE_PER_CLUSTER */
427 }
428
429 static unsigned int
uncmon_get_curid(void)430 uncmon_get_curid(void)
431 {
432 #if UNCORE_PER_CLUSTER
433 return cpu_cluster_id();
434 #else /* UNCORE_PER_CLUSTER */
435 return 0;
436 #endif /* !UNCORE_PER_CLUSTER */
437 }
438
439 /*
440 * Per-monitor locks are required to prevent races with the PMI handlers, not
441 * from other CPUs that are configuring (those are serialized with monotonic's
442 * per-device lock).
443 */
444
445 static int
uncmon_lock(struct uncore_monitor * mon)446 uncmon_lock(struct uncore_monitor *mon)
447 {
448 int intrs_en = ml_set_interrupts_enabled(FALSE);
449 lck_spin_lock(&mon->um_lock);
450 return intrs_en;
451 }
452
453 static void
uncmon_unlock(struct uncore_monitor * mon,int intrs_en)454 uncmon_unlock(struct uncore_monitor *mon, int intrs_en)
455 {
456 lck_spin_unlock(&mon->um_lock);
457 (void)ml_set_interrupts_enabled(intrs_en);
458 }
459
460 static bool
uncmon_is_remote(unsigned int monid)461 uncmon_is_remote(unsigned int monid)
462 {
463 if (monid >= MAX_NMONITORS) {
464 panic("monotonic: %s: invalid monid %u (> %u)", __FUNCTION__, monid, MAX_NMONITORS);
465 }
466 struct uncore_monitor *mon = &uncore_monitors[monid];
467 #pragma unused(mon)
468 LCK_SPIN_ASSERT(&mon->um_lock, LCK_ASSERT_OWNED);
469 return monid == uncmon_get_curid();
470 }
471
472 /*
473 * Helper functions for accessing the hardware -- these require the monitor be
474 * locked to prevent other CPUs' PMI handlers from making local modifications
475 * or updating the counts.
476 */
477
478 #if UNCORE_VERSION >= 2
479 #define UPMCR0_INTEN_POS 20
480 #define UPMCR0_INTGEN_POS 16
481 #else /* UNCORE_VERSION >= 2 */
482 #define UPMCR0_INTEN_POS 12
483 #define UPMCR0_INTGEN_POS 8
484 #endif /* UNCORE_VERSION < 2 */
485 enum {
486 UPMCR0_INTGEN_OFF = 0,
487 /* fast PMIs are only supported on core CPMU */
488 UPMCR0_INTGEN_AIC = 2,
489 UPMCR0_INTGEN_HALT = 3,
490 UPMCR0_INTGEN_FIQ = 4,
491 };
492 /* always enable interrupts for all counters */
493 #define UPMCR0_INTEN (((1ULL << UNCORE_NCTRS) - 1) << UPMCR0_INTEN_POS)
494 /* route uncore PMIs through the FIQ path */
495 #define UPMCR0_INIT (UPMCR0_INTEN | (UPMCR0_INTGEN_FIQ << UPMCR0_INTGEN_POS))
496
497 /*
498 * Turn counting on for counters set in the `enctrmask` and off, otherwise.
499 */
500 static inline void
uncmon_set_counting_locked_l(__unused unsigned int monid,uint64_t enctrmask)501 uncmon_set_counting_locked_l(__unused unsigned int monid, uint64_t enctrmask)
502 {
503 /*
504 * UPMCR0 controls which counters are enabled and how interrupts are generated
505 * for overflows.
506 */
507 __builtin_arm_wsr64("UPMCR0_EL1", UPMCR0_INIT | enctrmask);
508 }
509
510 #if UNCORE_PER_CLUSTER
511
512 /*
513 * Turn counting on for counters set in the `enctrmask` and off, otherwise.
514 */
515 static inline void
uncmon_set_counting_locked_r(unsigned int monid,uint64_t enctrmask)516 uncmon_set_counting_locked_r(unsigned int monid, uint64_t enctrmask)
517 {
518 const uintptr_t upmcr0_offset = 0x4180;
519 *(uint64_t *)(cpm_impl[monid] + upmcr0_offset) = UPMCR0_INIT | enctrmask;
520 }
521
522 #endif /* UNCORE_PER_CLUSTER */
523
524 /*
525 * The uncore performance monitoring counters (UPMCs) are 48/64-bits wide. The
526 * high bit is an overflow bit, triggering a PMI, providing 47/63 usable bits.
527 */
528
529 #define UPMC_MAX ((UINT64_C(1) << UPMC_WIDTH) - 1)
530
531 /*
532 * The `__builtin_arm_{r,w}sr` functions require constant strings, since the
533 * MSR/MRS instructions encode the registers as immediates. Otherwise, this
534 * would be indexing into an array of strings.
535 */
536
537 #define UPMC_0_7(X, A) X(0, A); X(1, A); X(2, A); X(3, A); X(4, A); X(5, A); \
538 X(6, A); X(7, A)
539 #if UNCORE_NCTRS <= 8
540 #define UPMC_ALL(X, A) UPMC_0_7(X, A)
541 #else /* UNCORE_NCTRS <= 8 */
542 #define UPMC_8_15(X, A) X(8, A); X(9, A); X(10, A); X(11, A); X(12, A); \
543 X(13, A); X(14, A); X(15, A)
544 #define UPMC_ALL(X, A) UPMC_0_7(X, A); UPMC_8_15(X, A)
545 #endif /* UNCORE_NCTRS > 8 */
546
547 __unused
548 static inline uint64_t
uncmon_read_counter_locked_l(__unused unsigned int monid,unsigned int ctr)549 uncmon_read_counter_locked_l(__unused unsigned int monid, unsigned int ctr)
550 {
551 assert(ctr < UNCORE_NCTRS);
552 switch (ctr) {
553 #define UPMC_RD(CTR, UNUSED) case (CTR): return __builtin_arm_rsr64(__MSR_STR(UPMC ## CTR))
554 UPMC_ALL(UPMC_RD, 0);
555 #undef UPMC_RD
556 default:
557 panic("monotonic: invalid counter read %u", ctr);
558 __builtin_unreachable();
559 }
560 }
561
562 static inline void
uncmon_write_counter_locked_l(__unused unsigned int monid,unsigned int ctr,uint64_t count)563 uncmon_write_counter_locked_l(__unused unsigned int monid, unsigned int ctr,
564 uint64_t count)
565 {
566 assert(count < UPMC_MAX);
567 assert(ctr < UNCORE_NCTRS);
568 switch (ctr) {
569 #define UPMC_WR(CTR, COUNT) case (CTR): \
570 return __builtin_arm_wsr64(__MSR_STR(UPMC ## CTR), (COUNT))
571 UPMC_ALL(UPMC_WR, count);
572 #undef UPMC_WR
573 default:
574 panic("monotonic: invalid counter write %u", ctr);
575 }
576 }
577
578 #if UNCORE_PER_CLUSTER
579
580 uintptr_t upmc_offs[UNCORE_NCTRS] = {
581 [0] = 0x4100, [1] = 0x4248, [2] = 0x4110, [3] = 0x4250, [4] = 0x4120,
582 [5] = 0x4258, [6] = 0x4130, [7] = 0x4260, [8] = 0x4140, [9] = 0x4268,
583 [10] = 0x4150, [11] = 0x4270, [12] = 0x4160, [13] = 0x4278,
584 [14] = 0x4170, [15] = 0x4280,
585 };
586
587 static inline uint64_t
uncmon_read_counter_locked_r(unsigned int mon_id,unsigned int ctr)588 uncmon_read_counter_locked_r(unsigned int mon_id, unsigned int ctr)
589 {
590 assert(mon_id < uncore_nmonitors());
591 assert(ctr < UNCORE_NCTRS);
592 return *(uint64_t *)(cpm_impl[mon_id] + upmc_offs[ctr]);
593 }
594
595 static inline void
uncmon_write_counter_locked_r(unsigned int mon_id,unsigned int ctr,uint64_t count)596 uncmon_write_counter_locked_r(unsigned int mon_id, unsigned int ctr,
597 uint64_t count)
598 {
599 assert(count < UPMC_MAX);
600 assert(ctr < UNCORE_NCTRS);
601 assert(mon_id < uncore_nmonitors());
602 *(uint64_t *)(cpm_impl[mon_id] + upmc_offs[ctr]) = count;
603 }
604
605 #endif /* UNCORE_PER_CLUSTER */
606
607 static inline void
uncmon_update_locked(unsigned int monid,unsigned int ctr)608 uncmon_update_locked(unsigned int monid, unsigned int ctr)
609 {
610 struct uncore_monitor *mon = &uncore_monitors[monid];
611 if (!mon->um_sleeping) {
612 uint64_t snap = 0;
613 #if UNCORE_PER_CLUSTER
614 snap = uncmon_read_counter_locked_r(monid, ctr);
615 #else /* UNCORE_PER_CLUSTER */
616 snap = uncmon_read_counter_locked_l(monid, ctr);
617 #endif /* UNCORE_PER_CLUSTER */
618 if (snap < mon->um_snaps[ctr]) {
619 #if MACH_ASSERT
620 #if UNCORE_PER_CLUSTER
621 uint64_t remote_value = uncmon_read_counter_locked_r(monid, ctr);
622 #endif /* UNCORE_PER_CLUSTER */
623 panic("monotonic: UPMC%d on UPMU %d went backwards from "
624 "%llx to %llx, read via %s, last was %s from UPMU %hhd%s"
625 #if UNCORE_PER_CLUSTER
626 ", re-read remote value is %llx"
627 #endif /* UNCORE_PER_CLUSTER */
628 , ctr,
629 monid, mon->um_snaps[ctr], snap,
630 uncmon_get_curid() == monid ? "local" : "remote",
631 mon->um_last_read_id == monid ? "local" : "remote",
632 mon->um_last_read_id,
633 mon->um_read_since_sleep ? "" : ", first read since sleep"
634 #if UNCORE_PER_CLUSTER
635 , remote_value
636 #endif /* UNCORE_PER_CLUSTER */
637 );
638 #else /* MACH_ASSERT */
639 snap = mon->um_snaps[ctr];
640 #endif /* !MACH_ASSERT */
641 }
642 mon->um_counts[ctr] += snap - mon->um_snaps[ctr];
643 mon->um_snaps[ctr] = snap;
644 }
645 }
646
647 static inline void
uncmon_program_events_locked_l(unsigned int monid)648 uncmon_program_events_locked_l(unsigned int monid)
649 {
650 /*
651 * UPMESR[01] is the event selection register that determines which event a
652 * counter will count.
653 */
654 CTRL_REG_SET("UPMESR0_EL1", uncore_config.uc_events.uce_regs[0]);
655
656 #if UNCORE_NCTRS > 8
657 CTRL_REG_SET("UPMESR1_EL1", uncore_config.uc_events.uce_regs[1]);
658 #endif /* UNCORE_NCTRS > 8 */
659
660 /*
661 * UPMECM[0123] are the event core masks for each counter -- whether or not
662 * that counter counts events generated by an agent. These are set to all
663 * ones so the uncore counters count events from all cores.
664 *
665 * The bits are based off the start of the cluster -- e.g. even if a core
666 * has a CPU ID of 4, it might be the first CPU in a cluster. Shift the
667 * registers right by the ID of the first CPU in the cluster.
668 */
669 CTRL_REG_SET("UPMECM0_EL1",
670 uncore_config.uc_cpu_masks[monid].uccm_regs[0]);
671 CTRL_REG_SET("UPMECM1_EL1",
672 uncore_config.uc_cpu_masks[monid].uccm_regs[1]);
673
674 #if UNCORE_NCTRS > 8
675 CTRL_REG_SET("UPMECM2_EL1",
676 uncore_config.uc_cpu_masks[monid].uccm_regs[2]);
677 CTRL_REG_SET("UPMECM3_EL1",
678 uncore_config.uc_cpu_masks[monid].uccm_regs[3]);
679 #endif /* UNCORE_NCTRS > 8 */
680 }
681
682 #if UNCORE_PER_CLUSTER
683
684 static inline void
uncmon_program_events_locked_r(unsigned int monid)685 uncmon_program_events_locked_r(unsigned int monid)
686 {
687 const uintptr_t upmesr_offs[2] = {[0] = 0x41b0, [1] = 0x41b8, };
688
689 for (unsigned int i = 0; i < sizeof(upmesr_offs) / sizeof(upmesr_offs[0]);
690 i++) {
691 *(uint64_t *)(cpm_impl[monid] + upmesr_offs[i]) =
692 uncore_config.uc_events.uce_regs[i];
693 }
694
695 const uintptr_t upmecm_offs[4] = {
696 [0] = 0x4190, [1] = 0x4198, [2] = 0x41a0, [3] = 0x41a8,
697 };
698
699 for (unsigned int i = 0; i < sizeof(upmecm_offs) / sizeof(upmecm_offs[0]);
700 i++) {
701 *(uint64_t *)(cpm_impl[monid] + upmecm_offs[i]) =
702 uncore_config.uc_cpu_masks[monid].uccm_regs[i];
703 }
704 }
705
706 #endif /* UNCORE_PER_CLUSTER */
707
708 static void
uncmon_clear_int_locked_l(__unused unsigned int monid)709 uncmon_clear_int_locked_l(__unused unsigned int monid)
710 {
711 __builtin_arm_wsr64("UPMSR_EL1", 0);
712 }
713
714 #if UNCORE_PER_CLUSTER
715
716 static void
uncmon_clear_int_locked_r(unsigned int monid)717 uncmon_clear_int_locked_r(unsigned int monid)
718 {
719 const uintptr_t upmsr_off = 0x41c0;
720 *(uint64_t *)(cpm_impl[monid] + upmsr_off) = 0;
721 }
722
723 #endif /* UNCORE_PER_CLUSTER */
724
725 /*
726 * Get the PMI mask for the provided `monid` -- that is, the bitmap of CPUs
727 * that should be sent PMIs for a particular monitor.
728 */
729 static uint64_t
uncmon_get_pmi_mask(unsigned int monid)730 uncmon_get_pmi_mask(unsigned int monid)
731 {
732 uint64_t pmi_mask = uncore_pmi_mask;
733
734 #if UNCORE_PER_CLUSTER
735 pmi_mask &= topology_info->clusters[monid].cpu_mask;
736 #else /* UNCORE_PER_CLUSTER */
737 #pragma unused(monid)
738 #endif /* !UNCORE_PER_CLUSTER */
739
740 return pmi_mask;
741 }
742
743 /*
744 * Initialization routines for the uncore counters.
745 */
746
747 static void
uncmon_init_locked_l(unsigned int monid)748 uncmon_init_locked_l(unsigned int monid)
749 {
750 /*
751 * UPMPCM defines the PMI core mask for the UPMCs -- which cores should
752 * receive interrupts on overflow.
753 */
754 CTRL_REG_SET("UPMPCM_EL1", uncmon_get_pmi_mask(monid));
755 uncmon_set_counting_locked_l(monid,
756 mt_uncore_enabled ? uncore_active_ctrs : 0);
757 }
758
759 #if UNCORE_PER_CLUSTER
760
761 static uintptr_t acc_impl[MAX_NMONITORS] = {};
762
763 static void
uncmon_init_locked_r(unsigned int monid)764 uncmon_init_locked_r(unsigned int monid)
765 {
766 const uintptr_t upmpcm_off = 0x1010;
767
768 *(uint64_t *)(acc_impl[monid] + upmpcm_off) = uncmon_get_pmi_mask(monid);
769 uncmon_set_counting_locked_r(monid,
770 mt_uncore_enabled ? uncore_active_ctrs : 0);
771 }
772
773 #endif /* UNCORE_PER_CLUSTER */
774
775 /*
776 * Initialize the uncore device for monotonic.
777 */
778 static int
uncore_init(__unused mt_device_t dev)779 uncore_init(__unused mt_device_t dev)
780 {
781 #if HAS_UNCORE_CTRS
782 assert(MT_NDEVS > 0);
783 mt_devices[MT_NDEVS - 1].mtd_nmonitors = (uint8_t)uncore_nmonitors();
784 #endif
785
786 #if DEVELOPMENT || DEBUG
787 /*
788 * Development and debug kernels observe the `uncore_pmi_mask` boot-arg,
789 * allowing PMIs to be routed to the CPUs present in the supplied bitmap.
790 * Do some sanity checks on the value provided.
791 */
792 bool parsed_arg = PE_parse_boot_argn("uncore_pmi_mask", &uncore_pmi_mask,
793 sizeof(uncore_pmi_mask));
794 if (parsed_arg) {
795 #if UNCORE_PER_CLUSTER
796 if (__builtin_popcount(uncore_pmi_mask) != (int)uncore_nmonitors()) {
797 panic("monotonic: invalid uncore PMI mask 0x%x", uncore_pmi_mask);
798 }
799 for (unsigned int i = 0; i < uncore_nmonitors(); i++) {
800 if (__builtin_popcountll(uncmon_get_pmi_mask(i)) != 1) {
801 panic("monotonic: invalid uncore PMI CPU for cluster %d in mask 0x%x",
802 i, uncore_pmi_mask);
803 }
804 }
805 #else /* UNCORE_PER_CLUSTER */
806 if (__builtin_popcount(uncore_pmi_mask) != 1) {
807 panic("monotonic: invalid uncore PMI mask 0x%x", uncore_pmi_mask);
808 }
809 #endif /* !UNCORE_PER_CLUSTER */
810 } else
811 #endif /* DEVELOPMENT || DEBUG */
812 {
813 /* arbitrarily route to core 0 in each cluster */
814 uncore_pmi_mask |= 1;
815 }
816 assert(uncore_pmi_mask != 0);
817
818 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
819 #if UNCORE_PER_CLUSTER
820 ml_topology_cluster_t *cluster = &topology_info->clusters[monid];
821 cpm_impl[monid] = (uintptr_t)cluster->cpm_IMPL_regs;
822 acc_impl[monid] = (uintptr_t)cluster->acc_IMPL_regs;
823 assert(cpm_impl[monid] != 0 && acc_impl[monid] != 0);
824 #endif /* UNCORE_PER_CLUSTER */
825
826 struct uncore_monitor *mon = &uncore_monitors[monid];
827 lck_spin_init(&mon->um_lock, &mt_lock_grp, LCK_ATTR_NULL);
828 }
829
830 mt_uncore_initted = true;
831
832 return 0;
833 }
834
835 /*
836 * Support for monotonic's mtd_read function.
837 */
838
839 static void
uncmon_read_all_counters(unsigned int monid,uint64_t ctr_mask,uint64_t * counts)840 uncmon_read_all_counters(unsigned int monid, uint64_t ctr_mask, uint64_t *counts)
841 {
842 struct uncore_monitor *mon = &uncore_monitors[monid];
843
844 int intrs_en = uncmon_lock(mon);
845
846 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
847 if (ctr_mask & (1ULL << ctr)) {
848 if (!mon->um_sleeping) {
849 uncmon_update_locked(monid, ctr);
850 }
851 counts[ctr] = mon->um_counts[ctr];
852 }
853 }
854 #if MACH_ASSERT
855 mon->um_read_since_sleep = true;
856 #endif /* MACH_ASSERT */
857
858 uncmon_unlock(mon, intrs_en);
859 }
860
861 /*
862 * Read all monitor's counters.
863 */
864 static int
uncore_read(uint64_t ctr_mask,uint64_t * counts_out)865 uncore_read(uint64_t ctr_mask, uint64_t *counts_out)
866 {
867 assert(ctr_mask != 0);
868 assert(counts_out != NULL);
869
870 if (!uncore_active_ctrs) {
871 return EPWROFF;
872 }
873 if (ctr_mask & ~uncore_active_ctrs) {
874 return EINVAL;
875 }
876
877 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
878 /*
879 * Find this monitor's starting offset into the `counts_out` array.
880 */
881 uint64_t *counts = counts_out + (UNCORE_NCTRS * monid);
882 uncmon_read_all_counters(monid, ctr_mask, counts);
883 }
884
885 return 0;
886 }
887
888 /*
889 * Support for monotonic's mtd_add function.
890 */
891
892 /*
893 * Add an event to the current uncore configuration. This doesn't take effect
894 * until the counters are enabled again, so there's no need to involve the
895 * monitors.
896 */
897 static int
uncore_add(struct monotonic_config * config,uint32_t * ctr_out)898 uncore_add(struct monotonic_config *config, uint32_t *ctr_out)
899 {
900 if (mt_uncore_enabled) {
901 return EBUSY;
902 }
903
904 uint8_t selector = (uint8_t)config->event;
905 uint32_t available = ~uncore_active_ctrs & config->allowed_ctr_mask;
906
907 if (available == 0) {
908 return ENOSPC;
909 }
910
911 if (!cpc_event_allowed(CPC_HW_UPMU, selector)) {
912 return EPERM;
913 }
914
915 uint32_t valid_ctrs = (UINT32_C(1) << UNCORE_NCTRS) - 1;
916 if ((available & valid_ctrs) == 0) {
917 return E2BIG;
918 }
919 /*
920 * Clear the UPMCs the first time an event is added.
921 */
922 if (uncore_active_ctrs == 0) {
923 /*
924 * Suspend powerdown until the next reset.
925 */
926 assert(!mt_uncore_suspended_cpd);
927 suspend_cluster_powerdown();
928 mt_uncore_suspended_cpd = true;
929
930 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
931 struct uncore_monitor *mon = &uncore_monitors[monid];
932
933 int intrs_en = uncmon_lock(mon);
934 bool remote = uncmon_is_remote(monid);
935
936 if (!mon->um_sleeping) {
937 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
938 if (remote) {
939 #if UNCORE_PER_CLUSTER
940 uncmon_write_counter_locked_r(monid, ctr, 0);
941 #endif /* UNCORE_PER_CLUSTER */
942 } else {
943 uncmon_write_counter_locked_l(monid, ctr, 0);
944 }
945 }
946 }
947 memset(&mon->um_snaps, 0, sizeof(mon->um_snaps));
948 memset(&mon->um_counts, 0, sizeof(mon->um_counts));
949 uncmon_unlock(mon, intrs_en);
950 }
951 }
952
953 uint32_t ctr = __builtin_ffsll(available) - 1;
954
955 uncore_active_ctrs |= UINT64_C(1) << ctr;
956 uncore_config.uc_events.uce_ctrs[ctr] = selector;
957 uint64_t cpu_mask = UINT64_MAX;
958 if (config->cpu_mask != 0) {
959 cpu_mask = config->cpu_mask;
960 }
961 for (unsigned int i = 0; i < uncore_nmonitors(); i++) {
962 #if UNCORE_PER_CLUSTER
963 const unsigned int shift = topology_info->clusters[i].first_cpu_id;
964 #else /* UNCORE_PER_CLUSTER */
965 const unsigned int shift = 0;
966 #endif /* !UNCORE_PER_CLUSTER */
967 uncore_config.uc_cpu_masks[i].uccm_masks[ctr] = (uint16_t)(cpu_mask >> shift);
968 }
969
970 *ctr_out = ctr;
971 return 0;
972 }
973
974 /*
975 * Support for monotonic's mtd_reset function.
976 */
977
978 /*
979 * Reset all configuration and disable the counters if they're currently
980 * counting.
981 */
982 static void
uncore_reset(void)983 uncore_reset(void)
984 {
985 mt_uncore_enabled = false;
986
987 if (!mt_uncore_suspended_cpd) {
988 /* If we haven't already suspended CPD, we need to do so now to ensure we can issue remote reads
989 * to every cluster. */
990 suspend_cluster_powerdown();
991 mt_uncore_suspended_cpd = true;
992 }
993
994 if (mt_owns_counters()) {
995 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
996 struct uncore_monitor *mon = &uncore_monitors[monid];
997
998 int intrs_en = uncmon_lock(mon);
999 bool remote = uncmon_is_remote(monid);
1000 if (!mon->um_sleeping) {
1001 if (remote) {
1002 #if UNCORE_PER_CLUSTER
1003 uncmon_set_counting_locked_r(monid, 0);
1004 #endif /* UNCORE_PER_CLUSTER */
1005 } else {
1006 uncmon_set_counting_locked_l(monid, 0);
1007 }
1008
1009 for (int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1010 if (uncore_active_ctrs & (1U << ctr)) {
1011 if (remote) {
1012 #if UNCORE_PER_CLUSTER
1013 uncmon_write_counter_locked_r(monid, ctr, 0);
1014 #endif /* UNCORE_PER_CLUSTER */
1015 } else {
1016 uncmon_write_counter_locked_l(monid, ctr, 0);
1017 }
1018 }
1019 }
1020 }
1021
1022 memset(&mon->um_snaps, 0, sizeof(mon->um_snaps));
1023 memset(&mon->um_counts, 0, sizeof(mon->um_counts));
1024 if (!mon->um_sleeping) {
1025 if (remote) {
1026 #if UNCORE_PER_CLUSTER
1027 uncmon_clear_int_locked_r(monid);
1028 #endif /* UNCORE_PER_CLUSTER */
1029 } else {
1030 uncmon_clear_int_locked_l(monid);
1031 }
1032 }
1033
1034 uncmon_unlock(mon, intrs_en);
1035 }
1036 }
1037
1038 uncore_active_ctrs = 0;
1039 memset(&uncore_config, 0, sizeof(uncore_config));
1040
1041 if (mt_owns_counters()) {
1042 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1043 struct uncore_monitor *mon = &uncore_monitors[monid];
1044
1045 int intrs_en = uncmon_lock(mon);
1046 bool remote = uncmon_is_remote(monid);
1047 if (!mon->um_sleeping) {
1048 if (remote) {
1049 #if UNCORE_PER_CLUSTER
1050 uncmon_program_events_locked_r(monid);
1051 #endif /* UNCORE_PER_CLUSTER */
1052 } else {
1053 uncmon_program_events_locked_l(monid);
1054 }
1055 }
1056 uncmon_unlock(mon, intrs_en);
1057 }
1058 }
1059
1060 /* After reset, no counters should be active, so we can allow powerdown again */
1061 if (mt_uncore_suspended_cpd) {
1062 resume_cluster_powerdown();
1063 mt_uncore_suspended_cpd = false;
1064 }
1065 }
1066
1067 /*
1068 * Support for monotonic's mtd_enable function.
1069 */
1070
1071 static void
uncmon_set_enabled_l_locked(unsigned int monid,bool enable)1072 uncmon_set_enabled_l_locked(unsigned int monid, bool enable)
1073 {
1074 struct uncore_monitor *mon = &uncore_monitors[monid];
1075 #pragma unused(mon)
1076 LCK_SPIN_ASSERT(&mon->um_lock, LCK_ASSERT_OWNED);
1077
1078 if (enable) {
1079 uncmon_init_locked_l(monid);
1080 uncmon_program_events_locked_l(monid);
1081 uncmon_set_counting_locked_l(monid, uncore_active_ctrs);
1082 } else {
1083 uncmon_set_counting_locked_l(monid, 0);
1084 }
1085 }
1086
1087 #if UNCORE_PER_CLUSTER
1088
1089 static void
uncmon_set_enabled_r_locked(unsigned int monid,bool enable)1090 uncmon_set_enabled_r_locked(unsigned int monid, bool enable)
1091 {
1092 struct uncore_monitor *mon = &uncore_monitors[monid];
1093 #pragma unused(mon)
1094 LCK_SPIN_ASSERT(&mon->um_lock, LCK_ASSERT_OWNED);
1095
1096 if (!mon->um_sleeping) {
1097 if (enable) {
1098 uncmon_init_locked_r(monid);
1099 uncmon_program_events_locked_r(monid);
1100 uncmon_set_counting_locked_r(monid, uncore_active_ctrs);
1101 } else {
1102 uncmon_set_counting_locked_r(monid, 0);
1103 }
1104 }
1105 }
1106
1107 #endif /* UNCORE_PER_CLUSTER */
1108
1109 static void
uncore_set_enabled(bool enable)1110 uncore_set_enabled(bool enable)
1111 {
1112 mt_uncore_enabled = enable;
1113
1114 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1115 struct uncore_monitor *mon = &uncore_monitors[monid];
1116 int intrs_en = uncmon_lock(mon);
1117 if (uncmon_is_remote(monid)) {
1118 #if UNCORE_PER_CLUSTER
1119 uncmon_set_enabled_r_locked(monid, enable);
1120 #endif /* UNCORE_PER_CLUSTER */
1121 } else {
1122 uncmon_set_enabled_l_locked(monid, enable);
1123 }
1124 uncmon_unlock(mon, intrs_en);
1125 }
1126 }
1127
1128 /*
1129 * Hooks in the machine layer.
1130 */
1131
1132 static void
uncore_fiq(uint64_t upmsr)1133 uncore_fiq(uint64_t upmsr)
1134 {
1135 /*
1136 * Determine which counters overflowed.
1137 */
1138 uint64_t disable_ctr_mask = (upmsr & UPMSR_OVF_MASK) >> UPMSR_OVF_POS;
1139 /* should not receive interrupts from inactive counters */
1140 assert(!(disable_ctr_mask & ~uncore_active_ctrs));
1141
1142 if (uncore_active_ctrs == 0) {
1143 return;
1144 }
1145
1146 unsigned int monid = uncmon_get_curid();
1147 struct uncore_monitor *mon = &uncore_monitors[monid];
1148
1149 int intrs_en = uncmon_lock(mon);
1150
1151 /*
1152 * Disable any counters that overflowed.
1153 */
1154 uncmon_set_counting_locked_l(monid,
1155 uncore_active_ctrs & ~disable_ctr_mask);
1156
1157 /*
1158 * With the overflowing counters disabled, capture their counts and reset
1159 * the UPMCs and their snapshots to 0.
1160 */
1161 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1162 if (UPMSR_OVF(upmsr, ctr)) {
1163 uncmon_update_locked(monid, ctr);
1164 mon->um_snaps[ctr] = 0;
1165 uncmon_write_counter_locked_l(monid, ctr, 0);
1166 }
1167 }
1168
1169 /*
1170 * Acknowledge the interrupt, now that any overflowed PMCs have been reset.
1171 */
1172 uncmon_clear_int_locked_l(monid);
1173
1174 /*
1175 * Re-enable all active counters.
1176 */
1177 uncmon_set_counting_locked_l(monid, uncore_active_ctrs);
1178
1179 uncmon_unlock(mon, intrs_en);
1180 }
1181
1182 static void
uncore_save(void)1183 uncore_save(void)
1184 {
1185 if (!uncore_active_ctrs) {
1186 return;
1187 }
1188
1189 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1190 struct uncore_monitor *mon = &uncore_monitors[monid];
1191 int intrs_en = uncmon_lock(mon);
1192
1193 if (mt_uncore_enabled) {
1194 if (uncmon_is_remote(monid)) {
1195 #if UNCORE_PER_CLUSTER
1196 uncmon_set_counting_locked_r(monid, 0);
1197 #endif /* UNCORE_PER_CLUSTER */
1198 } else {
1199 uncmon_set_counting_locked_l(monid, 0);
1200 }
1201 }
1202
1203 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1204 if (uncore_active_ctrs & (1U << ctr)) {
1205 uncmon_update_locked(monid, ctr);
1206 mon->um_snaps[ctr] = 0;
1207 uncmon_write_counter_locked_l(monid, ctr, 0);
1208 }
1209 }
1210
1211 mon->um_sleeping = true;
1212 uncmon_unlock(mon, intrs_en);
1213 }
1214 }
1215
1216 static void
uncore_restore(void)1217 uncore_restore(void)
1218 {
1219 if (!uncore_active_ctrs) {
1220 return;
1221 }
1222 /* Ensure interrupts disabled before reading uncmon_get_curid */
1223 bool intr = ml_set_interrupts_enabled(false);
1224 unsigned int curmonid = uncmon_get_curid();
1225
1226 struct uncore_monitor *mon = &uncore_monitors[curmonid];
1227 int intrs_en = uncmon_lock(mon);
1228 if (!mon->um_sleeping) {
1229 goto out;
1230 }
1231
1232 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1233 if (uncore_active_ctrs & (1U << ctr)) {
1234 uncmon_write_counter_locked_l(curmonid, ctr, mon->um_snaps[ctr]);
1235 }
1236 }
1237 uncmon_program_events_locked_l(curmonid);
1238 uncmon_init_locked_l(curmonid);
1239 mon->um_sleeping = false;
1240 #if MACH_ASSERT
1241 mon->um_read_since_sleep = false;
1242 #endif /* MACH_ASSERT */
1243
1244 out:
1245 uncmon_unlock(mon, intrs_en);
1246 ml_set_interrupts_enabled(intr);
1247 }
1248
1249 #endif /* HAS_UNCORE_CTRS */
1250
1251 #pragma mark common hooks
1252
1253 void
mt_early_init(void)1254 mt_early_init(void)
1255 {
1256 topology_info = ml_get_topology_info();
1257 }
1258
1259 void
mt_cpu_idle(cpu_data_t * cpu)1260 mt_cpu_idle(cpu_data_t *cpu)
1261 {
1262 core_idle(cpu);
1263 }
1264
1265 void
mt_cpu_run(cpu_data_t * cpu)1266 mt_cpu_run(cpu_data_t *cpu)
1267 {
1268 struct mt_cpu *mtc;
1269
1270 assert(cpu != NULL);
1271 assert(ml_get_interrupts_enabled() == FALSE);
1272
1273 mtc = &cpu->cpu_monotonic;
1274
1275 for (int i = 0; i < MT_CORE_NFIXED; i++) {
1276 mt_core_set_snap(i, mtc->mtc_snaps[i]);
1277 }
1278
1279 /* re-enable the counters */
1280 core_init_execution_modes();
1281
1282 core_set_enabled();
1283 }
1284
1285 void
mt_cpu_down(cpu_data_t * cpu)1286 mt_cpu_down(cpu_data_t *cpu)
1287 {
1288 mt_cpu_idle(cpu);
1289 }
1290
1291 void
mt_cpu_up(cpu_data_t * cpu)1292 mt_cpu_up(cpu_data_t *cpu)
1293 {
1294 mt_cpu_run(cpu);
1295 }
1296
1297 void
mt_sleep(void)1298 mt_sleep(void)
1299 {
1300 #if HAS_UNCORE_CTRS
1301 uncore_save();
1302 #endif /* HAS_UNCORE_CTRS */
1303 }
1304
1305 void
mt_wake_per_core(void)1306 mt_wake_per_core(void)
1307 {
1308 #if HAS_UNCORE_CTRS
1309 if (mt_uncore_initted) {
1310 uncore_restore();
1311 }
1312 #endif /* HAS_UNCORE_CTRS */
1313 }
1314
1315 uint64_t
mt_count_pmis(void)1316 mt_count_pmis(void)
1317 {
1318 uint64_t npmis = 0;
1319 for (unsigned int i = 0; i < topology_info->num_cpus; i++) {
1320 cpu_data_t *cpu = (cpu_data_t *)CpuDataEntries[topology_info->cpus[i].cpu_id].cpu_data_vaddr;
1321 npmis += cpu->cpu_monotonic.mtc_npmis;
1322 }
1323 return npmis;
1324 }
1325
1326 static void
mt_cpu_pmi(cpu_data_t * cpu,uint64_t pmcr0)1327 mt_cpu_pmi(cpu_data_t *cpu, uint64_t pmcr0)
1328 {
1329 assert(cpu != NULL);
1330 assert(ml_get_interrupts_enabled() == FALSE);
1331
1332 __builtin_arm_wsr64("PMCR0_EL1", PMCR0_INIT);
1333 /*
1334 * Ensure the CPMU has flushed any increments at this point, so PMSR is up
1335 * to date.
1336 */
1337 __builtin_arm_isb(ISB_SY);
1338
1339 cpu->cpu_monotonic.mtc_npmis += 1;
1340 cpu->cpu_stat.pmi_cnt_wake += 1;
1341
1342 #if MONOTONIC_DEBUG
1343 if (!PMCR0_PMI(pmcr0)) {
1344 kprintf("monotonic: mt_cpu_pmi but no PMI (PMCR0 = %#llx)\n",
1345 pmcr0);
1346 }
1347 #else /* MONOTONIC_DEBUG */
1348 #pragma unused(pmcr0)
1349 #endif /* !MONOTONIC_DEBUG */
1350
1351 uint64_t pmsr = __builtin_arm_rsr64("PMSR_EL1");
1352
1353 #if MONOTONIC_DEBUG
1354 printf("monotonic: cpu = %d, PMSR = 0x%llx, PMCR0 = 0x%llx\n",
1355 cpu_number(), pmsr, pmcr0);
1356 #endif /* MONOTONIC_DEBUG */
1357
1358 #if MACH_ASSERT
1359 uint64_t handled = 0;
1360 #endif /* MACH_ASSERT */
1361
1362 /*
1363 * monotonic handles any fixed counter PMIs.
1364 */
1365 for (unsigned int i = 0; i < MT_CORE_NFIXED; i++) {
1366 if ((pmsr & PMSR_OVF(i)) == 0) {
1367 continue;
1368 }
1369
1370 #if MACH_ASSERT
1371 handled |= 1ULL << i;
1372 #endif /* MACH_ASSERT */
1373 uint64_t count = mt_cpu_update_count(cpu, i);
1374 cpu->cpu_monotonic.mtc_counts[i] += count;
1375 mt_core_set_snap(i, mt_core_reset_values[i]);
1376 cpu->cpu_monotonic.mtc_snaps[i] = mt_core_reset_values[i];
1377
1378 if (mt_microstackshots && mt_microstackshot_ctr == i) {
1379 bool user_mode = false;
1380 arm_saved_state_t *state = get_user_regs(current_thread());
1381 if (state) {
1382 user_mode = PSR64_IS_USER(get_saved_state_cpsr(state));
1383 }
1384 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
1385 mt_microstackshot_ctr, user_mode);
1386 mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
1387 } else if (mt_debug) {
1388 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
1389 i, count);
1390 }
1391 }
1392
1393 /*
1394 * KPC handles the configurable counter PMIs.
1395 */
1396 for (unsigned int i = MT_CORE_NFIXED; i < CORE_NCTRS; i++) {
1397 if (pmsr & PMSR_OVF(i)) {
1398 #if MACH_ASSERT
1399 handled |= 1ULL << i;
1400 #endif /* MACH_ASSERT */
1401 extern void kpc_pmi_handler(unsigned int ctr);
1402 kpc_pmi_handler(i);
1403 }
1404 }
1405
1406 #if MACH_ASSERT
1407 uint64_t pmsr_after_handling = __builtin_arm_rsr64("PMSR_EL1");
1408 if (pmsr_after_handling != 0) {
1409 unsigned int first_ctr_ovf = __builtin_ffsll(pmsr_after_handling) - 1;
1410 uint64_t count = 0;
1411 const char *extra = "";
1412 if (first_ctr_ovf >= CORE_NCTRS) {
1413 extra = " (invalid counter)";
1414 } else {
1415 count = mt_core_snap(first_ctr_ovf);
1416 }
1417
1418 panic("monotonic: PMI status not cleared on exit from handler, "
1419 "PMSR = 0x%llx HANDLE -> -> 0x%llx, handled 0x%llx, "
1420 "PMCR0 = 0x%llx, PMC%d = 0x%llx%s", pmsr, pmsr_after_handling,
1421 handled, __builtin_arm_rsr64("PMCR0_EL1"), first_ctr_ovf, count, extra);
1422 }
1423 #endif /* MACH_ASSERT */
1424
1425 core_set_enabled();
1426 }
1427
1428 #if CPMU_AIC_PMI
1429 void
mt_cpmu_aic_pmi(cpu_id_t source)1430 mt_cpmu_aic_pmi(cpu_id_t source)
1431 {
1432 struct cpu_data *curcpu = getCpuDatap();
1433 if (source != curcpu->interrupt_nub) {
1434 panic("monotonic: PMI from IOCPU %p delivered to %p", source,
1435 curcpu->interrupt_nub);
1436 }
1437 mt_cpu_pmi(curcpu, __builtin_arm_rsr64("PMCR0_EL1"));
1438 }
1439 #endif /* CPMU_AIC_PMI */
1440
1441 void
mt_fiq(void * cpu,uint64_t pmcr0,uint64_t upmsr)1442 mt_fiq(void *cpu, uint64_t pmcr0, uint64_t upmsr)
1443 {
1444 #if CPMU_AIC_PMI
1445 #pragma unused(cpu, pmcr0)
1446 #else /* CPMU_AIC_PMI */
1447 mt_cpu_pmi(cpu, pmcr0);
1448 #endif /* !CPMU_AIC_PMI */
1449
1450 #if HAS_UNCORE_CTRS
1451 if (upmsr != 0) {
1452 uncore_fiq(upmsr);
1453 }
1454 #else /* HAS_UNCORE_CTRS */
1455 #pragma unused(upmsr)
1456 #endif /* !HAS_UNCORE_CTRS */
1457 }
1458
1459 void
mt_ownership_change(bool available)1460 mt_ownership_change(bool available)
1461 {
1462 #if HAS_UNCORE_CTRS
1463 /*
1464 * No need to take the lock here, as this is only manipulated in the UPMU
1465 * when the current task already owns the counters and is on its way out.
1466 */
1467 if (!available && uncore_active_ctrs) {
1468 uncore_reset();
1469 }
1470 #else
1471 #pragma unused(available)
1472 #endif /* HAS_UNCORE_CTRS */
1473 }
1474
1475 static uint32_t mt_xc_sync;
1476
1477 static void
mt_microstackshot_start_remote(__unused void * arg)1478 mt_microstackshot_start_remote(__unused void *arg)
1479 {
1480 cpu_data_t *cpu = getCpuDatap();
1481
1482 __builtin_arm_wsr64("PMCR0_EL1", PMCR0_INIT);
1483
1484 for (int i = 0; i < MT_CORE_NFIXED; i++) {
1485 uint64_t count = mt_cpu_update_count(cpu, i);
1486 cpu->cpu_monotonic.mtc_counts[i] += count;
1487 mt_core_set_snap(i, mt_core_reset_values[i]);
1488 cpu->cpu_monotonic.mtc_snaps[i] = mt_core_reset_values[i];
1489 }
1490
1491 core_set_enabled();
1492
1493 if (os_atomic_dec(&mt_xc_sync, relaxed) == 0) {
1494 thread_wakeup((event_t)&mt_xc_sync);
1495 }
1496 }
1497
1498 int
mt_microstackshot_start_arch(uint64_t period)1499 mt_microstackshot_start_arch(uint64_t period)
1500 {
1501 uint64_t reset_value = 0;
1502 int ovf = os_sub_overflow(CTR_MAX, period, &reset_value);
1503 if (ovf) {
1504 return ERANGE;
1505 }
1506
1507 mt_core_reset_values[mt_microstackshot_ctr] = reset_value;
1508 cpu_broadcast_xcall(&mt_xc_sync, TRUE, mt_microstackshot_start_remote,
1509 mt_microstackshot_start_remote /* cannot pass NULL */);
1510 return 0;
1511 }
1512
1513 #pragma mark dev nodes
1514
1515 struct mt_device mt_devices[] = {
1516 [0] = {
1517 .mtd_name = "core",
1518 .mtd_init = core_init,
1519 },
1520 #if HAS_UNCORE_CTRS
1521 [1] = {
1522 .mtd_name = "uncore",
1523 .mtd_init = uncore_init,
1524 .mtd_add = uncore_add,
1525 .mtd_reset = uncore_reset,
1526 .mtd_enable = uncore_set_enabled,
1527 .mtd_read = uncore_read,
1528
1529 .mtd_ncounters = UNCORE_NCTRS,
1530 }
1531 #endif /* HAS_UNCORE_CTRS */
1532 };
1533
1534 static_assert(
1535 (sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
1536 "MT_NDEVS macro should be same as the length of mt_devices");
1537