1 /*
2 * Copyright (c) 2017-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/cpu_data_internal.h>
30 #include <arm/machine_routines.h>
31 #include <arm64/monotonic.h>
32 #include <kern/assert.h>
33 #include <kern/cpc.h>
34 #include <kern/debug.h> /* panic */
35 #include <kern/kpc.h>
36 #include <kern/monotonic.h>
37 #include <machine/atomic.h>
38 #include <machine/limits.h> /* CHAR_BIT */
39 #include <os/overflow.h>
40 #include <pexpert/arm64/board_config.h>
41 #include <pexpert/device_tree.h> /* SecureDTFindEntry */
42 #include <pexpert/pexpert.h>
43 #include <stdatomic.h>
44 #include <stdint.h>
45 #include <string.h>
46 #include <sys/errno.h>
47 #include <sys/monotonic.h>
48
49 /*
50 * Ensure that control registers read back what was written under MACH_ASSERT
51 * kernels.
52 *
53 * A static inline function cannot be used due to passing the register through
54 * the builtin -- it requires a constant string as its first argument, since
55 * MSRs registers are encoded as an immediate in the instruction.
56 */
57 #if MACH_ASSERT
58 #define CTRL_REG_SET(reg, val) do { \
59 __builtin_arm_wsr64((reg), (val)); \
60 uint64_t __check_reg = __builtin_arm_rsr64((reg)); \
61 if (__check_reg != (val)) { \
62 panic("value written to %s was not read back (wrote %llx, read %llx)", \
63 #reg, (val), __check_reg); \
64 } \
65 } while (0)
66 #else /* MACH_ASSERT */
67 #define CTRL_REG_SET(reg, val) __builtin_arm_wsr64((reg), (val))
68 #endif /* MACH_ASSERT */
69
70 #pragma mark core counters
71
72 const bool mt_core_supported = true;
73
74 static const ml_topology_info_t *topology_info;
75
76 /*
77 * PMC[0-1] are the 48/64-bit fixed counters -- S3_2_C15_C0_0 is cycles and S3_2_C15_C1_0 is
78 * instructions (see arm64/monotonic.h).
79 *
80 * S3_2_C15_C2_0+ are currently handled by kpc.
81 */
82
83 #if CPMU_64BIT_PMCS
84 #define PMC_WIDTH (63)
85 #else // UPMU_64BIT_PMCS
86 #define PMC_WIDTH (47)
87 #endif // !UPMU_64BIT_PMCS
88
89 #define CTR_MAX ((UINT64_C(1) << PMC_WIDTH) - 1)
90
91 #define CYCLES 0
92 #define INSTRS 1
93
94 /*
95 * S3_2_C15_C0_0's offset into a core's PIO range.
96 *
97 * This allows cores to remotely query another core's counters.
98 */
99
100 #define PIO_S3_2_C15_C0_0_OFFSET (0x200)
101
102 /*
103 * The offset of the counter in the configuration registers. Post-Hurricane
104 * devices have additional counters that need a larger shift than the original
105 * counters.
106 *
107 * XXX For now, just support the lower-numbered counters.
108 */
109 #define CTR_POS(CTR) (CTR)
110
111 /*
112 * PMCR0 is the main control register for the performance monitor. It
113 * controls whether the counters are enabled, how they deliver interrupts, and
114 * other features.
115 */
116
117 #define PMCR0_CTR_EN(CTR) (UINT64_C(1) << CTR_POS(CTR))
118 #define PMCR0_FIXED_EN (PMCR0_CTR_EN(CYCLES) | PMCR0_CTR_EN(INSTRS))
119 /* how interrupts are delivered on a PMI */
120 enum {
121 PMCR0_INTGEN_OFF = 0,
122 PMCR0_INTGEN_PMI = 1,
123 PMCR0_INTGEN_AIC = 2,
124 PMCR0_INTGEN_HALT = 3,
125 PMCR0_INTGEN_FIQ = 4,
126 };
127 #define PMCR0_INTGEN_SET(X) ((uint64_t)(X) << 8)
128
129 #if CPMU_AIC_PMI
130 #define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_AIC)
131 #else /* CPMU_AIC_PMI */
132 #define PMCR0_INTGEN_INIT PMCR0_INTGEN_SET(PMCR0_INTGEN_FIQ)
133 #endif /* !CPMU_AIC_PMI */
134
135 #define PMCR0_PMI_SHIFT (12)
136 #define PMCR0_CTR_GE8_PMI_SHIFT (44)
137 #define PMCR0_PMI_EN(CTR) (UINT64_C(1) << (PMCR0_PMI_SHIFT + CTR_POS(CTR)))
138 /* fixed counters are always counting */
139 #define PMCR0_PMI_INIT (PMCR0_PMI_EN(CYCLES) | PMCR0_PMI_EN(INSTRS))
140 /* disable counting on a PMI */
141 #define PMCR0_DISCNT_EN (UINT64_C(1) << 20)
142 /* block PMIs until ERET retires */
143 #define PMCR0_WFRFE_EN (UINT64_C(1) << 22)
144 /* count global (not just core-local) L2C events */
145 #define PMCR0_L2CGLOBAL_EN (UINT64_C(1) << 23)
146 /* user mode access to configuration registers */
147 #define PMCR0_USEREN_EN (UINT64_C(1) << 30)
148 #define PMCR0_CTR_GE8_EN_SHIFT (32)
149
150 #if HAS_CPMU_PC_CAPTURE
151 #define PMCR0_PCC_INIT (UINT64_C(0x7) << 24)
152 #else /* HAS_CPMU_PC_CAPTURE */
153 #define PMCR0_PCC_INIT (0)
154 #endif /* !HAS_CPMU_PC_CAPTURE */
155
156 #define PMCR0_INIT (PMCR0_INTGEN_INIT | PMCR0_PMI_INIT | PMCR0_PCC_INIT)
157
158 /*
159 * PMCR1 controls which execution modes count events.
160 */
161 #define PMCR1_EL0A32_EN(CTR) (UINT64_C(1) << (0 + CTR_POS(CTR)))
162 #define PMCR1_EL0A64_EN(CTR) (UINT64_C(1) << (8 + CTR_POS(CTR)))
163 #define S3_1_C15_C1_0A64_EN(CTR) (UINT64_C(1) << (16 + CTR_POS(CTR)))
164 /* PMCR1_EL3A64 is not supported on systems with no monitor */
165 #if defined(APPLEHURRICANE)
166 #define PMCR1_EL3A64_EN(CTR) UINT64_C(0)
167 #else
168 #define PMCR1_EL3A64_EN(CTR) (UINT64_C(1) << (24 + CTR_POS(CTR)))
169 #endif
170 #define PMCR1_ALL_EN(CTR) (PMCR1_EL0A32_EN(CTR) | PMCR1_EL0A64_EN(CTR) | \
171 S3_1_C15_C1_0A64_EN(CTR) | PMCR1_EL3A64_EN(CTR))
172
173 /* fixed counters always count in all modes */
174 #define PMCR1_INIT (PMCR1_ALL_EN(CYCLES) | PMCR1_ALL_EN(INSTRS))
175
176 static inline void
core_init_execution_modes(void)177 core_init_execution_modes(void)
178 {
179 uint64_t pmcr1;
180
181 pmcr1 = __builtin_arm_rsr64("S3_1_C15_C1_0");
182 pmcr1 |= PMCR1_INIT;
183 __builtin_arm_wsr64("S3_1_C15_C1_0", pmcr1);
184 #if CONFIG_EXCLAVES
185 __builtin_arm_wsr64("S3_1_C15_C7_2", pmcr1);
186 #endif
187 }
188
189 #define PMSR_OVF(CTR) (1ULL << (CTR))
190
191 static int
core_init(__unused mt_device_t dev)192 core_init(__unused mt_device_t dev)
193 {
194 /* the dev node interface to the core counters is still unsupported */
195 return ENOTSUP;
196 }
197
198 struct mt_cpu *
mt_cur_cpu(void)199 mt_cur_cpu(void)
200 {
201 return &getCpuDatap()->cpu_monotonic;
202 }
203
204 uint64_t
mt_core_snap(unsigned int ctr)205 mt_core_snap(unsigned int ctr)
206 {
207 switch (ctr) {
208 case 0:
209 return __builtin_arm_rsr64("S3_2_C15_C0_0");
210 case 1:
211 return __builtin_arm_rsr64("S3_2_C15_C1_0");
212 case 2:
213 return __builtin_arm_rsr64("S3_2_C15_C2_0");
214 case 3:
215 return __builtin_arm_rsr64("S3_2_C15_C3_0");
216 case 4:
217 return __builtin_arm_rsr64("S3_2_C15_C4_0");
218 case 5:
219 return __builtin_arm_rsr64("S3_2_C15_C5_0");
220 case 6:
221 return __builtin_arm_rsr64("S3_2_C15_C6_0");
222 case 7:
223 return __builtin_arm_rsr64("S3_2_C15_C7_0");
224 #if CORE_NCTRS > 8
225 case 8:
226 return __builtin_arm_rsr64("S3_2_C15_C9_0");
227 case 9:
228 return __builtin_arm_rsr64("S3_2_C15_C10_0");
229 #endif /* CORE_NCTRS > 8 */
230 default:
231 panic("monotonic: invalid core counter read: %u", ctr);
232 __builtin_unreachable();
233 }
234 }
235
236 void
mt_core_set_snap(unsigned int ctr,uint64_t count)237 mt_core_set_snap(unsigned int ctr, uint64_t count)
238 {
239 switch (ctr) {
240 case 0:
241 __builtin_arm_wsr64("S3_2_C15_C0_0", count);
242 break;
243 case 1:
244 __builtin_arm_wsr64("S3_2_C15_C1_0", count);
245 break;
246 default:
247 panic("monotonic: invalid core counter %u write %llu", ctr, count);
248 __builtin_unreachable();
249 }
250 }
251
252 static void
core_set_enabled(void)253 core_set_enabled(void)
254 {
255 uint32_t kpc_mask = kpc_get_running() &
256 (KPC_CLASS_CONFIGURABLE_MASK | KPC_CLASS_POWER_MASK);
257 uint64_t pmcr0 = __builtin_arm_rsr64("S3_1_C15_C0_0");
258 pmcr0 |= PMCR0_INIT | PMCR0_FIXED_EN;
259
260 if (kpc_mask != 0) {
261 uint64_t kpc_ctrs = kpc_get_configurable_pmc_mask(kpc_mask) <<
262 MT_CORE_NFIXED;
263 #if KPC_ARM64_CONFIGURABLE_COUNT > 6
264 uint64_t ctrs_ge8 = kpc_ctrs >> 8;
265 pmcr0 |= ctrs_ge8 << PMCR0_CTR_GE8_EN_SHIFT;
266 pmcr0 |= ctrs_ge8 << PMCR0_CTR_GE8_PMI_SHIFT;
267 kpc_ctrs &= (1ULL << 8) - 1;
268 #endif /* KPC_ARM64_CONFIGURABLE_COUNT > 6 */
269 kpc_ctrs |= kpc_ctrs << PMCR0_PMI_SHIFT;
270 pmcr0 |= kpc_ctrs;
271 }
272
273 __builtin_arm_wsr64("S3_1_C15_C0_0", pmcr0);
274 #if MACH_ASSERT
275 /*
276 * Only check for the values that were ORed in.
277 */
278 uint64_t pmcr0_check = __builtin_arm_rsr64("S3_1_C15_C0_0");
279 if ((pmcr0_check & (PMCR0_INIT | PMCR0_FIXED_EN)) != (PMCR0_INIT | PMCR0_FIXED_EN)) {
280 panic("monotonic: hardware ignored enable (read %llx, wrote %llx)",
281 pmcr0_check, pmcr0);
282 }
283 #endif /* MACH_ASSERT */
284 }
285
286 static void
core_idle(__unused cpu_data_t * cpu)287 core_idle(__unused cpu_data_t *cpu)
288 {
289 assert(cpu != NULL);
290 assert(ml_get_interrupts_enabled() == FALSE);
291
292 #if DEBUG
293 uint64_t pmcr0 = __builtin_arm_rsr64("S3_1_C15_C0_0");
294 if ((pmcr0 & PMCR0_FIXED_EN) == 0) {
295 panic("monotonic: counters disabled before idling, pmcr0 = 0x%llx", pmcr0);
296 }
297 uint64_t pmcr1 = __builtin_arm_rsr64("S3_1_C15_C1_0");
298 if ((pmcr1 & PMCR1_INIT) == 0) {
299 panic("monotonic: counter modes disabled before idling, pmcr1 = 0x%llx", pmcr1);
300 }
301 #endif /* DEBUG */
302
303 /* disable counters before updating */
304 __builtin_arm_wsr64("S3_1_C15_C0_0", PMCR0_INIT);
305
306 mt_update_fixed_counts();
307 }
308
309 #pragma mark uncore performance monitor
310
311 #if HAS_UNCORE_CTRS
312
313 static bool mt_uncore_initted = false;
314
315 static bool mt_uncore_suspended_cpd = false;
316
317 /*
318 * Uncore Performance Monitor
319 *
320 * Uncore performance monitors provide event-counting for the last-level caches
321 * (LLCs). Each LLC has its own uncore performance monitor, which can only be
322 * accessed by cores that use that LLC. Like the core performance monitoring
323 * unit, uncore counters are configured globally. If there is more than one
324 * LLC on the system, PIO reads must be used to satisfy uncore requests (using
325 * the `_r` remote variants of the access functions). Otherwise, local MSRs
326 * suffice (using the `_l` local variants of the access functions).
327 */
328
329 #if UNCORE_PER_CLUSTER
330 #define MAX_NMONITORS MAX_CPU_CLUSTERS
331 static uintptr_t cpm_impl[MAX_NMONITORS] = {};
332 #else
333 #define MAX_NMONITORS (1)
334 #endif /* UNCORE_PER_CLUSTER */
335
336 #if UNCORE_VERSION >= 2
337 /*
338 * V2 uncore monitors feature a CTI mechanism -- the second bit of UPMSR is
339 * used to track if a CTI has been triggered due to an overflow.
340 */
341 #define UPMSR_OVF_POS 2
342 #else /* UNCORE_VERSION >= 2 */
343 #define UPMSR_OVF_POS 1
344 #endif /* UNCORE_VERSION < 2 */
345 #define UPMSR_OVF(R, CTR) ((R) >> ((CTR) + UPMSR_OVF_POS) & 0x1)
346 #define UPMSR_OVF_MASK (((UINT64_C(1) << UNCORE_NCTRS) - 1) << UPMSR_OVF_POS)
347
348 #define UPMPCM_CORE(ID) (UINT64_C(1) << (ID))
349
350 #if UPMU_64BIT_PMCS
351 #define UPMC_WIDTH (63)
352 #else // UPMU_64BIT_PMCS
353 #define UPMC_WIDTH (47)
354 #endif // !UPMU_64BIT_PMCS
355
356 /*
357 * The uncore_pmi_mask is a bitmask of CPUs that receive uncore PMIs. It's
358 * initialized by uncore_init and controllable by the uncore_pmi_mask boot-arg.
359 */
360 static int32_t uncore_pmi_mask = 0;
361
362 /*
363 * The uncore_active_ctrs is a bitmask of uncore counters that are currently
364 * requested.
365 */
366 static uint16_t uncore_active_ctrs = 0;
367 static_assert(sizeof(uncore_active_ctrs) * CHAR_BIT >= UNCORE_NCTRS,
368 "counter mask should fit the full range of counters");
369
370 #if UPMU_9BIT_SELECTORS
371 static uint16_t uncore_selectors_bit9 = 0;
372 #endif /* UPMU_9BIT_SELECTORS */
373
374 static uint64_t
_upmcr0_value(void)375 _upmcr0_value(void)
376 {
377 #if UPMU_9BIT_SELECTORS
378 return (uint64_t)uncore_selectors_bit9 << 36 | (uint64_t)uncore_active_ctrs;
379 #else /* UPMU_9BIT_SELECTORS */
380 return uncore_active_ctrs;
381 #endif /* !UPMU_9BIT_SELECTORS */
382 }
383
384 /*
385 * mt_uncore_enabled is true when any uncore counters are active.
386 */
387 bool mt_uncore_enabled = false;
388
389 /*
390 * The uncore_events are the event configurations for each uncore counter -- as
391 * a union to make it easy to program the hardware registers.
392 */
393 static struct uncore_config {
394 union {
395 uint8_t uce_ctrs[UNCORE_NCTRS];
396 uint64_t uce_regs[UNCORE_NCTRS / 8];
397 } uc_events;
398 union {
399 uint16_t uccm_masks[UNCORE_NCTRS];
400 uint64_t uccm_regs[UNCORE_NCTRS / 4];
401 } uc_cpu_masks[MAX_NMONITORS];
402 } uncore_config;
403
404 static struct uncore_monitor {
405 /*
406 * The last snapshot of each of the hardware counter values.
407 */
408 uint64_t um_snaps[UNCORE_NCTRS];
409
410 /*
411 * The accumulated counts for each counter.
412 */
413 uint64_t um_counts[UNCORE_NCTRS];
414
415 /*
416 * Protects accessing the hardware registers and fields in this structure.
417 */
418 lck_spin_t um_lock;
419
420 /*
421 * Whether this monitor needs its registers restored after wake.
422 */
423 bool um_sleeping;
424
425 #if MACH_ASSERT
426 /*
427 * Save the last ID that read from this monitor.
428 */
429 uint8_t um_last_read_id;
430
431 /*
432 * Save whether this monitor has been read since sleeping.
433 */
434 bool um_read_since_sleep;
435 #endif /* MACH_ASSERT */
436 } uncore_monitors[MAX_NMONITORS];
437
438 /*
439 * Each uncore unit has its own monitor, corresponding to the memory hierarchy
440 * of the LLCs.
441 */
442 static unsigned int
uncore_nmonitors(void)443 uncore_nmonitors(void)
444 {
445 #if UNCORE_PER_CLUSTER
446 return topology_info->num_clusters;
447 #else /* UNCORE_PER_CLUSTER */
448 return 1;
449 #endif /* !UNCORE_PER_CLUSTER */
450 }
451
452 static unsigned int
uncmon_get_curid(void)453 uncmon_get_curid(void)
454 {
455 #if UNCORE_PER_CLUSTER
456 return cpu_cluster_id();
457 #else /* UNCORE_PER_CLUSTER */
458 return 0;
459 #endif /* !UNCORE_PER_CLUSTER */
460 }
461
462 /*
463 * Per-monitor locks are required to prevent races with the PMI handlers, not
464 * from other CPUs that are configuring (those are serialized with monotonic's
465 * per-device lock).
466 */
467
468 static int
uncmon_lock(struct uncore_monitor * mon)469 uncmon_lock(struct uncore_monitor *mon)
470 {
471 int intrs_en = ml_set_interrupts_enabled(FALSE);
472 lck_spin_lock(&mon->um_lock);
473 return intrs_en;
474 }
475
476 static void
uncmon_unlock(struct uncore_monitor * mon,int intrs_en)477 uncmon_unlock(struct uncore_monitor *mon, int intrs_en)
478 {
479 lck_spin_unlock(&mon->um_lock);
480 (void)ml_set_interrupts_enabled(intrs_en);
481 }
482
483 static bool
uncmon_is_remote(unsigned int monid)484 uncmon_is_remote(unsigned int monid)
485 {
486 if (monid >= MAX_NMONITORS) {
487 panic("monotonic: %s: invalid monid %u (> %u)", __FUNCTION__, monid, MAX_NMONITORS);
488 }
489 struct uncore_monitor *mon = &uncore_monitors[monid];
490 #pragma unused(mon)
491 LCK_SPIN_ASSERT(&mon->um_lock, LCK_ASSERT_OWNED);
492 return monid == uncmon_get_curid();
493 }
494
495 /*
496 * Helper functions for accessing the hardware -- these require the monitor be
497 * locked to prevent other CPUs' PMI handlers from making local modifications
498 * or updating the counts.
499 */
500
501 #if UNCORE_VERSION >= 2
502 #define UPMCR0_INTEN_POS 20
503 #define UPMCR0_INTGEN_POS 16
504 #else /* UNCORE_VERSION >= 2 */
505 #define UPMCR0_INTEN_POS 12
506 #define UPMCR0_INTGEN_POS 8
507 #endif /* UNCORE_VERSION < 2 */
508 enum {
509 UPMCR0_INTGEN_OFF = 0,
510 /* fast PMIs are only supported on core CPMU */
511 UPMCR0_INTGEN_AIC = 2,
512 UPMCR0_INTGEN_HALT = 3,
513 UPMCR0_INTGEN_FIQ = 4,
514 };
515 /* always enable interrupts for all counters */
516 #define UPMCR0_INTEN (((1ULL << UNCORE_NCTRS) - 1) << UPMCR0_INTEN_POS)
517 /* route uncore PMIs through the FIQ path */
518 #define UPMCR0_INIT (UPMCR0_INTEN | (UPMCR0_INTGEN_FIQ << UPMCR0_INTGEN_POS))
519
520 /*
521 * Turn counting on for counters set in the `enctrmask` and off, otherwise.
522 */
523 static inline void
uncmon_set_counting_locked_l(__unused unsigned int monid,uint64_t enctrmask)524 uncmon_set_counting_locked_l(__unused unsigned int monid, uint64_t enctrmask)
525 {
526 /*
527 * UPMCR0 controls which counters are enabled and how interrupts are generated
528 * for overflows.
529 */
530 __builtin_arm_wsr64("S3_7_C15_C0_4", UPMCR0_INIT | enctrmask);
531 }
532
533 #if UNCORE_PER_CLUSTER
534
535 /*
536 * Turn counting on for counters set in the `enctrmask` and off, otherwise.
537 */
538 static inline void
uncmon_set_counting_locked_r(unsigned int monid,uint64_t enctrmask)539 uncmon_set_counting_locked_r(unsigned int monid, uint64_t enctrmask)
540 {
541 const uintptr_t upmcr0_offset = 0x4180;
542 *(uint64_t *)(cpm_impl[monid] + upmcr0_offset) = UPMCR0_INIT | enctrmask;
543 }
544
545 #endif /* UNCORE_PER_CLUSTER */
546
547 /*
548 * The uncore performance monitoring counters (UPMCs) are 48/64-bits wide. The
549 * high bit is an overflow bit, triggering a PMI, providing 47/63 usable bits.
550 */
551
552 #define UPMC_MAX ((UINT64_C(1) << UPMC_WIDTH) - 1)
553
554 static void
_broadcast_block_trampoline(void * blk)555 _broadcast_block_trampoline(void *blk)
556 {
557 void (^cb)(unsigned int) = blk;
558 const ml_topology_info_t *topo = ml_get_topology_info();
559 unsigned int cpu = cpu_number();
560 unsigned int cluster = topo->cpus[cpu].cluster_id;
561 if (topo->clusters[cluster].first_cpu_id == cpu) {
562 cb(topo->cpus[cpu].cluster_id);
563 }
564 }
565
566 __unused
567 static void
568 _broadcast_each_cluster(void (^cb)(unsigned int cluster_id))
569 {
570 cpu_broadcast_xcall_simple(TRUE, _broadcast_block_trampoline, cb);
571 }
572
573 __unused
574 static inline uint64_t
uncmon_read_counter_locked_l(__unused unsigned int monid,unsigned int ctr)575 uncmon_read_counter_locked_l(__unused unsigned int monid, unsigned int ctr)
576 {
577 assert(ctr < UNCORE_NCTRS);
578 switch (ctr) {
579 case 0:
580 return __builtin_arm_rsr64("S3_7_C15_C7_4");
581 case 1:
582 return __builtin_arm_rsr64("S3_7_C15_C8_4");
583 case 2:
584 return __builtin_arm_rsr64("S3_7_C15_C9_4");
585 case 3:
586 return __builtin_arm_rsr64("S3_7_C15_C10_4");
587 case 4:
588 return __builtin_arm_rsr64("S3_7_C15_C11_4");
589 case 5:
590 return __builtin_arm_rsr64("S3_7_C15_C12_4");
591 case 6:
592 return __builtin_arm_rsr64("S3_7_C15_C13_4");
593 case 7:
594 return __builtin_arm_rsr64("S3_7_C15_C14_4");
595 #if UNCORE_NCTRS > 8
596 case 8:
597 return __builtin_arm_rsr64("S3_7_C15_C0_5");
598 case 9:
599 return __builtin_arm_rsr64("S3_7_C15_C1_5");
600 case 10:
601 return __builtin_arm_rsr64("S3_7_C15_C2_5");
602 case 11:
603 return __builtin_arm_rsr64("S3_7_C15_C3_5");
604 case 12:
605 return __builtin_arm_rsr64("S3_7_C15_C4_5");
606 case 13:
607 return __builtin_arm_rsr64("S3_7_C15_C5_5");
608 case 14:
609 return __builtin_arm_rsr64("S3_7_C15_C6_5");
610 case 15:
611 return __builtin_arm_rsr64("S3_7_C15_C7_5");
612 #endif /* UNCORE_NCTRS > 8 */
613 default:
614 panic("monotonic: invalid counter read %u", ctr);
615 __builtin_unreachable();
616 }
617 }
618
619 static inline void
uncmon_write_counter_locked_l(__unused unsigned int monid,unsigned int ctr,uint64_t count)620 uncmon_write_counter_locked_l(__unused unsigned int monid, unsigned int ctr,
621 uint64_t count)
622 {
623 assert(count < UPMC_MAX);
624 assert(ctr < UNCORE_NCTRS);
625 switch (ctr) {
626 case 0:
627 return __builtin_arm_wsr64("S3_7_C15_C7_4", count);
628 case 1:
629 return __builtin_arm_wsr64("S3_7_C15_C8_4", count);
630 case 2:
631 return __builtin_arm_wsr64("S3_7_C15_C9_4", count);
632 case 3:
633 return __builtin_arm_wsr64("S3_7_C15_C10_4", count);
634 case 4:
635 return __builtin_arm_wsr64("S3_7_C15_C11_4", count);
636 case 5:
637 return __builtin_arm_wsr64("S3_7_C15_C12_4", count);
638 case 6:
639 return __builtin_arm_wsr64("S3_7_C15_C13_4", count);
640 case 7:
641 return __builtin_arm_wsr64("S3_7_C15_C14_4", count);
642 #if UNCORE_NCTRS > 8
643 case 8:
644 return __builtin_arm_wsr64("S3_7_C15_C0_5", count);
645 case 9:
646 return __builtin_arm_wsr64("S3_7_C15_C1_5", count);
647 case 10:
648 return __builtin_arm_wsr64("S3_7_C15_C2_5", count);
649 case 11:
650 return __builtin_arm_wsr64("S3_7_C15_C3_5", count);
651 case 12:
652 return __builtin_arm_wsr64("S3_7_C15_C4_5", count);
653 case 13:
654 return __builtin_arm_wsr64("S3_7_C15_C5_5", count);
655 case 14:
656 return __builtin_arm_wsr64("S3_7_C15_C6_5", count);
657 case 15:
658 return __builtin_arm_wsr64("S3_7_C15_C7_5", count);
659 #endif /* UNCORE_NCTRS > 8 */
660 default:
661 panic("monotonic: invalid counter write %u", ctr);
662 }
663 }
664
665 #if UNCORE_PER_CLUSTER
666
667 uintptr_t upmc_offs[UNCORE_NCTRS] = {
668 [0] = 0x4100, [1] = 0x4248, [2] = 0x4110, [3] = 0x4250, [4] = 0x4120,
669 [5] = 0x4258, [6] = 0x4130, [7] = 0x4260, [8] = 0x4140, [9] = 0x4268,
670 [10] = 0x4150, [11] = 0x4270, [12] = 0x4160, [13] = 0x4278,
671 [14] = 0x4170, [15] = 0x4280,
672 };
673
674 static inline uint64_t
uncmon_read_counter_locked_r(unsigned int mon_id,unsigned int ctr)675 uncmon_read_counter_locked_r(unsigned int mon_id, unsigned int ctr)
676 {
677 assert(mon_id < uncore_nmonitors());
678 assert(ctr < UNCORE_NCTRS);
679 return *(uint64_t *)(cpm_impl[mon_id] + upmc_offs[ctr]);
680 }
681
682 static inline void
uncmon_write_counter_locked_r(unsigned int mon_id,unsigned int ctr,uint64_t count)683 uncmon_write_counter_locked_r(unsigned int mon_id, unsigned int ctr,
684 uint64_t count)
685 {
686 assert(count < UPMC_MAX);
687 assert(ctr < UNCORE_NCTRS);
688 assert(mon_id < uncore_nmonitors());
689 *(uint64_t *)(cpm_impl[mon_id] + upmc_offs[ctr]) = count;
690 }
691
692 #endif /* UNCORE_PER_CLUSTER */
693
694 static inline void
uncmon_update_locked(unsigned int monid,unsigned int ctr)695 uncmon_update_locked(unsigned int monid, unsigned int ctr)
696 {
697 struct uncore_monitor *mon = &uncore_monitors[monid];
698 if (!mon->um_sleeping) {
699 uint64_t snap = 0;
700 #if UNCORE_PER_CLUSTER
701 snap = uncmon_read_counter_locked_r(monid, ctr);
702 #else /* UNCORE_PER_CLUSTER */
703 snap = uncmon_read_counter_locked_l(monid, ctr);
704 #endif /* UNCORE_PER_CLUSTER */
705 if (snap < mon->um_snaps[ctr]) {
706 #if MACH_ASSERT
707 #if UNCORE_PER_CLUSTER
708 uint64_t remote_value = uncmon_read_counter_locked_r(monid, ctr);
709 #endif /* UNCORE_PER_CLUSTER */
710 panic("monotonic: UPMC%d on UPMU %d went backwards from "
711 "%llx to %llx, read via %s, last was %s from UPMU %hhd%s"
712 #if UNCORE_PER_CLUSTER
713 ", re-read remote value is %llx"
714 #endif /* UNCORE_PER_CLUSTER */
715 , ctr,
716 monid, mon->um_snaps[ctr], snap,
717 uncmon_get_curid() == monid ? "local" : "remote",
718 mon->um_last_read_id == monid ? "local" : "remote",
719 mon->um_last_read_id,
720 mon->um_read_since_sleep ? "" : ", first read since sleep"
721 #if UNCORE_PER_CLUSTER
722 , remote_value
723 #endif /* UNCORE_PER_CLUSTER */
724 );
725 #else /* MACH_ASSERT */
726 snap = mon->um_snaps[ctr];
727 #endif /* !MACH_ASSERT */
728 }
729 mon->um_counts[ctr] += snap - mon->um_snaps[ctr];
730 mon->um_snaps[ctr] = snap;
731 }
732 }
733
734 static inline void
uncmon_program_events_locked_l(unsigned int monid)735 uncmon_program_events_locked_l(unsigned int monid)
736 {
737 /*
738 * UPMESR[01] is the event selection register that determines which event a
739 * counter will count.
740 */
741 CTRL_REG_SET("S3_7_C15_C1_4", uncore_config.uc_events.uce_regs[0]);
742
743 #if UNCORE_NCTRS > 8
744 CTRL_REG_SET("S3_7_C15_C11_5", uncore_config.uc_events.uce_regs[1]);
745 #endif /* UNCORE_NCTRS > 8 */
746
747 /*
748 * UPMECM[0123] are the event core masks for each counter -- whether or not
749 * that counter counts events generated by an agent. These are set to all
750 * ones so the uncore counters count events from all cores.
751 *
752 * The bits are based off the start of the cluster -- e.g. even if a core
753 * has a CPU ID of 4, it might be the first CPU in a cluster. Shift the
754 * registers right by the ID of the first CPU in the cluster.
755 */
756 CTRL_REG_SET("S3_7_C15_C3_4",
757 uncore_config.uc_cpu_masks[monid].uccm_regs[0]);
758 CTRL_REG_SET("S3_7_C15_C4_4",
759 uncore_config.uc_cpu_masks[monid].uccm_regs[1]);
760
761 #if UNCORE_NCTRS > 8
762 CTRL_REG_SET("S3_7_C15_C8_5",
763 uncore_config.uc_cpu_masks[monid].uccm_regs[2]);
764 CTRL_REG_SET("S3_7_C15_C9_5",
765 uncore_config.uc_cpu_masks[monid].uccm_regs[3]);
766 #endif /* UNCORE_NCTRS > 8 */
767 }
768
769 #if UNCORE_PER_CLUSTER
770
771 static inline void
uncmon_program_events_locked_r(unsigned int monid)772 uncmon_program_events_locked_r(unsigned int monid)
773 {
774 const uintptr_t upmesr_offs[2] = {[0] = 0x41b0, [1] = 0x41b8, };
775
776 for (unsigned int i = 0; i < sizeof(upmesr_offs) / sizeof(upmesr_offs[0]);
777 i++) {
778 *(uint64_t *)(cpm_impl[monid] + upmesr_offs[i]) =
779 uncore_config.uc_events.uce_regs[i];
780 }
781
782 const uintptr_t upmecm_offs[4] = {
783 [0] = 0x4190, [1] = 0x4198, [2] = 0x41a0, [3] = 0x41a8,
784 };
785
786 for (unsigned int i = 0; i < sizeof(upmecm_offs) / sizeof(upmecm_offs[0]);
787 i++) {
788 *(uint64_t *)(cpm_impl[monid] + upmecm_offs[i]) =
789 uncore_config.uc_cpu_masks[monid].uccm_regs[i];
790 }
791 }
792
793 #endif /* UNCORE_PER_CLUSTER */
794
795 static void
uncmon_clear_int_locked_l(__unused unsigned int monid)796 uncmon_clear_int_locked_l(__unused unsigned int monid)
797 {
798 __builtin_arm_wsr64("S3_7_C15_C6_4", 0);
799 }
800
801 #if UNCORE_PER_CLUSTER
802
803 static void
uncmon_clear_int_locked_r(unsigned int monid)804 uncmon_clear_int_locked_r(unsigned int monid)
805 {
806 const uintptr_t upmsr_off = 0x41c0;
807 *(uint64_t *)(cpm_impl[monid] + upmsr_off) = 0;
808 }
809
810 #endif /* UNCORE_PER_CLUSTER */
811
812 /*
813 * Get the PMI mask for the provided `monid` -- that is, the bitmap of CPUs
814 * that should be sent PMIs for a particular monitor.
815 */
816 static uint64_t
uncmon_get_pmi_mask(unsigned int monid)817 uncmon_get_pmi_mask(unsigned int monid)
818 {
819 uint64_t pmi_mask = uncore_pmi_mask;
820
821 #if UNCORE_PER_CLUSTER
822 pmi_mask &= topology_info->clusters[monid].cpu_mask;
823 #else /* UNCORE_PER_CLUSTER */
824 #pragma unused(monid)
825 #endif /* !UNCORE_PER_CLUSTER */
826
827 return pmi_mask;
828 }
829
830 /*
831 * Initialization routines for the uncore counters.
832 */
833
834 static void
uncmon_init_locked_l(unsigned int monid)835 uncmon_init_locked_l(unsigned int monid)
836 {
837 /*
838 * UPMPCM defines the PMI core mask for the UPMCs -- which cores should
839 * receive interrupts on overflow.
840 */
841 CTRL_REG_SET("S3_7_C15_C5_4", uncmon_get_pmi_mask(monid));
842 uncmon_set_counting_locked_l(monid,
843 mt_uncore_enabled ? _upmcr0_value() : 0);
844 }
845
846 #if UNCORE_PER_CLUSTER
847
848 static uintptr_t acc_impl[MAX_NMONITORS] = {};
849
850 static void
uncmon_init_locked_r(unsigned int monid)851 uncmon_init_locked_r(unsigned int monid)
852 {
853 const uintptr_t upmpcm_off = 0x1010;
854
855 *(uint64_t *)(acc_impl[monid] + upmpcm_off) = uncmon_get_pmi_mask(monid);
856 uncmon_set_counting_locked_r(monid,
857 mt_uncore_enabled ? _upmcr0_value() : 0);
858 }
859
860 #endif /* UNCORE_PER_CLUSTER */
861
862 /*
863 * Initialize the uncore device for monotonic.
864 */
865 static int
uncore_init(__unused mt_device_t dev)866 uncore_init(__unused mt_device_t dev)
867 {
868 #if HAS_UNCORE_CTRS
869 assert(MT_NDEVS > 0);
870 mt_devices[MT_NDEVS - 1].mtd_nmonitors = (uint8_t)uncore_nmonitors();
871 #endif
872
873 #if DEVELOPMENT || DEBUG
874 /*
875 * Development and debug kernels observe the `uncore_pmi_mask` boot-arg,
876 * allowing PMIs to be routed to the CPUs present in the supplied bitmap.
877 * Do some sanity checks on the value provided.
878 */
879 bool parsed_arg = PE_parse_boot_argn("uncore_pmi_mask", &uncore_pmi_mask,
880 sizeof(uncore_pmi_mask));
881 if (parsed_arg) {
882 #if UNCORE_PER_CLUSTER
883 if (__builtin_popcount(uncore_pmi_mask) != (int)uncore_nmonitors()) {
884 panic("monotonic: invalid uncore PMI mask 0x%x", uncore_pmi_mask);
885 }
886 for (unsigned int i = 0; i < uncore_nmonitors(); i++) {
887 if (__builtin_popcountll(uncmon_get_pmi_mask(i)) != 1) {
888 panic("monotonic: invalid uncore PMI CPU for cluster %d in mask 0x%x",
889 i, uncore_pmi_mask);
890 }
891 }
892 #else /* UNCORE_PER_CLUSTER */
893 if (__builtin_popcount(uncore_pmi_mask) != 1) {
894 panic("monotonic: invalid uncore PMI mask 0x%x", uncore_pmi_mask);
895 }
896 #endif /* !UNCORE_PER_CLUSTER */
897 } else
898 #endif /* DEVELOPMENT || DEBUG */
899 {
900 /* arbitrarily route to core 0 in each cluster */
901 uncore_pmi_mask |= 1;
902 }
903 assert(uncore_pmi_mask != 0);
904
905 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
906 #if UNCORE_PER_CLUSTER
907 ml_topology_cluster_t *cluster = &topology_info->clusters[monid];
908 cpm_impl[monid] = (uintptr_t)cluster->cpm_IMPL_regs;
909 acc_impl[monid] = (uintptr_t)cluster->acc_IMPL_regs;
910 assert(cpm_impl[monid] != 0 && acc_impl[monid] != 0);
911 #endif /* UNCORE_PER_CLUSTER */
912
913 struct uncore_monitor *mon = &uncore_monitors[monid];
914 lck_spin_init(&mon->um_lock, &mt_lock_grp, LCK_ATTR_NULL);
915 }
916
917 mt_uncore_initted = true;
918
919 return 0;
920 }
921
922 /*
923 * Support for monotonic's mtd_read function.
924 */
925
926 static void
uncmon_read_all_counters(unsigned int monid,uint64_t ctr_mask,uint64_t * counts)927 uncmon_read_all_counters(unsigned int monid, uint64_t ctr_mask, uint64_t *counts)
928 {
929 struct uncore_monitor *mon = &uncore_monitors[monid];
930
931 int intrs_en = uncmon_lock(mon);
932
933 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
934 if (ctr_mask & (1ULL << ctr)) {
935 if (!mon->um_sleeping) {
936 uncmon_update_locked(monid, ctr);
937 }
938 counts[ctr] = mon->um_counts[ctr];
939 }
940 }
941 #if MACH_ASSERT
942 mon->um_read_since_sleep = true;
943 #endif /* MACH_ASSERT */
944
945 uncmon_unlock(mon, intrs_en);
946 }
947
948 /*
949 * Read all monitor's counters.
950 */
951 static int
uncore_read(uint64_t ctr_mask,uint64_t * counts_out)952 uncore_read(uint64_t ctr_mask, uint64_t *counts_out)
953 {
954 assert(ctr_mask != 0);
955 assert(counts_out != NULL);
956
957 if (!uncore_active_ctrs) {
958 return EPWROFF;
959 }
960 if (ctr_mask & ~uncore_active_ctrs) {
961 return EINVAL;
962 }
963
964 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
965 /*
966 * Find this monitor's starting offset into the `counts_out` array.
967 */
968 uint64_t *counts = counts_out + (UNCORE_NCTRS * monid);
969 uncmon_read_all_counters(monid, ctr_mask, counts);
970 }
971
972 return 0;
973 }
974
975 /*
976 * Support for monotonic's mtd_add function.
977 */
978
979 /*
980 * Add an event to the current uncore configuration. This doesn't take effect
981 * until the counters are enabled again, so there's no need to involve the
982 * monitors.
983 */
984 static int
uncore_add(struct monotonic_config * config,uint32_t * ctr_out)985 uncore_add(struct monotonic_config *config, uint32_t *ctr_out)
986 {
987 if (mt_uncore_enabled) {
988 return EBUSY;
989 }
990
991 uint16_t selector = (uint16_t)config->event;
992 uint32_t available = ~uncore_active_ctrs & config->allowed_ctr_mask;
993
994 if (available == 0) {
995 return ENOSPC;
996 }
997
998 if (!cpc_event_allowed(CPC_HW_UPMU, selector)) {
999 return EPERM;
1000 }
1001
1002 uint32_t valid_ctrs = (UINT32_C(1) << UNCORE_NCTRS) - 1;
1003 if ((available & valid_ctrs) == 0) {
1004 return E2BIG;
1005 }
1006 /*
1007 * Clear the UPMCs the first time an event is added.
1008 */
1009 if (uncore_active_ctrs == 0) {
1010 /*
1011 * Suspend powerdown until the next reset.
1012 */
1013 assert(!mt_uncore_suspended_cpd);
1014 suspend_cluster_powerdown();
1015 mt_uncore_suspended_cpd = true;
1016
1017 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1018 struct uncore_monitor *mon = &uncore_monitors[monid];
1019
1020 int intrs_en = uncmon_lock(mon);
1021 bool remote = uncmon_is_remote(monid);
1022
1023 if (!mon->um_sleeping) {
1024 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1025 if (remote) {
1026 #if UNCORE_PER_CLUSTER
1027 uncmon_write_counter_locked_r(monid, ctr, 0);
1028 #endif /* UNCORE_PER_CLUSTER */
1029 } else {
1030 uncmon_write_counter_locked_l(monid, ctr, 0);
1031 }
1032 }
1033 }
1034 memset(&mon->um_snaps, 0, sizeof(mon->um_snaps));
1035 memset(&mon->um_counts, 0, sizeof(mon->um_counts));
1036 uncmon_unlock(mon, intrs_en);
1037 }
1038 }
1039
1040 uint32_t ctr = __builtin_ffsll(available) - 1;
1041
1042 uncore_active_ctrs |= UINT64_C(1) << ctr;
1043 uncore_config.uc_events.uce_ctrs[ctr] = (uint8_t)selector;
1044 #if UPMU_9BIT_SELECTORS
1045 uncore_selectors_bit9 &= ~(1 << ctr);
1046 uncore_selectors_bit9 |= ((selector >> 8) & 1) << ctr;
1047 #endif /* UPMU_9BIT_SELECTORS */
1048 uint64_t cpu_mask = UINT64_MAX;
1049 if (config->cpu_mask != 0) {
1050 cpu_mask = config->cpu_mask;
1051 }
1052 for (unsigned int i = 0; i < uncore_nmonitors(); i++) {
1053 #if UNCORE_PER_CLUSTER
1054 const unsigned int shift = topology_info->clusters[i].first_cpu_id;
1055 #else /* UNCORE_PER_CLUSTER */
1056 const unsigned int shift = 0;
1057 #endif /* !UNCORE_PER_CLUSTER */
1058 uncore_config.uc_cpu_masks[i].uccm_masks[ctr] = (uint16_t)(cpu_mask >> shift);
1059 }
1060
1061 *ctr_out = ctr;
1062 return 0;
1063 }
1064
1065 /*
1066 * Support for monotonic's mtd_reset function.
1067 */
1068
1069 /*
1070 * Reset all configuration and disable the counters if they're currently
1071 * counting.
1072 */
1073 static void
uncore_reset(void)1074 uncore_reset(void)
1075 {
1076 mt_uncore_enabled = false;
1077
1078 if (!mt_uncore_suspended_cpd) {
1079 /* If we haven't already suspended CPD, we need to do so now to ensure we can issue remote reads
1080 * to every cluster. */
1081 suspend_cluster_powerdown();
1082 mt_uncore_suspended_cpd = true;
1083 }
1084
1085 if (mt_owns_counters()) {
1086 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1087 struct uncore_monitor *mon = &uncore_monitors[monid];
1088
1089 int intrs_en = uncmon_lock(mon);
1090 bool remote = uncmon_is_remote(monid);
1091 if (!mon->um_sleeping) {
1092 if (remote) {
1093 #if UNCORE_PER_CLUSTER
1094 uncmon_set_counting_locked_r(monid, 0);
1095 #endif /* UNCORE_PER_CLUSTER */
1096 } else {
1097 uncmon_set_counting_locked_l(monid, 0);
1098 }
1099
1100 for (int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1101 if (uncore_active_ctrs & (1U << ctr)) {
1102 if (remote) {
1103 #if UNCORE_PER_CLUSTER
1104 uncmon_write_counter_locked_r(monid, ctr, 0);
1105 #endif /* UNCORE_PER_CLUSTER */
1106 } else {
1107 uncmon_write_counter_locked_l(monid, ctr, 0);
1108 }
1109 }
1110 }
1111 }
1112
1113 memset(&mon->um_snaps, 0, sizeof(mon->um_snaps));
1114 memset(&mon->um_counts, 0, sizeof(mon->um_counts));
1115 if (!mon->um_sleeping) {
1116 if (remote) {
1117 #if UNCORE_PER_CLUSTER
1118 uncmon_clear_int_locked_r(monid);
1119 #endif /* UNCORE_PER_CLUSTER */
1120 } else {
1121 uncmon_clear_int_locked_l(monid);
1122 }
1123 }
1124
1125 uncmon_unlock(mon, intrs_en);
1126 }
1127 }
1128
1129 uncore_active_ctrs = 0;
1130 #if UPMU_9BIT_SELECTORS
1131 uncore_selectors_bit9 = 0;
1132 #endif /* UPMU_9BIT_SELECTORS */
1133 memset(&uncore_config, 0, sizeof(uncore_config));
1134
1135 if (mt_owns_counters()) {
1136 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1137 struct uncore_monitor *mon = &uncore_monitors[monid];
1138
1139 int intrs_en = uncmon_lock(mon);
1140 bool remote = uncmon_is_remote(monid);
1141 if (!mon->um_sleeping) {
1142 if (remote) {
1143 #if UNCORE_PER_CLUSTER
1144 uncmon_program_events_locked_r(monid);
1145 #endif /* UNCORE_PER_CLUSTER */
1146 } else {
1147 uncmon_program_events_locked_l(monid);
1148 }
1149 }
1150 uncmon_unlock(mon, intrs_en);
1151 }
1152 }
1153
1154 /* After reset, no counters should be active, so we can allow powerdown again */
1155 if (mt_uncore_suspended_cpd) {
1156 resume_cluster_powerdown();
1157 mt_uncore_suspended_cpd = false;
1158 }
1159 }
1160
1161 /*
1162 * Support for monotonic's mtd_enable function.
1163 */
1164
1165 static void
uncmon_set_enabled_l_locked(unsigned int monid,bool enable)1166 uncmon_set_enabled_l_locked(unsigned int monid, bool enable)
1167 {
1168 struct uncore_monitor *mon = &uncore_monitors[monid];
1169 #pragma unused(mon)
1170 LCK_SPIN_ASSERT(&mon->um_lock, LCK_ASSERT_OWNED);
1171
1172 if (enable) {
1173 uncmon_init_locked_l(monid);
1174 uncmon_program_events_locked_l(monid);
1175 uncmon_set_counting_locked_l(monid, _upmcr0_value());
1176 } else {
1177 uncmon_set_counting_locked_l(monid, 0);
1178 }
1179 }
1180
1181 #if UNCORE_PER_CLUSTER
1182
1183 __unused
1184 static void
uncmon_set_enabled_r_locked(unsigned int monid,bool enable)1185 uncmon_set_enabled_r_locked(unsigned int monid, bool enable)
1186 {
1187 struct uncore_monitor *mon = &uncore_monitors[monid];
1188 #pragma unused(mon)
1189 LCK_SPIN_ASSERT(&mon->um_lock, LCK_ASSERT_OWNED);
1190
1191 if (!mon->um_sleeping) {
1192 if (enable) {
1193 uncmon_init_locked_r(monid);
1194 uncmon_program_events_locked_r(monid);
1195 uncmon_set_counting_locked_r(monid, _upmcr0_value());
1196 } else {
1197 uncmon_set_counting_locked_r(monid, 0);
1198 }
1199 }
1200 }
1201
1202 #endif /* UNCORE_PER_CLUSTER */
1203
1204 static void
uncore_set_enabled(bool enable)1205 uncore_set_enabled(bool enable)
1206 {
1207 mt_uncore_enabled = enable;
1208
1209 _broadcast_each_cluster(^(unsigned int cluster_id) {
1210 struct uncore_monitor *mon = &uncore_monitors[cluster_id];
1211 int intrs_en = uncmon_lock(mon);
1212 uncmon_set_enabled_l_locked(cluster_id, enable);
1213 uncmon_unlock(mon, intrs_en);
1214 });
1215 }
1216
1217 /*
1218 * Hooks in the machine layer.
1219 */
1220
1221 static void
uncore_fiq(uint64_t upmsr)1222 uncore_fiq(uint64_t upmsr)
1223 {
1224 /*
1225 * Determine which counters overflowed.
1226 */
1227 uint64_t disable_ctr_mask = (upmsr & UPMSR_OVF_MASK) >> UPMSR_OVF_POS;
1228 /* should not receive interrupts from inactive counters */
1229 assert(!(disable_ctr_mask & ~uncore_active_ctrs));
1230
1231 if (uncore_active_ctrs == 0) {
1232 return;
1233 }
1234
1235 unsigned int monid = uncmon_get_curid();
1236 struct uncore_monitor *mon = &uncore_monitors[monid];
1237
1238 int intrs_en = uncmon_lock(mon);
1239
1240 /*
1241 * Disable any counters that overflowed.
1242 */
1243 uncmon_set_counting_locked_l(monid,
1244 uncore_active_ctrs & ~disable_ctr_mask);
1245
1246 /*
1247 * With the overflowing counters disabled, capture their counts and reset
1248 * the UPMCs and their snapshots to 0.
1249 */
1250 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1251 if (UPMSR_OVF(upmsr, ctr)) {
1252 uncmon_update_locked(monid, ctr);
1253 mon->um_snaps[ctr] = 0;
1254 uncmon_write_counter_locked_l(monid, ctr, 0);
1255 }
1256 }
1257
1258 /*
1259 * Acknowledge the interrupt, now that any overflowed PMCs have been reset.
1260 */
1261 uncmon_clear_int_locked_l(monid);
1262
1263 /*
1264 * Re-enable all active counters.
1265 */
1266 uncmon_set_counting_locked_l(monid, uncore_active_ctrs);
1267
1268 uncmon_unlock(mon, intrs_en);
1269 }
1270
1271 static void
uncore_save(void)1272 uncore_save(void)
1273 {
1274 if (!uncore_active_ctrs) {
1275 return;
1276 }
1277
1278 for (unsigned int monid = 0; monid < uncore_nmonitors(); monid++) {
1279 struct uncore_monitor *mon = &uncore_monitors[monid];
1280 int intrs_en = uncmon_lock(mon);
1281
1282 if (mt_uncore_enabled) {
1283 if (uncmon_is_remote(monid)) {
1284 #if UNCORE_PER_CLUSTER
1285 uncmon_set_counting_locked_r(monid, 0);
1286 #endif /* UNCORE_PER_CLUSTER */
1287 } else {
1288 uncmon_set_counting_locked_l(monid, 0);
1289 }
1290 }
1291
1292 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1293 if (uncore_active_ctrs & (1U << ctr)) {
1294 uncmon_update_locked(monid, ctr);
1295 mon->um_snaps[ctr] = 0;
1296 uncmon_write_counter_locked_l(monid, ctr, 0);
1297 }
1298 }
1299
1300 mon->um_sleeping = true;
1301 uncmon_unlock(mon, intrs_en);
1302 }
1303 }
1304
1305 static void
uncore_restore(void)1306 uncore_restore(void)
1307 {
1308 if (!uncore_active_ctrs) {
1309 return;
1310 }
1311 /* Ensure interrupts disabled before reading uncmon_get_curid */
1312 bool intr = ml_set_interrupts_enabled(false);
1313 unsigned int curmonid = uncmon_get_curid();
1314
1315 struct uncore_monitor *mon = &uncore_monitors[curmonid];
1316 int intrs_en = uncmon_lock(mon);
1317 if (!mon->um_sleeping) {
1318 goto out;
1319 }
1320
1321 for (unsigned int ctr = 0; ctr < UNCORE_NCTRS; ctr++) {
1322 if (uncore_active_ctrs & (1U << ctr)) {
1323 uncmon_write_counter_locked_l(curmonid, ctr, mon->um_snaps[ctr]);
1324 }
1325 }
1326 uncmon_program_events_locked_l(curmonid);
1327 uncmon_init_locked_l(curmonid);
1328 mon->um_sleeping = false;
1329 #if MACH_ASSERT
1330 mon->um_read_since_sleep = false;
1331 #endif /* MACH_ASSERT */
1332
1333 out:
1334 uncmon_unlock(mon, intrs_en);
1335 ml_set_interrupts_enabled(intr);
1336 }
1337
1338 #endif /* HAS_UNCORE_CTRS */
1339
1340 #pragma mark common hooks
1341
1342 void
mt_early_init(void)1343 mt_early_init(void)
1344 {
1345 topology_info = ml_get_topology_info();
1346 }
1347
1348 void
mt_cpu_idle(cpu_data_t * cpu)1349 mt_cpu_idle(cpu_data_t *cpu)
1350 {
1351 core_idle(cpu);
1352 }
1353
1354 void
mt_cpu_run(cpu_data_t * cpu)1355 mt_cpu_run(cpu_data_t *cpu)
1356 {
1357 struct mt_cpu *mtc;
1358
1359 assert(cpu != NULL);
1360 assert(ml_get_interrupts_enabled() == FALSE);
1361
1362 mtc = &cpu->cpu_monotonic;
1363
1364 for (int i = 0; i < MT_CORE_NFIXED; i++) {
1365 mt_core_set_snap(i, mtc->mtc_snaps[i]);
1366 }
1367
1368 /* re-enable the counters */
1369 core_init_execution_modes();
1370
1371 core_set_enabled();
1372 }
1373
1374 void
mt_cpu_down(cpu_data_t * cpu)1375 mt_cpu_down(cpu_data_t *cpu)
1376 {
1377 mt_cpu_idle(cpu);
1378 }
1379
1380 void
mt_cpu_up(cpu_data_t * cpu)1381 mt_cpu_up(cpu_data_t *cpu)
1382 {
1383 mt_cpu_run(cpu);
1384 }
1385
1386 void
mt_sleep(void)1387 mt_sleep(void)
1388 {
1389 #if HAS_UNCORE_CTRS
1390 uncore_save();
1391 #endif /* HAS_UNCORE_CTRS */
1392 }
1393
1394 void
mt_wake_per_core(void)1395 mt_wake_per_core(void)
1396 {
1397 #if HAS_UNCORE_CTRS
1398 if (mt_uncore_initted) {
1399 uncore_restore();
1400 }
1401 #endif /* HAS_UNCORE_CTRS */
1402 }
1403
1404 uint64_t
mt_count_pmis(void)1405 mt_count_pmis(void)
1406 {
1407 uint64_t npmis = 0;
1408 for (unsigned int i = 0; i < topology_info->num_cpus; i++) {
1409 cpu_data_t *cpu = (cpu_data_t *)CpuDataEntries[topology_info->cpus[i].cpu_id].cpu_data_vaddr;
1410 npmis += cpu->cpu_monotonic.mtc_npmis;
1411 }
1412 return npmis;
1413 }
1414
1415 static void
mt_cpu_pmi(cpu_data_t * cpu,uint64_t pmcr0)1416 mt_cpu_pmi(cpu_data_t *cpu, uint64_t pmcr0)
1417 {
1418 assert(cpu != NULL);
1419 assert(ml_get_interrupts_enabled() == FALSE);
1420
1421 __builtin_arm_wsr64("S3_1_C15_C0_0", PMCR0_INIT);
1422 /*
1423 * Ensure the CPMU has flushed any increments at this point, so PMSR is up
1424 * to date.
1425 */
1426 __builtin_arm_isb(ISB_SY);
1427
1428 cpu->cpu_monotonic.mtc_npmis += 1;
1429 cpu->cpu_stat.pmi_cnt_wake += 1;
1430
1431 #if MONOTONIC_DEBUG
1432 if (!PMCR0_PMI(pmcr0)) {
1433 kprintf("monotonic: mt_cpu_pmi but no PMI (PMCR0 = %#llx)\n",
1434 pmcr0);
1435 }
1436 #else /* MONOTONIC_DEBUG */
1437 #pragma unused(pmcr0)
1438 #endif /* !MONOTONIC_DEBUG */
1439
1440 uint64_t pmsr = __builtin_arm_rsr64("S3_1_C15_C13_0");
1441
1442 #if MONOTONIC_DEBUG
1443 printf("monotonic: cpu = %d, PMSR = 0x%llx, PMCR0 = 0x%llx\n",
1444 cpu_number(), pmsr, pmcr0);
1445 #endif /* MONOTONIC_DEBUG */
1446
1447 #if MACH_ASSERT
1448 uint64_t handled = 0;
1449 #endif /* MACH_ASSERT */
1450
1451 /*
1452 * monotonic handles any fixed counter PMIs.
1453 */
1454 for (unsigned int i = 0; i < MT_CORE_NFIXED; i++) {
1455 if ((pmsr & PMSR_OVF(i)) == 0) {
1456 continue;
1457 }
1458
1459 #if MACH_ASSERT
1460 handled |= 1ULL << i;
1461 #endif /* MACH_ASSERT */
1462 uint64_t count = mt_cpu_update_count(cpu, i);
1463 cpu->cpu_monotonic.mtc_counts[i] += count;
1464 mt_core_set_snap(i, mt_core_reset_values[i]);
1465 cpu->cpu_monotonic.mtc_snaps[i] = mt_core_reset_values[i];
1466
1467 if (mt_microstackshots && mt_microstackshot_ctr == i) {
1468 bool user_mode = false;
1469 arm_saved_state_t *state = get_user_regs(current_thread());
1470 if (state) {
1471 user_mode = PSR64_IS_USER(get_saved_state_cpsr(state));
1472 }
1473 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
1474 mt_microstackshot_ctr, user_mode);
1475 mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
1476 } else if (mt_debug) {
1477 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
1478 i, count);
1479 }
1480 }
1481
1482 /*
1483 * KPC handles the configurable counter PMIs.
1484 */
1485 for (unsigned int i = MT_CORE_NFIXED; i < CORE_NCTRS; i++) {
1486 if (pmsr & PMSR_OVF(i)) {
1487 #if MACH_ASSERT
1488 handled |= 1ULL << i;
1489 #endif /* MACH_ASSERT */
1490 extern void kpc_pmi_handler(unsigned int ctr);
1491 kpc_pmi_handler(i);
1492 }
1493 }
1494
1495 #if MACH_ASSERT
1496 uint64_t pmsr_after_handling = __builtin_arm_rsr64("S3_1_C15_C13_0");
1497 if (pmsr_after_handling != 0) {
1498 unsigned int first_ctr_ovf = __builtin_ffsll(pmsr_after_handling) - 1;
1499 uint64_t count = 0;
1500 const char *extra = "";
1501 if (first_ctr_ovf >= CORE_NCTRS) {
1502 extra = " (invalid counter)";
1503 } else {
1504 count = mt_core_snap(first_ctr_ovf);
1505 }
1506
1507 panic("monotonic: PMI status not cleared on exit from handler, "
1508 "PMSR = 0x%llx HANDLE -> -> 0x%llx, handled 0x%llx, "
1509 "PMCR0 = 0x%llx, PMC%d = 0x%llx%s", pmsr, pmsr_after_handling,
1510 handled, __builtin_arm_rsr64("S3_1_C15_C0_0"), first_ctr_ovf, count, extra);
1511 }
1512 #endif /* MACH_ASSERT */
1513
1514 core_set_enabled();
1515 }
1516
1517 #if CPMU_AIC_PMI
1518 void
mt_cpmu_aic_pmi(cpu_id_t source)1519 mt_cpmu_aic_pmi(cpu_id_t source)
1520 {
1521 struct cpu_data *curcpu = getCpuDatap();
1522 if (source != curcpu->interrupt_nub) {
1523 panic("monotonic: PMI from IOCPU %p delivered to %p", source,
1524 curcpu->interrupt_nub);
1525 }
1526 mt_cpu_pmi(curcpu, __builtin_arm_rsr64("S3_1_C15_C0_0"));
1527 }
1528 #endif /* CPMU_AIC_PMI */
1529
1530 void
mt_fiq(void * cpu,uint64_t pmcr0,uint64_t upmsr)1531 mt_fiq(void *cpu, uint64_t pmcr0, uint64_t upmsr)
1532 {
1533 #if CPMU_AIC_PMI
1534 #pragma unused(cpu, pmcr0)
1535 #else /* CPMU_AIC_PMI */
1536 mt_cpu_pmi(cpu, pmcr0);
1537 #endif /* !CPMU_AIC_PMI */
1538
1539 #if HAS_UNCORE_CTRS
1540 if (upmsr != 0) {
1541 uncore_fiq(upmsr);
1542 }
1543 #else /* HAS_UNCORE_CTRS */
1544 #pragma unused(upmsr)
1545 #endif /* !HAS_UNCORE_CTRS */
1546 }
1547
1548 void
mt_ownership_change(bool available)1549 mt_ownership_change(bool available)
1550 {
1551 #if HAS_UNCORE_CTRS
1552 /*
1553 * No need to take the lock here, as this is only manipulated in the UPMU
1554 * when the current task already owns the counters and is on its way out.
1555 */
1556 if (!available && uncore_active_ctrs) {
1557 uncore_reset();
1558 }
1559 #else
1560 #pragma unused(available)
1561 #endif /* HAS_UNCORE_CTRS */
1562 }
1563
1564 static uint32_t mt_xc_sync;
1565
1566 static void
mt_microstackshot_start_remote(__unused void * arg)1567 mt_microstackshot_start_remote(__unused void *arg)
1568 {
1569 cpu_data_t *cpu = getCpuDatap();
1570
1571 __builtin_arm_wsr64("S3_1_C15_C0_0", PMCR0_INIT);
1572
1573 for (int i = 0; i < MT_CORE_NFIXED; i++) {
1574 uint64_t count = mt_cpu_update_count(cpu, i);
1575 cpu->cpu_monotonic.mtc_counts[i] += count;
1576 mt_core_set_snap(i, mt_core_reset_values[i]);
1577 cpu->cpu_monotonic.mtc_snaps[i] = mt_core_reset_values[i];
1578 }
1579
1580 core_set_enabled();
1581
1582 if (os_atomic_dec(&mt_xc_sync, relaxed) == 0) {
1583 thread_wakeup((event_t)&mt_xc_sync);
1584 }
1585 }
1586
1587 int
mt_microstackshot_start_arch(uint64_t period)1588 mt_microstackshot_start_arch(uint64_t period)
1589 {
1590 uint64_t reset_value = 0;
1591 int ovf = os_sub_overflow(CTR_MAX, period, &reset_value);
1592 if (ovf) {
1593 return ERANGE;
1594 }
1595
1596 mt_core_reset_values[mt_microstackshot_ctr] = reset_value;
1597 cpu_broadcast_xcall(&mt_xc_sync, TRUE, mt_microstackshot_start_remote,
1598 mt_microstackshot_start_remote /* cannot pass NULL */);
1599 return 0;
1600 }
1601
1602 #pragma mark dev nodes
1603
1604 struct mt_device mt_devices[] = {
1605 [0] = {
1606 .mtd_name = "core",
1607 .mtd_init = core_init,
1608 },
1609 #if HAS_UNCORE_CTRS
1610 [1] = {
1611 .mtd_name = "uncore",
1612 .mtd_init = uncore_init,
1613 .mtd_add = uncore_add,
1614 .mtd_reset = uncore_reset,
1615 .mtd_enable = uncore_set_enabled,
1616 .mtd_read = uncore_read,
1617
1618 .mtd_ncounters = UNCORE_NCTRS,
1619 }
1620 #endif /* HAS_UNCORE_CTRS */
1621 };
1622
1623 static_assert(
1624 (sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
1625 "MT_NDEVS macro should be same as the length of mt_devices");
1626