1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <i386/cpu_data.h>
30 #include <i386/cpuid.h>
31 #include <i386/lapic.h>
32 #include <i386/mp.h>
33 #include <i386/proc_reg.h>
34 #include <kern/assert.h> /* static_assert, assert */
35 #include <kern/monotonic.h>
36 #include <os/overflow.h>
37 #include <sys/errno.h>
38 #include <sys/monotonic.h>
39 #include <x86_64/monotonic.h>
40 #include <kern/kpc.h>
41
42 /*
43 * Sanity check the compiler.
44 */
45
46 #ifndef __has_builtin
47 #define __has_builtin(x) 0
48 #endif /* !defined(__has_builtin) */
49 #if !__has_builtin(__builtin_ia32_rdpmc)
50 #error requires __builtin_ia32_rdpmc builtin
51 #endif /* !__has_builtin(__builtin_ia32_rdpmc) */
52
53 #pragma mark core counters
54
55 bool mt_core_supported = false;
56
57 /*
58 * PMC[0-3]_{RD,WR} allow reading and writing the fixed PMCs.
59 *
60 * There are separate defines for access type because the read side goes through
61 * the rdpmc instruction, which has a different counter encoding than the msr
62 * path.
63 */
64 #define PMC_FIXED_RD(CTR) ((UINT64_C(1) << 30) | (CTR))
65 #define PMC_FIXED_WR(CTR) (MSR_IA32_PERF_FIXED_CTR0 + (CTR))
66 #define PMC0_RD PMC_FIXED_RD(0)
67 #define PMC0_WR PMC_FIXED_WR(0)
68 #define PMC1_RD PMC_FIXED_RD(1)
69 #define PMC1_WR PMC_FIXED_WR(1)
70 #define PMC2_RD PMC_FIXED_RD(2)
71 #define PMC2_WR PMC_FIXED_WR(2)
72 #define PMC3_RD PMC_FIXED_RD(3)
73 #define PMC3_WR PMC_FIXED_WR(3)
74
75 struct mt_cpu *
mt_cur_cpu(void)76 mt_cur_cpu(void)
77 {
78 return ¤t_cpu_datap()->cpu_monotonic;
79 }
80
81 uint64_t
mt_core_snap(unsigned int ctr)82 mt_core_snap(unsigned int ctr)
83 {
84 if (!mt_core_supported || ctr >= kpc_fixed_count()) {
85 return 0;
86 }
87
88 switch (ctr) {
89 case 0:
90 return __builtin_ia32_rdpmc(PMC0_RD);
91 case 1:
92 return __builtin_ia32_rdpmc(PMC1_RD);
93 case 2:
94 return __builtin_ia32_rdpmc(PMC2_RD);
95 case 3:
96 return __builtin_ia32_rdpmc(PMC3_RD);
97 default:
98 panic("monotonic: invalid core counter read: %u", ctr);
99 __builtin_unreachable();
100 }
101 }
102
103 void
mt_core_set_snap(unsigned int ctr,uint64_t count)104 mt_core_set_snap(unsigned int ctr, uint64_t count)
105 {
106 if (!mt_core_supported || ctr >= kpc_fixed_count()) {
107 return;
108 }
109
110 switch (ctr) {
111 case 0:
112 wrmsr64(PMC0_WR, count);
113 break;
114 case 1:
115 wrmsr64(PMC1_WR, count);
116 break;
117 case 2:
118 wrmsr64(PMC2_WR, count);
119 break;
120 case 3:
121 wrmsr64(PMC3_WR, count);
122 break;
123 default:
124 panic("monotonic: invalid core counter write: %u", ctr);
125 __builtin_unreachable();
126 }
127 }
128
129 /*
130 * MSR_IA32_PERF_FIXED_CTR_CTRL controls which rings fixed counters are
131 * enabled in and if they deliver PMIs.
132 *
133 * Each fixed counters has 4 bits: [0:1] controls which ring it's enabled in,
134 * [2] counts all hardware threads in each logical core (we don't want this),
135 * and [3] enables PMIs on overflow.
136 */
137
138 /*
139 * Fixed counters are enabled in all rings, so each _SINGLE definition is
140 * compounded in successive nibbles, dependent upon how many fixed counters
141 * are reported by cpuid.
142 */
143 #define FIXED_CTR_CTRL_INIT_SINGLE (0x8)
144 #define FIXED_CTR_CTRL_ENABLE_SINGLE (0x3)
145
146 static inline void
mt_fixed_counter_set_ctrl_mask(uint8_t ctrlbits)147 mt_fixed_counter_set_ctrl_mask(uint8_t ctrlbits)
148 {
149 uint64_t mask = 0;
150
151 assert((ctrlbits & 0xF0) == 0);
152
153 for (uint32_t i = 0; i < kpc_fixed_count(); i++) {
154 mask |= ctrlbits << (4 * i);
155 }
156
157 wrmsr64(MSR_IA32_PERF_FIXED_CTR_CTRL, mask);
158 }
159
160
161 /*
162 * GLOBAL_CTRL controls which counters are enabled -- the high 32-bits control
163 * the fixed counters and the lower half is for the configurable counters.
164 */
165
166 #define GLOBAL_CTRL 0x38f
167
168 /*
169 * Fixed counters are always enabled -- and there are three of them (except on
170 * Icelake, where there are four).
171 */
172 #define GLOBAL_CTRL_FIXED_EN_3FIXED (((UINT64_C(1) << 3) - 1) << 32)
173 #define GLOBAL_CTRL_FIXED_EN_4FIXED (((UINT64_C(1) << 4) - 1) << 32)
174
175 /*
176 * GLOBAL_STATUS reports the state of counters, like those that have overflowed.
177 */
178 #define GLOBAL_STATUS 0x38e
179
180 #define CTR_MAX ((UINT64_C(1) << 48) - 1)
181 #define CTR_FIX_POS(CTR) ((UINT64_C(1) << (CTR)) << 32)
182
183 #define GLOBAL_OVF 0x390
184
185 static void mt_check_for_pmi(struct mt_cpu *mtc, x86_saved_state_t *state);
186
187 static void
enable_counters(void)188 enable_counters(void)
189 {
190 mt_fixed_counter_set_ctrl_mask(FIXED_CTR_CTRL_INIT_SINGLE | FIXED_CTR_CTRL_ENABLE_SINGLE);
191
192 int fixed_count = kpc_fixed_count();
193 uint64_t global_en = (fixed_count == 4) ? GLOBAL_CTRL_FIXED_EN_4FIXED : GLOBAL_CTRL_FIXED_EN_3FIXED;
194 if (kpc_get_running() & KPC_CLASS_CONFIGURABLE_MASK) {
195 global_en |= kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK);
196 }
197
198 wrmsr64(GLOBAL_CTRL, global_en);
199 }
200
201 static void
disable_counters(void)202 disable_counters(void)
203 {
204 wrmsr64(GLOBAL_CTRL, 0);
205 }
206
207 static void
core_down(cpu_data_t * cpu)208 core_down(cpu_data_t *cpu)
209 {
210 if (!mt_core_supported) {
211 return;
212 }
213 assert(ml_get_interrupts_enabled() == FALSE);
214 struct mt_cpu *mtc = &cpu->cpu_monotonic;
215
216 disable_counters();
217 mt_mtc_update_fixed_counts(mtc, NULL, NULL);
218 mtc->mtc_active = false;
219 }
220
221 static void
core_up(cpu_data_t * cpu)222 core_up(cpu_data_t *cpu)
223 {
224 struct mt_cpu *mtc;
225
226 if (!mt_core_supported) {
227 return;
228 }
229
230 assert(ml_get_interrupts_enabled() == FALSE);
231
232 mtc = &cpu->cpu_monotonic;
233
234 for (uint32_t i = 0; i < kpc_fixed_count(); i++) {
235 mt_core_set_snap(i, mtc->mtc_snaps[i]);
236 }
237 enable_counters();
238 mtc->mtc_active = true;
239 }
240
241 void
mt_cpu_down(cpu_data_t * cpu)242 mt_cpu_down(cpu_data_t *cpu)
243 {
244 core_down(cpu);
245 }
246
247 void
mt_cpu_up(cpu_data_t * cpu)248 mt_cpu_up(cpu_data_t *cpu)
249 {
250 boolean_t intrs_en;
251 intrs_en = ml_set_interrupts_enabled(FALSE);
252 core_up(cpu);
253 ml_set_interrupts_enabled(intrs_en);
254 }
255
256 uint64_t
mt_count_pmis(void)257 mt_count_pmis(void)
258 {
259 uint64_t npmis = 0;
260 for (unsigned int i = 0; i < real_ncpus; i++) {
261 cpu_data_t *cpu = cpu_data_ptr[i];
262 npmis += cpu->cpu_monotonic.mtc_npmis;
263 }
264 return npmis;
265 }
266
267 static void
mt_check_for_pmi(struct mt_cpu * mtc,x86_saved_state_t * state)268 mt_check_for_pmi(struct mt_cpu *mtc, x86_saved_state_t *state)
269 {
270 uint64_t status = rdmsr64(GLOBAL_STATUS);
271
272 mtc->mtc_npmis += 1;
273
274 if (mtc->mtc_active) {
275 disable_counters();
276 }
277 for (uint32_t i = 0; i < kpc_fixed_count(); i++) {
278 if (status & CTR_FIX_POS(i)) {
279 uint64_t prior = CTR_MAX - mtc->mtc_snaps[i];
280 assert(prior <= CTR_MAX);
281 prior += 1; /* wrapped */
282
283 uint64_t delta = mt_mtc_update_count(mtc, i);
284 mtc->mtc_counts[i] += delta;
285
286 if (mt_microstackshots && mt_microstackshot_ctr == i) {
287 bool user_mode = false;
288 if (state) {
289 x86_saved_state64_t *state64 = saved_state64(state);
290 user_mode = (state64->isf.cs & 0x3) != 0;
291 }
292 KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
293 mt_microstackshot_ctr, user_mode);
294 mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
295 } else if (mt_debug) {
296 KDBG(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
297 mt_microstackshot_ctr, i);
298 }
299
300 mtc->mtc_snaps[i] = mt_core_reset_values[i];
301 mt_core_set_snap(i, mt_core_reset_values[i]);
302 }
303 }
304
305 /* if any of the configurable counters overflowed, tell kpc */
306 if (status & ((UINT64_C(1) << 4) - 1)) {
307 extern void kpc_pmi_handler(void);
308 kpc_pmi_handler();
309 }
310
311 if (mtc->mtc_active) {
312 enable_counters();
313 }
314 }
315
316 static int
mt_pmi_x86_64(x86_saved_state_t * state)317 mt_pmi_x86_64(x86_saved_state_t *state)
318 {
319 assert(ml_get_interrupts_enabled() == FALSE);
320 mt_check_for_pmi(mt_cur_cpu(), state);
321 return 0;
322 }
323
324 void
mt_ownership_change(bool __unused available)325 mt_ownership_change(bool __unused available)
326 {
327 }
328
329 static void
mt_microstackshot_start_remote(__unused void * arg)330 mt_microstackshot_start_remote(__unused void *arg)
331 {
332 struct mt_cpu *mtc = mt_cur_cpu();
333
334 mt_fixed_counter_set_ctrl_mask(FIXED_CTR_CTRL_INIT_SINGLE);
335
336 for (uint32_t i = 0; i < kpc_fixed_count(); i++) {
337 uint64_t delta = mt_mtc_update_count(mtc, i);
338 mtc->mtc_counts[i] += delta;
339 mt_core_set_snap(i, mt_core_reset_values[i]);
340 mtc->mtc_snaps[i] = mt_core_reset_values[i];
341 }
342
343 mt_fixed_counter_set_ctrl_mask(FIXED_CTR_CTRL_INIT_SINGLE | FIXED_CTR_CTRL_ENABLE_SINGLE);
344 }
345
346 int
mt_microstackshot_start_arch(uint64_t period)347 mt_microstackshot_start_arch(uint64_t period)
348 {
349 if (!mt_core_supported) {
350 return ENOTSUP;
351 }
352
353 uint64_t reset_value = 0;
354 int ovf = os_sub_overflow(CTR_MAX, period, &reset_value);
355 if (ovf) {
356 return ERANGE;
357 }
358
359 mt_core_reset_values[mt_microstackshot_ctr] = CTR_MAX - period;
360 mp_cpus_call(CPUMASK_ALL, ASYNC, mt_microstackshot_start_remote,
361 NULL);
362 return 0;
363 }
364
365 void
mt_early_init(void)366 mt_early_init(void)
367 {
368 if (PE_parse_boot_argn("-nomt_core", NULL, 0)) {
369 return;
370 }
371 i386_cpu_info_t *info = cpuid_info();
372 if (info->cpuid_arch_perf_leaf.version >= 2) {
373 lapic_set_pmi_func((i386_intr_func_t)mt_pmi_x86_64);
374 mt_core_supported = true;
375 }
376 }
377
378 static int
core_init(__unused mt_device_t dev)379 core_init(__unused mt_device_t dev)
380 {
381 return ENOTSUP;
382 }
383
384 #pragma mark common hooks
385
386 struct mt_device mt_devices[] = {
387 [0] = {
388 .mtd_name = "core",
389 .mtd_init = core_init
390 }
391 };
392
393 static_assert(
394 (sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
395 "MT_NDEVS macro should be same as the length of mt_devices");
396