1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/assert.h>
30 #include <kern/kpc.h>
31 #include <kern/monotonic.h>
32 #include <kern/thread.h>
33 #include <machine/atomic.h>
34 #include <machine/monotonic.h>
35 #include <mach/mach_traps.h>
36 #include <stdatomic.h>
37 #include <sys/errno.h>
38
39 bool mt_debug = false;
40 _Atomic uint64_t mt_pmis = 0;
41 _Atomic uint64_t mt_retrograde = 0;
42
43 #define MT_KDBG_INSTRS_CYCLES(CODE) \
44 KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_INSTRS_CYCLES, CODE)
45
46 static void mt_fixed_counts_internal(uint64_t *counts, uint64_t *counts_since);
47
48 uint64_t
mt_mtc_update_count(struct mt_cpu * mtc,unsigned int ctr)49 mt_mtc_update_count(struct mt_cpu *mtc, unsigned int ctr)
50 {
51 uint64_t snap = mt_core_snap(ctr);
52 if (snap < mtc->mtc_snaps[ctr]) {
53 if (mt_debug) {
54 kprintf("monotonic: cpu %d: thread %#llx: "
55 "retrograde counter %u value: %llu, last read = %llu\n",
56 cpu_number(), thread_tid(current_thread()), ctr, snap,
57 mtc->mtc_snaps[ctr]);
58 }
59 (void)atomic_fetch_add_explicit(&mt_retrograde, 1,
60 memory_order_relaxed);
61 mtc->mtc_snaps[ctr] = snap;
62 return 0;
63 }
64
65 uint64_t count = snap - mtc->mtc_snaps[ctr];
66 mtc->mtc_snaps[ctr] = snap;
67
68 return count;
69 }
70
71 uint64_t
mt_cpu_update_count(cpu_data_t * cpu,unsigned int ctr)72 mt_cpu_update_count(cpu_data_t *cpu, unsigned int ctr)
73 {
74 return mt_mtc_update_count(&cpu->cpu_monotonic, ctr);
75 }
76
77 static void
mt_fixed_counts_internal(uint64_t * counts,uint64_t * counts_since)78 mt_fixed_counts_internal(uint64_t *counts, uint64_t *counts_since)
79 {
80 assert(ml_get_interrupts_enabled() == FALSE);
81
82 struct mt_cpu *mtc = mt_cur_cpu();
83 assert(mtc != NULL);
84
85 mt_mtc_update_fixed_counts(mtc, counts, counts_since);
86 }
87
88 void
mt_mtc_update_fixed_counts(struct mt_cpu * mtc,uint64_t * counts,uint64_t * counts_since)89 mt_mtc_update_fixed_counts(struct mt_cpu *mtc, uint64_t *counts,
90 uint64_t *counts_since)
91 {
92 if (!mt_core_supported) {
93 return;
94 }
95
96 for (int i = 0; i < (int) kpc_fixed_count(); i++) {
97 uint64_t last_delta;
98 uint64_t count;
99
100 last_delta = mt_mtc_update_count(mtc, i);
101 count = mtc->mtc_counts[i] + last_delta;
102
103 if (counts) {
104 counts[i] = count;
105 }
106 if (counts_since) {
107 assert(counts != NULL);
108 counts_since[i] = count - mtc->mtc_counts_last[i];
109 mtc->mtc_counts_last[i] = count;
110 }
111
112 mtc->mtc_counts[i] = count;
113 }
114 }
115
116 void
mt_update_fixed_counts(void)117 mt_update_fixed_counts(void)
118 {
119 assert(ml_get_interrupts_enabled() == FALSE);
120
121 #if defined(__x86_64__)
122 __builtin_ia32_lfence();
123 #elif defined(__arm64__)
124 __builtin_arm_isb(ISB_SY);
125 #endif /* !defined(__x86_64__) && defined(__arm64__) */
126
127 mt_fixed_counts_internal(NULL, NULL);
128 }
129
130 void
mt_fixed_counts(uint64_t * counts)131 mt_fixed_counts(uint64_t *counts)
132 {
133 #if defined(__x86_64__)
134 __builtin_ia32_lfence();
135 #elif defined(__arm64__)
136 __builtin_arm_isb(ISB_SY);
137 #endif /* !defined(__x86_64__) && defined(__arm64__) */
138
139 int intrs_en = ml_set_interrupts_enabled(FALSE);
140 mt_fixed_counts_internal(counts, NULL);
141 ml_set_interrupts_enabled(intrs_en);
142 }
143
144 uint64_t
mt_cur_cpu_instrs(void)145 mt_cur_cpu_instrs(void)
146 {
147 uint64_t counts[MT_CORE_NFIXED];
148
149 if (!mt_core_supported) {
150 return 0;
151 }
152
153 mt_fixed_counts(counts);
154 return counts[MT_CORE_INSTRS];
155 }
156
157 uint64_t
mt_cur_cpu_cycles(void)158 mt_cur_cpu_cycles(void)
159 {
160 uint64_t counts[MT_CORE_NFIXED];
161
162 if (!mt_core_supported) {
163 return 0;
164 }
165
166 mt_fixed_counts(counts);
167 return counts[MT_CORE_CYCLES];
168 }
169
170 void
mt_cur_cpu_cycles_instrs_speculative(uint64_t * cycles,uint64_t * instrs)171 mt_cur_cpu_cycles_instrs_speculative(uint64_t *cycles, uint64_t *instrs)
172 {
173 #if defined(__arm64__)
174 if (!mt_core_supported) {
175 return;
176 }
177 struct mt_cpu *mtc = mt_cur_cpu();
178
179 /*
180 * Keep the MSRs back-to-back to improve throughput.
181 */
182 uint64_t cur_cycles = __builtin_arm_rsr64("PMC0");
183 uint64_t cur_instrs = __builtin_arm_rsr64("PMC1");
184
185 uint64_t cycles_sum = mtc->mtc_counts[MT_CORE_CYCLES];
186 uint64_t instrs_sum = mtc->mtc_counts[MT_CORE_INSTRS];
187 if (__probable(cur_cycles > mtc->mtc_snaps[MT_CORE_CYCLES])) {
188 cycles_sum += cur_cycles - mtc->mtc_snaps[MT_CORE_CYCLES];
189 }
190 if (__probable(cur_instrs > mtc->mtc_snaps[MT_CORE_INSTRS])) {
191 instrs_sum += cur_instrs - mtc->mtc_snaps[MT_CORE_INSTRS];
192 }
193
194 *cycles = cycles_sum;
195 *instrs = instrs_sum;
196 mtc->mtc_counts[MT_CORE_CYCLES] = cycles_sum;
197 mtc->mtc_counts[MT_CORE_INSTRS] = instrs_sum;
198 mtc->mtc_snaps[MT_CORE_CYCLES] = cur_cycles;
199 mtc->mtc_snaps[MT_CORE_INSTRS] = cur_instrs;
200 #else // defined(__arm64__)
201 uint64_t counts[MT_CORE_NFIXED] = {0};
202 struct mt_cpu *mtc = mt_cur_cpu();
203
204 assert(ml_get_interrupts_enabled() == FALSE);
205 assert(mtc != NULL);
206
207 mt_mtc_update_fixed_counts(mtc, counts, NULL);
208
209 *cycles = counts[MT_CORE_CYCLES];
210 *instrs = counts[MT_CORE_INSTRS];
211 #endif // !defined(__arm64__)
212 }
213
214 void
mt_perfcontrol(uint64_t * instrs,uint64_t * cycles)215 mt_perfcontrol(uint64_t *instrs, uint64_t *cycles)
216 {
217 if (!mt_core_supported) {
218 *instrs = 0;
219 *cycles = 0;
220 return;
221 }
222
223 struct mt_cpu *mtc = mt_cur_cpu();
224
225 /*
226 * The performance controller queries the hardware directly, so provide the
227 * last snapshot we took for the core. This is the value from when we
228 * updated the thread counts.
229 */
230
231 *instrs = mtc->mtc_snaps[MT_CORE_INSTRS];
232 *cycles = mtc->mtc_snaps[MT_CORE_CYCLES];
233 }
234
235 bool
mt_acquire_counters(void)236 mt_acquire_counters(void)
237 {
238 if (kpc_get_force_all_ctrs()) {
239 extern bool kpc_task_get_forced_all_ctrs(task_t);
240 if (kpc_task_get_forced_all_ctrs(current_task())) {
241 return true;
242 }
243 return false;
244 }
245 kpc_force_all_ctrs(current_task(), 1);
246 return true;
247 }
248
249 bool
mt_owns_counters(void)250 mt_owns_counters(void)
251 {
252 return kpc_get_force_all_ctrs();
253 }
254
255 void
mt_release_counters(void)256 mt_release_counters(void)
257 {
258 if (kpc_get_force_all_ctrs()) {
259 kpc_force_all_ctrs(current_task(), 0);
260 }
261 }
262
263 /*
264 * Maintain reset values for the fixed instruction and cycle counters so
265 * clients can be notified after a given number of those events occur. This is
266 * only used by microstackshot.
267 */
268
269 bool mt_microstackshots = false;
270 unsigned int mt_microstackshot_ctr = 0;
271 uint64_t mt_microstackshot_period = 0;
272 mt_pmi_fn mt_microstackshot_pmi_handler = NULL;
273 void *mt_microstackshot_ctx = NULL;
274 uint64_t mt_core_reset_values[MT_CORE_NFIXED] = { 0 };
275
276 #define MT_MIN_FIXED_PERIOD (10 * 1000 * 1000)
277
278 int
mt_microstackshot_start(unsigned int ctr,uint64_t period,mt_pmi_fn handler,void * ctx)279 mt_microstackshot_start(unsigned int ctr, uint64_t period, mt_pmi_fn handler,
280 void *ctx)
281 {
282 assert(ctr < MT_CORE_NFIXED);
283
284 if (period < MT_MIN_FIXED_PERIOD) {
285 return EINVAL;
286 }
287 if (mt_microstackshots) {
288 return EBUSY;
289 }
290
291 mt_microstackshot_ctr = ctr;
292 mt_microstackshot_pmi_handler = handler;
293 mt_microstackshot_ctx = ctx;
294
295 int error = mt_microstackshot_start_arch(period);
296 if (error) {
297 mt_microstackshot_ctr = 0;
298 mt_microstackshot_pmi_handler = NULL;
299 mt_microstackshot_ctx = NULL;
300 return error;
301 }
302
303 mt_microstackshot_period = period;
304 mt_microstackshots = true;
305
306 return 0;
307 }
308
309 int
mt_microstackshot_stop(void)310 mt_microstackshot_stop(void)
311 {
312 mt_microstackshots = false;
313 mt_microstackshot_period = 0;
314 memset(mt_core_reset_values, 0, sizeof(mt_core_reset_values));
315
316 return 0;
317 }
318