1 /*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef KERN_KPC_H
30 #define KERN_KPC_H
31
32 /* Kernel interfaces to KPC PMC infrastructure. */
33
34 #include <machine/machine_kpc.h>
35 #include <kern/thread.h> /* thread_* */
36
37 __BEGIN_DECLS
38
39 /* cross-platform class constants */
40 #define KPC_CLASS_FIXED (0)
41 #define KPC_CLASS_CONFIGURABLE (1)
42 #define KPC_CLASS_POWER (2)
43 #define KPC_CLASS_RAWPMU (3)
44
45 #define KPC_CLASS_FIXED_MASK (1u << KPC_CLASS_FIXED)
46 #define KPC_CLASS_CONFIGURABLE_MASK (1u << KPC_CLASS_CONFIGURABLE)
47 #define KPC_CLASS_POWER_MASK (1u << KPC_CLASS_POWER)
48 #define KPC_CLASS_RAWPMU_MASK (1u << KPC_CLASS_RAWPMU)
49
50 #define KPC_PMU_ERROR (0)
51 #define KPC_PMU_INTEL_V3 (1)
52 #define KPC_PMU_ARM_APPLE (2)
53 #define KPC_PMU_INTEL_V2 (3)
54 #define KPC_PMU_ARM_V2 (4)
55
56 #define KPC_ALL_CPUS (1u << 31)
57
58 /* action id setters/getters */
59 #define FIXED_ACTIONID(ctr) (kpc_actionid[(ctr)])
60 #define CONFIGURABLE_ACTIONID(ctr) (kpc_actionid[(ctr) + kpc_fixed_count()])
61
62 /* reload counter setters/getters */
63 #define FIXED_RELOAD(ctr) (current_cpu_datap()->cpu_kpc_reload[(ctr)])
64 #define FIXED_RELOAD_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_reload[(ctr)])
65 #define CONFIGURABLE_RELOAD(ctr) (current_cpu_datap()->cpu_kpc_reload[(ctr) + kpc_fixed_count()])
66 #define CONFIGURABLE_RELOAD_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_reload[(ctr) + kpc_fixed_count()])
67
68 /* shadow counter setters/getters */
69 #define FIXED_SHADOW(ctr) (current_cpu_datap()->cpu_kpc_shadow[(ctr)])
70 #define FIXED_SHADOW_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_shadow[(ctr)])
71 #define CONFIGURABLE_SHADOW(ctr) (current_cpu_datap()->cpu_kpc_shadow[(ctr) + kpc_fixed_count()])
72 #define CONFIGURABLE_SHADOW_CPU(cpu, ctr) (cpu_datap(cpu)->cpu_kpc_shadow[(ctr) + kpc_fixed_count()])
73
74 /**
75 * Callback for notification when PMCs are acquired/released by a task. The
76 * argument is equal to TRUE if the Power Manager (PM) can use its reserved PMCs.
77 * Otherwise, the argument is equal to FALSE.
78 */
79 typedef void (*kpc_pm_handler_t)(boolean_t);
80
81 struct cpu_data;
82 extern boolean_t kpc_register_cpu(struct cpu_data *cpu_data);
83 extern void kpc_unregister_cpu(struct cpu_data *cpu_data);
84
85 extern bool kpc_supported;
86
87 /* bootstrap */
88 extern void kpc_init(void);
89
90 /* Architecture specific initialisation */
91 extern void kpc_arch_init(void);
92
93 /* Get the bitmask of available classes */
94 extern uint32_t kpc_get_classes(void);
95
96 /* Get the bitmask of currently running counter classes */
97 extern uint32_t kpc_get_running(void);
98
99 /* Get the version of KPC that's being run */
100 extern int kpc_get_pmu_version(void);
101
102 /* Set the bitmask of currently running counter classes. Specify
103 * classes = 0 to stop counters
104 */
105 extern int kpc_set_running(uint32_t classes);
106
107 /* Read CPU counters */
108 extern int kpc_get_cpu_counters(boolean_t all_cpus, uint32_t classes,
109 int *curcpu, uint64_t *buf);
110
111 /* Read shadow counters */
112 extern int kpc_get_shadow_counters( boolean_t all_cpus, uint32_t classes,
113 int *curcpu, uint64_t *buf );
114
115 /* Read current thread's counter accumulations */
116 extern int kpc_get_curthread_counters(uint32_t *inoutcount, uint64_t *buf);
117
118 /* Given a config, how many counters and config registers there are */
119 extern uint32_t kpc_get_counter_count(uint32_t classes);
120 extern uint32_t kpc_get_config_count(uint32_t classes);
121
122 /* enable/disable thread counting */
123 extern uint32_t kpc_get_thread_counting(void);
124 extern int kpc_set_thread_counting(uint32_t classes);
125
126 /* get and set config registers */
127 extern int kpc_get_config(uint32_t classes, kpc_config_t *current_config);
128 extern int kpc_set_config(uint32_t classes, kpc_config_t *new_config);
129
130 /* get and set PMI period */
131 extern int kpc_get_period(uint32_t classes, uint64_t *period);
132 extern int kpc_set_period(uint32_t classes, uint64_t *period);
133
134 /* get and set kperf actionid */
135 extern int kpc_get_actionid(uint32_t classes, uint32_t *actionid);
136 extern int kpc_set_actionid(uint32_t classes, uint32_t *actionid);
137
138 /* hooks on thread create and delete */
139 extern void kpc_thread_create(thread_t thread);
140 extern void kpc_thread_destroy(thread_t thread);
141
142 /* allocate a buffer big enough for all counters */
143 extern uint64_t *kpc_counterbuf_alloc(void);
144 extern void kpc_counterbuf_free(uint64_t*);
145 extern uint32_t kpc_get_counterbuf_size(void);
146
147 /* whether we're currently accounting into threads */
148 extern int kpc_threads_counting;
149
150 /* AST callback for KPC */
151 extern void kpc_thread_ast_handler( thread_t thread );
152
153 #if XNU_KERNEL_PRIVATE
154 int kpc_set_config_kernel(uint32_t classes, kpc_config_t *new_config);
155 #endif /* XNU_KERNEL_PRIVATE */
156
157 #ifdef MACH_KERNEL_PRIVATE
158
159 /* context switch callback for KPC */
160
161 extern boolean_t kpc_off_cpu_active;
162
163 extern void kpc_off_cpu_internal(thread_t thread);
164 extern void kpc_off_cpu_update(void);
165
166 static inline void
kpc_off_cpu(thread_t thread)167 kpc_off_cpu(thread_t thread)
168 {
169 if (__improbable(kpc_off_cpu_active)) {
170 kpc_off_cpu_internal(thread);
171 }
172 }
173
174 #endif /* defined(MACH_KERNEL_PRIVATE) */
175
176 /* acquire/release the counters used by the Power Manager */
177 extern int kpc_force_all_ctrs( task_t task, int val );
178 extern int kpc_get_force_all_ctrs( void );
179
180 /* arch-specific routine for acquire/release the counters used by the Power Manager */
181 extern int kpc_force_all_ctrs_arch( task_t task, int val );
182
183 extern int kpc_set_sw_inc( uint32_t mask );
184
185 /*
186 * Register the Power Manager as a PMCs user.
187 *
188 * This is a deprecated function used by old Power Managers, new Power Managers
189 * should use the @em kpc_reserve_pm_counters() function. This function actually
190 * calls @em kpc_reserve_pm_counters() with the following arguments:
191 * - handler = handler
192 * - pmc_mask = 0x83
193 * - custom_config = TRUE
194 *
195 * See @em kpc_reserve_pm_counters() for more details about the return value.
196 */
197 extern boolean_t kpc_register_pm_handler(void (*handler)(boolean_t));
198
199 /*
200 * Register the Power Manager as a PMCs user.
201 *
202 * @param handler
203 * Notification callback to use when PMCs are acquired/released by a task.
204 * Power management must acknowledge the change using kpc_pm_acknowledge.
205 *
206 * @param pmc_mask
207 * Bitmask of the configurable PMCs used by the Power Manager. The number of bits
208 * set must less or equal than the number of configurable counters
209 * available on the SoC.
210 *
211 * @param custom_config
212 * If custom_config=TRUE, the legacy sharing mode is enabled, otherwise the
213 * Modern Sharing mode is enabled. These modes are explained in more details in
214 * the kperf documentation.
215 *
216 * @return
217 * FALSE if a task has acquired all the PMCs, otherwise TRUE and the Power
218 * Manager can start using the reserved PMCs.
219 */
220 extern boolean_t kpc_reserve_pm_counters(uint64_t pmc_mask, kpc_pm_handler_t handler,
221 boolean_t custom_config);
222
223 /*
224 * Unregister the Power Manager as a PMCs user, and release the previously
225 * reserved counters.
226 */
227 extern void kpc_release_pm_counters(void);
228
229 /*
230 * Acknowledge the callback that PMCs are available to power management.
231 *
232 * @param available_to_pm Whether the counters were made available to power
233 * management in the callback. Pass in whatever was passed into the handler
234 * function. After this point, power management is able to use POWER_CLASS
235 * counters.
236 */
237 extern void kpc_pm_acknowledge(boolean_t available_to_pm);
238
239 /*
240 * Is the PMU used by both the power manager and userspace?
241 *
242 * This is true when the power manager has been registered. It disables certain
243 * counter configurations (like RAWPMU) that are incompatible with sharing
244 * counters.
245 */
246 extern boolean_t kpc_multiple_clients(void);
247
248 /*
249 * Is kpc controlling the fixed counters?
250 *
251 * This returns false when the power manager has requested custom configuration
252 * control.
253 */
254 extern boolean_t kpc_controls_fixed_counters(void);
255
256 /*
257 * Is kpc controlling a specific PMC ?
258 */
259 extern boolean_t kpc_controls_counter(uint32_t ctr);
260
261
262 extern void kpc_idle(void);
263 extern void kpc_idle_exit(void);
264
265
266 /*
267 * KPC PRIVATE
268 */
269
270 extern uint32_t kpc_actionid[KPC_MAX_COUNTERS];
271
272 /* handler for mp operations */
273 struct kpc_config_remote {
274 uint32_t classes;
275 kpc_config_t *configv;
276 uint64_t pmc_mask;
277 bool allow_list;
278 };
279
280 /* handler for mp operations */
281 struct kpc_running_remote {
282 uint32_t classes; /* classes to run */
283 uint64_t cfg_target_mask; /* configurable counters selected */
284 uint64_t cfg_state_mask; /* configurable counters new state */
285 };
286
287 /* handler for mp operations */
288 struct kpc_get_counters_remote {
289 uint32_t classes;
290 uint32_t nb_counters;
291 uint32_t buf_stride;
292 uint64_t *buf;
293 };
294
295 int kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf);
296 int kpc_get_curcpu_counters(uint32_t classes, int *curcpu, uint64_t *buf);
297 int kpc_get_fixed_counters(uint64_t *counterv);
298 int kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask);
299 boolean_t kpc_is_running_fixed(void);
300 boolean_t kpc_is_running_configurable(uint64_t pmc_mask);
301 uint32_t kpc_fixed_count(void);
302 uint32_t kpc_configurable_count(void);
303 uint32_t kpc_fixed_config_count(void);
304 uint32_t kpc_configurable_config_count(uint64_t pmc_mask);
305 uint32_t kpc_rawpmu_config_count(void);
306 int kpc_get_fixed_config(kpc_config_t *configv);
307 int kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask);
308 int kpc_get_rawpmu_config(kpc_config_t *configv);
309 uint64_t kpc_fixed_max(void);
310 uint64_t kpc_configurable_max(void);
311 int kpc_set_config_arch(struct kpc_config_remote *mp_config);
312 int kpc_set_period_arch(struct kpc_config_remote *mp_config);
313
314 __options_decl(kperf_kpc_flags_t, uint16_t, {
315 KPC_KERNEL_PC = 0x01, // the PC is a kernel address
316 KPC_KERNEL_COUNTING = 0x02, // the counter counts while running in the kernel
317 KPC_USER_COUNTING = 0x04, // the counter counts while running in user space
318 KPC_CAPTURED_PC = 0x08, // the PC was captured by hardware
319 });
320
321 void kpc_sample_kperf(uint32_t actionid, uint32_t counter, uint64_t config,
322 uint64_t count, uintptr_t pc, kperf_kpc_flags_t flags);
323
324 int kpc_set_running_arch(struct kpc_running_remote *mp_config);
325
326
327 /*
328 * Helpers
329 */
330
331 /* count the number of bits set */
332 extern uint8_t kpc_popcount(uint64_t value);
333
334 /* for a set of classes, retrieve the configurable PMCs mask */
335 extern uint64_t kpc_get_configurable_pmc_mask(uint32_t classes);
336
337
338 /* Interface for kexts to publish a kpc interface */
339 struct kpc_driver {
340 uint32_t (*get_classes)(void);
341 uint32_t (*get_running)(void);
342 int (*set_running)(uint32_t classes);
343 int (*get_cpu_counters)(boolean_t all_cpus, uint32_t classes,
344 int *curcpu, uint64_t *buf);
345 int (*get_curthread_counters)(uint32_t *inoutcount, uint64_t *buf);
346 uint32_t (*get_counter_count)(uint32_t classes);
347 uint32_t (*get_config_count)(uint32_t classes);
348 int (*get_config)(uint32_t classes, kpc_config_t *current_config);
349 int (*set_config)(uint32_t classes, kpc_config_t *new_config);
350 int (*get_period)(uint32_t classes, uint64_t *period);
351 int (*set_period)(uint32_t classes, uint64_t *period);
352 };
353
354 __END_DECLS
355
356 #endif /* !definde(KERN_KPC_H) */
357