1 // Copyright (c) 2021 Apple Inc. All rights reserved. 2 // 3 // @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 4 // 5 // This file contains Original Code and/or Modifications of Original Code 6 // as defined in and that are subject to the Apple Public Source License 7 // Version 2.0 (the 'License'). You may not use this file except in 8 // compliance with the License. The rights granted to you under the License 9 // may not be used to create, or enable the creation or redistribution of, 10 // unlawful or unlicensed copies of an Apple operating system, or to 11 // circumvent, violate, or enable the circumvention or violation of, any 12 // terms of an Apple operating system software license agreement. 13 // 14 // Please obtain a copy of the License at 15 // http://www.opensource.apple.com/apsl/ and read it before using this file. 16 // 17 // The Original Code and all software distributed under the License are 18 // distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 19 // EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 20 // INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 21 // FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 22 // Please see the License for the specific language governing rights and 23 // limitations under the License. 24 // 25 // @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 26 27 #ifndef KERN_RECOUNT_H 28 #define KERN_RECOUNT_H 29 30 #include <os/base.h> 31 #include <stdbool.h> 32 #include <stdint.h> 33 #include <sys/cdefs.h> 34 #include <sys/_types/_size_t.h> 35 36 __BEGIN_DECLS; 37 38 // Recount maintains counters for resources used by software, like CPU time and cycles. 39 // These counters are tracked at different levels of granularity depending on what execution bucket they're tracked in. 40 // For instance, while threads only differentiate on the broad CPU kinds due to memory constraints, 41 // the fewer number of tasks are free to use more memory and accumulate counters per-CPU. 42 // 43 // At context-switch, the scheduler calls `recount_context_switch` to update the counters. 44 // The difference between the current counter values and per-CPU snapshots are added to each thread. 45 // On modern systems with fast timebase reads, the counters are also updated on entering and exiting the kernel. 46 47 #pragma mark - config 48 49 // A domain of the system's CPU topology, used as granularity when tracking counter values. 50 __enum_decl(recount_topo_t, unsigned int, { 51 // Attribute counts to the entire system, i.e. only a single counter. 52 // Note that mutual exclusion must be provided to update this kind of counter. 53 RCT_TOPO_SYSTEM, 54 // Attribute counts to the CPU they accumulated on. 55 // Mutual exclusion is not required to update this counter, but preemption must be disabled. 56 RCT_TOPO_CPU, 57 // Attribute counts to the CPU kind (e.g. P or E). 58 // Note that mutual exclusion must be provided to update this kind of counter. 59 RCT_TOPO_CPU_KIND, 60 // The number of different topographies. 61 RCT_TOPO_COUNT, 62 }); 63 64 // Get the number of elements in an array for per-topography data. 65 size_t recount_topo_count(recount_topo_t topo); 66 67 // Recount's definitions of CPU kinds, in lieu of one from the platform layers. 68 __enum_decl(recount_cpu_kind_t, unsigned int, { 69 RCT_CPU_EFFICIENCY, 70 RCT_CPU_PERFORMANCE, 71 RCT_CPU_KIND_COUNT, 72 }); 73 74 // A `recount_plan` structure controls the granularity of counting for a set of tracks and must be consulted when updating their counters. 75 typedef const struct recount_plan { 76 const char *rpl_name; 77 recount_topo_t rpl_topo; 78 } *recount_plan_t; 79 80 #define RECOUNT_PLAN_DECLARE(_name) \ 81 extern const struct recount_plan _name; 82 83 #define RECOUNT_PLAN_DEFINE(_name, _topo) \ 84 const struct recount_plan _name = { \ 85 .rpl_name = #_name, \ 86 .rpl_topo = _topo, \ 87 } 88 89 // The current objects with resource accounting policies. 90 RECOUNT_PLAN_DECLARE(recount_thread_plan); 91 RECOUNT_PLAN_DECLARE(recount_task_plan); 92 RECOUNT_PLAN_DECLARE(recount_task_terminated_plan); 93 RECOUNT_PLAN_DECLARE(recount_coalition_plan); 94 RECOUNT_PLAN_DECLARE(recount_processor_plan); 95 96 #pragma mark - generic accounting 97 98 // A track is where counter values can be updated atomically for readers by a 99 // single writer. 100 struct recount_track { 101 // Used to synchronize updates so multiple values appear to be updated atomically. 102 uint32_t rt_sync; 103 uint32_t rt_pad; 104 105 // The CPU usage metrics currently supported by Recount. 106 struct recount_usage { 107 // Basic time tracking, in units of Mach time. 108 uint64_t ru_user_time_mach; 109 uint64_t ru_system_time_mach; 110 #if CONFIG_PERVASIVE_CPI 111 // CPU performance counter metrics, when available. 112 uint64_t ru_cycles; 113 uint64_t ru_instructions; 114 #endif // CONFIG_PERVASIVE_CPI 115 #if CONFIG_PERVASIVE_ENERGY 116 // CPU energy in nanojoules, when available. 117 // This metric is updated out-of-band from the others because it can only be sampled by ApplePMGR through CLPC. 118 uint64_t ru_energy_nj; 119 #endif // CONFIG_PERVASIVE_ENERGY 120 } rt_usage; 121 }; 122 123 // Memory management routines for tracks and usage structures. 124 struct recount_track *recount_tracks_create(recount_plan_t plan); 125 void recount_tracks_destroy(recount_plan_t plan, struct recount_track *tracks); 126 struct recount_usage *recount_usage_alloc(recount_topo_t topo); 127 void recount_usage_free(recount_topo_t topo, struct recount_usage *usage); 128 129 // Attribute tracks to usage structures, to read their values for typical high-level interfaces. 130 131 // Sum any tracks to a single sum. 132 void recount_sum(recount_plan_t plan, const struct recount_track *tracks, 133 struct recount_usage *sum); 134 135 // Summarize tracks into a total sum and another for a particular CPU kind. 136 void recount_sum_and_isolate_cpu_kind(recount_plan_t plan, 137 struct recount_track *tracks, recount_cpu_kind_t kind, 138 struct recount_usage *sum, struct recount_usage *only_kind); 139 // The same as above, but for usage-only objects, like coalitions. 140 void recount_sum_usage_and_isolate_cpu_kind(recount_plan_t plan, 141 struct recount_usage *usage_list, recount_cpu_kind_t kind, 142 struct recount_usage *sum, struct recount_usage *only_kind); 143 144 // Sum the counters for each perf-level, in the order returned by the sysctls. 145 void recount_sum_perf_levels(recount_plan_t plan, 146 struct recount_track *tracks, struct recount_usage *sums); 147 148 #pragma mark - xnu internals 149 150 #if XNU_KERNEL_PRIVATE 151 152 struct thread; 153 struct task; 154 struct proc; 155 156 // A smaller usage structure if only times are needed by a client. 157 struct recount_times_mach { 158 uint64_t rtm_user; 159 uint64_t rtm_system; 160 }; 161 162 // Access another thread's usage data. 163 void recount_thread_usage(struct thread *thread, struct recount_usage *usage); 164 void recount_thread_perf_level_usage(struct thread *thread, 165 struct recount_usage *usage_levels); 166 uint64_t recount_thread_time_mach(struct thread *thread); 167 struct recount_times_mach recount_thread_times(struct thread *thread); 168 169 // Read the current thread's usage data, accumulating counts until now. 170 // 171 // Interrupts must be disabled. 172 void recount_current_thread_usage(struct recount_usage *usage); 173 struct recount_times_mach recount_current_thread_times(void); 174 void recount_current_thread_usage_perf_only(struct recount_usage *usage, 175 struct recount_usage *usage_perf_only); 176 void recount_current_thread_perf_level_usage(struct recount_usage 177 *usage_levels); 178 uint64_t recount_current_thread_time_mach(void); 179 uint64_t recount_current_thread_user_time_mach(void); 180 uint64_t recount_current_thread_energy_nj(void); 181 void recount_current_task_usage(struct recount_usage *usage); 182 void recount_current_task_usage_perf_only(struct recount_usage *usage, 183 struct recount_usage *usage_perf_only); 184 185 // Access another task's usage data. 186 void recount_task_usage(struct task *task, struct recount_usage *usage); 187 struct recount_times_mach recount_task_times(struct task *task); 188 void recount_task_usage_perf_only(struct task *task, struct recount_usage *sum, 189 struct recount_usage *sum_perf_only); 190 void recount_task_times_perf_only(struct task *task, 191 struct recount_times_mach *sum, struct recount_times_mach *sum_perf_only); 192 uint64_t recount_task_energy_nj(struct task *task); 193 bool recount_task_thread_perf_level_usage(struct task *task, uint64_t tid, 194 struct recount_usage *usage_levels); 195 196 // Get the sum of all terminated threads in the task (not including active threads). 197 void recount_task_terminated_usage(struct task *task, 198 struct recount_usage *sum); 199 struct recount_times_mach recount_task_terminated_times(struct task *task); 200 void recount_task_terminated_usage_perf_only(struct task *task, 201 struct recount_usage *sum, struct recount_usage *perf_only); 202 203 int proc_pidthreadcounts(struct proc *p, uint64_t thuniqueid, user_addr_t uaddr, 204 size_t usize, int *ret); 205 206 #endif // XNU_KERNEL_PRIVATE 207 208 #if MACH_KERNEL_PRIVATE 209 210 #include <kern/smp.h> 211 #include <mach/machine/thread_status.h> 212 #include <machine/machine_routines.h> 213 214 #if __arm64__ 215 static_assert((RCT_CPU_EFFICIENCY > RCT_CPU_PERFORMANCE) == 216 (CLUSTER_TYPE_E > CLUSTER_TYPE_P)); 217 #endif // __arm64__ 218 219 #pragma mark threads 220 221 // The per-thread resource accounting structure. 222 struct recount_thread { 223 // Resources consumed across the lifetime of the thread, according to 224 // `recount_thread_plan`. 225 struct recount_track *rth_lifetime; 226 }; 227 void recount_thread_init(struct recount_thread *th); 228 void recount_thread_copy(struct recount_thread *dst, 229 struct recount_thread *src); 230 void recount_thread_deinit(struct recount_thread *th); 231 232 #pragma mark tasks 233 234 // The per-task resource accounting structure. 235 struct recount_task { 236 // Resources consumed across the lifetime of the task, including active 237 // threads, according to `recount_task_plan`. 238 // 239 // The `recount_task_plan` must be per-CPU to provide mutual exclusion for 240 // writers. 241 struct recount_track *rtk_lifetime; 242 // Usage from threads that have terminated or child tasks that have exited, 243 // according to `recount_task_terminated_plan`. 244 // 245 // Protected by the task lock when threads terminate. 246 struct recount_usage *rtk_terminated; 247 }; 248 void recount_task_init(struct recount_task *tk); 249 // Called on tasks that are moving their accounting information to a 250 // synthetic or re-exec-ed task. 251 void recount_task_copy(struct recount_task *dst, 252 const struct recount_task *src); 253 void recount_task_deinit(struct recount_task *tk); 254 255 #pragma mark coalitions 256 257 // The per-coalition resource accounting structure. 258 struct recount_coalition { 259 // Resources consumed by exited tasks only, according to 260 // `recount_coalition_plan`. 261 // 262 // Protected by the coalition lock when tasks exit and roll-up their 263 // statistics. 264 struct recount_usage *rco_exited; 265 }; 266 void recount_coalition_init(struct recount_coalition *co); 267 void recount_coalition_deinit(struct recount_coalition *co); 268 269 // Get the sum of all currently-exited tasks in the coalition, and a separate P-only structure. 270 void recount_coalition_usage_perf_only(struct recount_coalition *coal, 271 struct recount_usage *sum, struct recount_usage *sum_perf_only); 272 273 #pragma mark processors 274 275 struct processor; 276 277 // A snap records counter values at a specific point in time. 278 struct recount_snap { 279 uint64_t rsn_time_mach; 280 #if CONFIG_PERVASIVE_CPI 281 uint64_t rsn_insns; 282 uint64_t rsn_cycles; 283 #endif // CONFIG_PERVASIVE_CPI 284 }; 285 286 // The per-processor resource accounting structure. 287 struct recount_processor { 288 struct recount_snap rpr_snap; 289 struct recount_track rpr_active; 290 uint64_t rpr_idle_time_mach; 291 _Atomic uint64_t rpr_state_last_abs_time; 292 #if __AMP__ 293 // Cache the RCT_TOPO_CPU_KIND offset, which cannot change. 294 uint8_t rpr_cpu_kind_index; 295 #endif // __AMP__ 296 }; 297 void recount_processor_init(struct processor *processor); 298 299 // Get a snapshot of the processor's usage, along with an up-to-date snapshot 300 // of its idle time (to now if the processor is currently idle). 301 void recount_processor_usage(struct recount_processor *pr, 302 struct recount_usage *usage, uint64_t *idle_time_mach); 303 304 #pragma mark updates 305 306 // The following interfaces are meant for specific adopters, like the 307 // scheduler or platform code responsible for entering and exiting the kernel. 308 309 // Fill in a snap with the current values from time- and count-keeping hardware. 310 void recount_snapshot(struct recount_snap *snap); 311 312 // During user/kernel transitions, other serializing events provide enough 313 // serialization around reading the counter values. 314 void recount_snapshot_speculative(struct recount_snap *snap); 315 316 // Called by the scheduler when a context switch occurs. 317 void recount_switch_thread(struct recount_snap *snap, struct thread *off_thread, 318 struct task *off_task); 319 // Called by the machine-dependent code to accumulate energy. 320 void recount_add_energy(struct thread *off_thread, struct task *off_task, 321 uint64_t energy_nj); 322 // Log a kdebug event on switching threads. 323 void recount_log_switch_thread(const struct recount_snap *snap); 324 325 // Called by the startup threads on each CPU to initialize Recount. 326 void recount_update_snap(struct recount_snap *cur); 327 328 // This function requires that no writers race with it -- this is only safe in 329 // debugger context or while running in the context of the track being 330 // inspected. 331 void recount_sum_unsafe(recount_plan_t plan, const struct recount_track *tracks, 332 struct recount_usage *sum); 333 334 // For handling precise user/kernel time updates. 335 void recount_leave_user(void); 336 void recount_enter_user(void); 337 #if __x86_64__ 338 // Handle interrupt time-keeping on Intel, which aren't unified with the trap 339 // handlers, so whether the user or system timers are updated depends on the 340 // save-state. 341 void recount_enter_intel_interrupt(x86_saved_state_t *state); 342 void recount_leave_intel_interrupt(void); 343 #endif // __x86_64__ 344 345 // Hooks for each processor idling and running. 346 void recount_processor_idle(struct recount_processor *pr, 347 struct recount_snap *snap); 348 void recount_processor_run(struct recount_processor *pr, 349 struct recount_snap *snap); 350 351 #pragma mark rollups 352 353 // Called by the thread termination queue with the task lock held. 354 void recount_task_rollup_thread(struct recount_task *tk, 355 const struct recount_thread *th); 356 357 // Called by the coalition roll-up statistics functions with coalition lock 358 // held. 359 void recount_coalition_rollup_task(struct recount_coalition *co, 360 struct recount_task *tk); 361 362 #endif // MACH_KERNEL_PRIVATE 363 364 __END_DECLS 365 366 #endif // KERN_RECOUNT_H 367