1 // Copyright (c) 2021 Apple Inc. All rights reserved. 2 // 3 // @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 4 // 5 // This file contains Original Code and/or Modifications of Original Code 6 // as defined in and that are subject to the Apple Public Source License 7 // Version 2.0 (the 'License'). You may not use this file except in 8 // compliance with the License. The rights granted to you under the License 9 // may not be used to create, or enable the creation or redistribution of, 10 // unlawful or unlicensed copies of an Apple operating system, or to 11 // circumvent, violate, or enable the circumvention or violation of, any 12 // terms of an Apple operating system software license agreement. 13 // 14 // Please obtain a copy of the License at 15 // http://www.opensource.apple.com/apsl/ and read it before using this file. 16 // 17 // The Original Code and all software distributed under the License are 18 // distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 19 // EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 20 // INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 21 // FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 22 // Please see the License for the specific language governing rights and 23 // limitations under the License. 24 // 25 // @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 26 27 #ifndef KERN_RECOUNT_H 28 #define KERN_RECOUNT_H 29 30 #include <os/base.h> 31 #include <stdbool.h> 32 #include <stdint.h> 33 #include <sys/cdefs.h> 34 #include <sys/_types/_size_t.h> 35 36 __BEGIN_DECLS; 37 38 // Recount maintains counters for resources used by software, like CPU time and cycles. 39 // These counters are tracked at different levels of granularity depending on what execution bucket they're tracked in. 40 // For instance, while threads only differentiate on the broad CPU kinds due to memory constraints, 41 // the fewer number of tasks are free to use more memory and accumulate counters per-CPU. 42 // 43 // At context-switch, the scheduler calls `recount_context_switch` to update the counters. 44 // The difference between the current counter values and per-CPU snapshots are added to each thread. 45 // On modern systems with fast timebase reads, the counters are also updated on entering and exiting the kernel. 46 47 #pragma mark - config 48 49 // A level of the system's CPU topology, used as the precision when tracking 50 // counter values. 51 __enum_decl(recount_topo_t, unsigned int, { 52 RCT_TOPO_SYSTEM, 53 RCT_TOPO_CPU, 54 RCT_TOPO_CPU_KIND, 55 RCT_TOPO_COUNT, 56 }); 57 58 // Get the number of elements in an array for per-topography data. 59 size_t recount_topo_count(recount_topo_t topo); 60 61 // Recount's definitions of CPU kinds, in lieu of one from the platform layers. 62 __enum_decl(recount_cpu_kind_t, unsigned int, { 63 RCT_CPU_EFFICIENCY, 64 RCT_CPU_PERFORMANCE, 65 RCT_CPU_KIND_COUNT, 66 }); 67 68 // A `recount_plan` structure controls the granularity of counting for a set of 69 // tracks and must be consulted when updating their counters. 70 typedef const struct recount_plan { 71 const char *rpl_name; 72 recount_topo_t rpl_topo; 73 } *recount_plan_t; 74 75 #define RECOUNT_PLAN_DECLARE(_name) \ 76 extern const struct recount_plan _name; 77 78 #define RECOUNT_PLAN_DEFINE(_name, _topo) \ 79 const struct recount_plan _name = { \ 80 .rpl_name = #_name, \ 81 .rpl_topo = _topo, \ 82 } 83 84 // The current objects with resource accounting policies. 85 RECOUNT_PLAN_DECLARE(recount_thread_plan); 86 RECOUNT_PLAN_DECLARE(recount_task_plan); 87 RECOUNT_PLAN_DECLARE(recount_task_terminated_plan); 88 RECOUNT_PLAN_DECLARE(recount_coalition_plan); 89 RECOUNT_PLAN_DECLARE(recount_processor_plan); 90 91 #pragma mark - generic accounting 92 93 // A track is where counter values can be updated atomically for readers by a 94 // single writer. 95 struct recount_track { 96 uint32_t rt_sync; 97 uint32_t rt_pad; 98 99 // The CPU usage metrics currently supported by Recount. 100 struct recount_usage { 101 uint64_t ru_user_time_mach; 102 uint64_t ru_system_time_mach; 103 #if CONFIG_PERVASIVE_CPI 104 uint64_t ru_cycles; 105 uint64_t ru_instructions; 106 #endif // CONFIG_PERVASIVE_CPI 107 #if CONFIG_PERVASIVE_ENERGY 108 uint64_t ru_energy_nj; 109 #endif // CONFIG_PERVASIVE_ENERGY 110 } rt_usage; 111 }; 112 113 // Memory management routines for tracks and usage structures. 114 struct recount_track *recount_tracks_create(recount_plan_t plan); 115 void recount_tracks_destroy(recount_plan_t plan, struct recount_track *tracks); 116 struct recount_usage *recount_usage_alloc(recount_topo_t topo); 117 void recount_usage_free(recount_topo_t topo, struct recount_usage *usage); 118 119 // Attribute tracks to usage structures, to read their values for typical high-level interfaces. 120 121 // Sum any tracks to a single sum. 122 void recount_sum(recount_plan_t plan, const struct recount_track *tracks, 123 struct recount_usage *sum); 124 125 // Summarize tracks into a total sum and another for a particular CPU kind. 126 void recount_sum_and_isolate_cpu_kind(recount_plan_t plan, 127 struct recount_track *tracks, recount_cpu_kind_t kind, 128 struct recount_usage *sum, struct recount_usage *only_kind); 129 // The same as above, but for usage-only objects, like coalitions. 130 void recount_sum_usage_and_isolate_cpu_kind(recount_plan_t plan, 131 struct recount_usage *usage_list, recount_cpu_kind_t kind, 132 struct recount_usage *sum, struct recount_usage *only_kind); 133 134 // Sum the counters for each perf-level, in the order returned by the sysctls. 135 void recount_sum_perf_levels(recount_plan_t plan, 136 struct recount_track *tracks, struct recount_usage *sums); 137 138 #pragma mark - xnu internals 139 140 #if XNU_KERNEL_PRIVATE 141 142 struct thread; 143 struct task; 144 struct proc; 145 146 struct recount_times_mach { 147 uint64_t rtm_user; 148 uint64_t rtm_system; 149 }; 150 151 // Access another thread's usage data. 152 void recount_thread_usage(struct thread *thread, struct recount_usage *usage); 153 void recount_thread_perf_level_usage(struct thread *thread, 154 struct recount_usage *usage_levels); 155 uint64_t recount_thread_time_mach(struct thread *thread); 156 struct recount_times_mach recount_thread_times(struct thread *thread); 157 158 // Read the current thread's usage data, accumulating counts until now. 159 void recount_current_thread_usage(struct recount_usage *usage); 160 struct recount_times_mach recount_current_thread_times(void); 161 void recount_current_thread_usage_perf_only(struct recount_usage *usage, 162 struct recount_usage *usage_perf_only); 163 void recount_current_thread_perf_level_usage(struct recount_usage 164 *usage_levels); 165 uint64_t recount_current_thread_time_mach(void); 166 uint64_t recount_current_thread_user_time_mach(void); 167 uint64_t recount_current_thread_energy_nj(void); 168 void recount_current_task_usage(struct recount_usage *usage); 169 void recount_current_task_usage_perf_only(struct recount_usage *usage, 170 struct recount_usage *usage_perf_only); 171 172 // Access another task's usage data. 173 void recount_task_usage(struct task *task, struct recount_usage *usage); 174 struct recount_times_mach recount_task_times(struct task *task); 175 void recount_task_usage_perf_only(struct task *task, struct recount_usage *sum, 176 struct recount_usage *sum_perf_only); 177 void recount_task_times_perf_only(struct task *task, 178 struct recount_times_mach *sum, struct recount_times_mach *sum_perf_only); 179 uint64_t recount_task_energy_nj(struct task *task); 180 bool recount_task_thread_perf_level_usage(struct task *task, uint64_t tid, 181 struct recount_usage *usage_levels); 182 183 // Get the sum of all terminated threads in the task (not including active threads). 184 void recount_task_terminated_usage(struct task *task, 185 struct recount_usage *sum); 186 struct recount_times_mach recount_task_terminated_times(struct task *task); 187 void recount_task_terminated_usage_perf_only(struct task *task, 188 struct recount_usage *sum, struct recount_usage *perf_only); 189 190 int proc_pidthreadcounts(struct proc *p, uint64_t thuniqueid, user_addr_t uaddr, 191 size_t usize, int *ret); 192 193 #endif // XNU_KERNEL_PRIVATE 194 195 #if MACH_KERNEL_PRIVATE 196 197 #include <kern/smp.h> 198 #include <mach/machine/thread_status.h> 199 #include <machine/machine_routines.h> 200 201 #if __arm64__ 202 static_assert((RCT_CPU_EFFICIENCY > RCT_CPU_PERFORMANCE) == 203 (CLUSTER_TYPE_E > CLUSTER_TYPE_P)); 204 #endif // __arm64__ 205 206 #pragma mark threads 207 208 // The per-thread resource accounting structure. 209 struct recount_thread { 210 // Resources consumed across the lifetime of the thread, according to 211 // `recount_thread_plan`. 212 struct recount_track *rth_lifetime; 213 }; 214 void recount_thread_init(struct recount_thread *th); 215 void recount_thread_copy(struct recount_thread *dst, 216 struct recount_thread *src); 217 void recount_thread_deinit(struct recount_thread *th); 218 219 #pragma mark tasks 220 221 // The per-task resource accounting structure. 222 struct recount_task { 223 // Resources consumed across the lifetime of the task, including active 224 // threads, according to `recount_task_plan`. 225 // 226 // The `recount_task_plan` must be per-CPU to provide mutual exclusion for 227 // writers. 228 struct recount_track *rtk_lifetime; 229 // Usage from threads that have terminated or child tasks that have exited, 230 // according to `recount_task_terminated_plan`. 231 // 232 // Protected by the task lock when threads terminate. 233 struct recount_usage *rtk_terminated; 234 }; 235 void recount_task_init(struct recount_task *tk); 236 // Called on tasks that are moving their accounting information to a 237 // synthetic or re-exec-ed task. 238 void recount_task_copy(struct recount_task *dst, 239 const struct recount_task *src); 240 void recount_task_deinit(struct recount_task *tk); 241 242 #pragma mark coalitions 243 244 // The per-coalition resource accounting structure. 245 struct recount_coalition { 246 // Resources consumed by exited tasks only, according to 247 // `recount_coalition_plan`. 248 // 249 // Protected by the coalition lock when tasks exit and roll-up their 250 // statistics. 251 struct recount_usage *rco_exited; 252 }; 253 void recount_coalition_init(struct recount_coalition *co); 254 void recount_coalition_deinit(struct recount_coalition *co); 255 256 // Get the sum of all currently-exited tasks in the coalition, and a separate P-only structure. 257 void recount_coalition_usage_perf_only(struct recount_coalition *coal, 258 struct recount_usage *sum, struct recount_usage *sum_perf_only); 259 260 #pragma mark processors 261 262 struct processor; 263 264 // The per-processor resource accounting structure. 265 struct recount_processor { 266 struct recount_track rpr_active; 267 uint64_t rpr_idle_time_mach; 268 _Atomic uint64_t rpr_state_last_abs_time; 269 #if __AMP__ 270 // Cache the RCT_TOPO_CPU_KIND offset, which cannot change. 271 uint8_t rpr_cpu_kind_index; 272 #endif // __AMP__ 273 }; 274 void recount_processor_init(struct processor *processor); 275 276 // Get a snapshot of the processor's usage, along with an up-to-date snapshot 277 // of its idle time (to now if the processor is currently idle). 278 void recount_processor_usage(struct recount_processor *pr, 279 struct recount_usage *usage, uint64_t *idle_time_mach); 280 281 #pragma mark updates 282 283 // The following interfaces are meant for specific adopters, like the 284 // scheduler or platform code responsible for entering and exiting the kernel. 285 286 // A snap records counter values at a specific point in time. 287 struct recount_snap { 288 uint64_t rsn_time_mach; 289 #if CONFIG_PERVASIVE_CPI 290 uint64_t rsn_insns; 291 uint64_t rsn_cycles; 292 #endif // CONFIG_PERVASIVE_CPI 293 }; 294 295 // Fill in a snap with the current values from time- and count-keeping hardware. 296 void recount_snapshot(struct recount_snap *snap); 297 298 // During user/kernel transitions, other serializing events provide enough 299 // serialization around reading the counter values. 300 void recount_snapshot_speculative(struct recount_snap *snap); 301 302 // Called by the scheduler when a context switch occurs. 303 void recount_switch_thread(struct recount_snap *snap, struct thread *off_thread, 304 struct task *off_task); 305 // Called by the machine-dependent code to accumulate energy. 306 void recount_add_energy(struct thread *off_thread, struct task *off_task, 307 uint64_t energy_nj); 308 // Log a kdebug event on switching threads. 309 void recount_log_switch_thread(const struct recount_snap *snap); 310 311 // Called by the startup threads on each CPU to initialize Recount. 312 void recount_update_snap(struct recount_snap *cur); 313 314 // This function requires that no writers race with it -- this is only safe in 315 // debugger context or while running in the context of the track being 316 // inspected. 317 void recount_sum_unsafe(recount_plan_t plan, const struct recount_track *tracks, 318 struct recount_usage *sum); 319 320 // For handling precise user/kernel time updates. 321 void recount_leave_user(void); 322 void recount_enter_user(void); 323 #if __x86_64__ 324 // Handle interrupt time-keeping on Intel, which aren't unified with the trap 325 // handlers, so whether the user or system timers are updated depends on the 326 // save-state. 327 void recount_enter_intel_interrupt(x86_saved_state_t *state); 328 void recount_leave_intel_interrupt(void); 329 #endif // __x86_64__ 330 331 // Hooks for each processor idling and running. 332 void recount_processor_idle(struct recount_processor *pr, 333 struct recount_snap *snap); 334 void recount_processor_run(struct recount_processor *pr, 335 struct recount_snap *snap); 336 337 #pragma mark rollups 338 339 // Called by the thread termination queue with the task lock held. 340 void recount_task_rollup_thread(struct recount_task *tk, 341 const struct recount_thread *th); 342 343 // Called by the coalition roll-up statistics functions with coalition lock 344 // held. 345 void recount_coalition_rollup_task(struct recount_coalition *co, 346 struct recount_task *tk); 347 348 #endif // MACH_KERNEL_PRIVATE 349 350 __END_DECLS 351 352 #endif // KERN_RECOUNT_H 353