1 /*
2 * Copyright (c) 2012-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Boot Profiling
31 *
32 * The boot-profiling support is a mechanism to sample activity happening on the
33 * system during boot. This mechanism sets up a periodic timer and on every timer fire,
34 * captures a full backtrace into the boot profiling buffer. This buffer can be pulled
35 * out and analyzed from user-space. It is turned on using the following boot-args:
36 * "bootprofile_buffer_size" specifies the size of the boot profile buffer
37 * "bootprofile_interval_ms" specifies the interval for the profiling timer
38 *
39 * Process Specific Boot Profiling
40 *
41 * The boot-arg "bootprofile_proc_name" can be used to specify a certain
42 * process that needs to profiled during boot. Setting this boot-arg changes
43 * the way stackshots are captured. At every timer fire, the code looks at the
44 * currently running process and takes a stackshot only if the requested process
45 * is on-core (which makes it unsuitable for MP systems).
46 *
47 * Trigger Events
48 *
49 * The boot-arg "bootprofile_type=boot" starts the timer during early boot. Using
50 * "wake" starts the timer at AP wake from suspend-to-RAM.
51 */
52
53 #include <stdint.h>
54 #include <vm/vm_kern_xnu.h>
55 #include <kern/locks.h>
56 #include <kern/timer_call.h>
57 #include <kern/telemetry.h>
58 #include <pexpert/pexpert.h>
59
60 extern char *proc_name_address(void *p);
61 extern int proc_selfpid(void);
62
63 #define BOOTPROFILE_MAX_BUFFER_SIZE (64*1024*1024) /* see also COPYSIZELIMIT_PANIC */
64
65 vm_offset_t bootprofile_buffer = 0;
66 uint32_t bootprofile_buffer_current_position = 0;
67 uint64_t bootprofile_interval_abs = 0;
68 uint64_t bootprofile_next_deadline = 0;
69 uint32_t bootprofile_all_procs = 0;
70 uint64_t bootprofile_delta_since_timestamp = 0;
71 LCK_GRP_DECLARE(bootprofile_lck_grp, "bootprofile_group");
72 LCK_MTX_DECLARE(bootprofile_mtx, &bootprofile_lck_grp);
73
74 enum {
75 kBootProfileDisabled = 0,
76 kBootProfileStartTimerAtBoot,
77 kBootProfileStartTimerAtWake
78 } bootprofile_type = kBootProfileDisabled;
79
80 static timer_call_data_t bootprofile_timer_call_entry;
81
82 #define BOOTPROFILE_LOCK() do { lck_mtx_lock(&bootprofile_mtx); } while(0)
83 #define BOOTPROFILE_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&bootprofile_mtx)
84 #define BOOTPROFILE_UNLOCK() do { lck_mtx_unlock(&bootprofile_mtx); } while(0)
85
86 static void bootprofile_timer_call(
87 timer_call_param_t param0,
88 timer_call_param_t param1);
89
90 TUNABLE(uint32_t, bootprofile_buffer_size, "bootprofile_buffer_size", 0);
91 TUNABLE(uint32_t, bootprofile_interval_ms, "bootprofile_interval_ms", 0);
92 TUNABLE(uint64_t, bootprofile_stackshot_flags, "bootprofile_stackshot_flags", 0);
93 TUNABLE_STR(bootprofile_proc_name, 17, "bootprofile_proc_name", "");
94 TUNABLE_STR(bootprofile_type_name, 5, "bootprofile_type", "");
95
96 static void
_bootprofile_init(void)97 _bootprofile_init(void)
98 {
99 if (bootprofile_buffer_size > BOOTPROFILE_MAX_BUFFER_SIZE) {
100 bootprofile_buffer_size = BOOTPROFILE_MAX_BUFFER_SIZE;
101 }
102
103 if (bootprofile_proc_name[0] == '\0') {
104 bootprofile_all_procs = 1;
105 }
106
107 if (0 == strcmp(bootprofile_type_name, "boot")) {
108 bootprofile_type = kBootProfileStartTimerAtBoot;
109 } else if (0 == strcmp(bootprofile_type_name, "wake")) {
110 bootprofile_type = kBootProfileStartTimerAtWake;
111 } else {
112 bootprofile_type = kBootProfileDisabled;
113 }
114
115 clock_interval_to_absolutetime_interval(bootprofile_interval_ms, NSEC_PER_MSEC, &bootprofile_interval_abs);
116
117 /* Both boot args must be set to enable */
118 if ((bootprofile_type == kBootProfileDisabled) || (bootprofile_buffer_size == 0) || (bootprofile_interval_abs == 0)) {
119 return;
120 }
121
122 kern_return_t ret = kmem_alloc(kernel_map, &bootprofile_buffer, bootprofile_buffer_size,
123 KMA_DATA | KMA_ZERO | KMA_PERMANENT, VM_KERN_MEMORY_DIAG);
124 if (ret != KERN_SUCCESS) {
125 kprintf("Boot profile: Allocation failed: %d\n", ret);
126 return;
127 }
128
129 kprintf("Boot profile: Sampling %s once per %u ms at %s\n",
130 bootprofile_all_procs ? "all procs" : bootprofile_proc_name, bootprofile_interval_ms,
131 bootprofile_type == kBootProfileStartTimerAtBoot ? "boot" : (bootprofile_type == kBootProfileStartTimerAtWake ? "wake" : "unknown"));
132
133 timer_call_setup(&bootprofile_timer_call_entry,
134 bootprofile_timer_call,
135 NULL);
136
137 if (bootprofile_type == kBootProfileStartTimerAtBoot) {
138 bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs;
139 timer_call_enter_with_leeway(&bootprofile_timer_call_entry,
140 NULL,
141 bootprofile_next_deadline,
142 0,
143 TIMER_CALL_SYS_NORMAL,
144 false);
145 }
146 }
147
148 STARTUP(SYSCTL, STARTUP_RANK_FIRST, _bootprofile_init);
149
150 void
bootprofile_wake_from_sleep(void)151 bootprofile_wake_from_sleep(void)
152 {
153 if (bootprofile_type == kBootProfileStartTimerAtWake) {
154 bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs;
155 timer_call_enter_with_leeway(&bootprofile_timer_call_entry,
156 NULL,
157 bootprofile_next_deadline,
158 0,
159 TIMER_CALL_SYS_NORMAL,
160 false);
161 }
162 }
163
164 static void
bootprofile_timer_call(timer_call_param_t param0 __unused,timer_call_param_t param1 __unused)165 bootprofile_timer_call(
166 timer_call_param_t param0 __unused,
167 timer_call_param_t param1 __unused)
168 {
169 unsigned retbytes = 0;
170 int pid_to_profile = -1;
171
172 if (!BOOTPROFILE_TRY_SPIN_LOCK()) {
173 goto reprogram;
174 }
175
176 /* Check if process-specific boot profiling is turned on */
177 if (!bootprofile_all_procs) {
178 /*
179 * Since boot profiling initializes really early in boot, it is
180 * possible that at this point, the task/proc is not initialized.
181 * Nothing to do in that case.
182 */
183
184 if ((current_task() != NULL) && (get_bsdtask_info(current_task()) != NULL) &&
185 (0 == strncmp(bootprofile_proc_name, proc_name_address(get_bsdtask_info(current_task())), 17))) {
186 pid_to_profile = proc_selfpid();
187 } else {
188 /*
189 * Process-specific boot profiling requested but the on-core process is
190 * something else. Nothing to do here.
191 */
192 BOOTPROFILE_UNLOCK();
193 goto reprogram;
194 }
195 }
196
197 /* initiate a stackshot with whatever portion of the buffer is left */
198 if (bootprofile_buffer_current_position < bootprofile_buffer_size) {
199 uint64_t flags = STACKSHOT_KCDATA_FORMAT | STACKSHOT_TRYLOCK | STACKSHOT_SAVE_LOADINFO
200 | STACKSHOT_GET_GLOBAL_MEM_STATS;
201 #if defined(XNU_TARGET_OS_OSX)
202 flags |= STACKSHOT_SAVE_KEXT_LOADINFO;
203 #endif
204
205
206 /* OR on flags specified in boot-args */
207 flags |= bootprofile_stackshot_flags;
208 if ((flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) && (bootprofile_delta_since_timestamp == 0)) {
209 /* Can't take deltas until the first one */
210 flags &= ~STACKSHOT_COLLECT_DELTA_SNAPSHOT;
211 }
212
213 uint64_t timestamp = 0;
214 if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT) {
215 timestamp = mach_absolute_time();
216 }
217
218 kern_return_t r = stack_snapshot_from_kernel(
219 pid_to_profile, (void *)(bootprofile_buffer + bootprofile_buffer_current_position),
220 bootprofile_buffer_size - bootprofile_buffer_current_position,
221 flags, bootprofile_delta_since_timestamp, 0, &retbytes);
222
223 /*
224 * We call with STACKSHOT_TRYLOCK because the stackshot lock is coarser
225 * than the bootprofile lock. If someone else has the lock we'll just
226 * try again later.
227 */
228
229 if (r == KERN_LOCK_OWNED) {
230 BOOTPROFILE_UNLOCK();
231 goto reprogram;
232 }
233
234 if (bootprofile_stackshot_flags & STACKSHOT_COLLECT_DELTA_SNAPSHOT &&
235 r == KERN_SUCCESS) {
236 bootprofile_delta_since_timestamp = timestamp;
237 }
238
239 bootprofile_buffer_current_position += retbytes;
240 }
241
242 BOOTPROFILE_UNLOCK();
243
244 /* If we didn't get any data or have run out of buffer space, stop profiling */
245 if ((retbytes == 0) || (bootprofile_buffer_current_position == bootprofile_buffer_size)) {
246 return;
247 }
248
249
250 reprogram:
251 /* If the user gathered the buffer, no need to keep profiling */
252 if (bootprofile_interval_abs == 0) {
253 return;
254 }
255
256 clock_deadline_for_periodic_event(bootprofile_interval_abs,
257 mach_absolute_time(),
258 &bootprofile_next_deadline);
259 timer_call_enter_with_leeway(&bootprofile_timer_call_entry,
260 NULL,
261 bootprofile_next_deadline,
262 0,
263 TIMER_CALL_SYS_NORMAL,
264 false);
265 }
266
267 void
bootprofile_get(void ** buffer,uint32_t * length)268 bootprofile_get(void **buffer, uint32_t *length)
269 {
270 BOOTPROFILE_LOCK();
271 *buffer = (void*) bootprofile_buffer;
272 *length = bootprofile_buffer_current_position;
273 BOOTPROFILE_UNLOCK();
274 }
275
276 int
bootprofile_gather(user_addr_t buffer,uint32_t * length)277 bootprofile_gather(user_addr_t buffer, uint32_t *length)
278 {
279 int result = 0;
280
281 BOOTPROFILE_LOCK();
282
283 if (bootprofile_buffer == 0) {
284 *length = 0;
285 goto out;
286 }
287
288 if (*length < bootprofile_buffer_current_position) {
289 result = KERN_NO_SPACE;
290 goto out;
291 }
292
293 if ((result = copyout((void *)bootprofile_buffer, buffer,
294 bootprofile_buffer_current_position)) != 0) {
295 *length = 0;
296 goto out;
297 }
298 *length = bootprofile_buffer_current_position;
299
300 /* cancel future timers */
301 bootprofile_interval_abs = 0;
302
303 out:
304
305 BOOTPROFILE_UNLOCK();
306
307 return result;
308 }
309