xref: /xnu-8020.101.4/tests/stackshot_accuracy.m (revision e7776783b89a353188416a9a346c6cdb4928faad)
1#include <darwintest.h>
2#include <darwintest_utils.h>
3#include <sys/kern_memorystatus.h>
4#include <kern/debug.h>
5#include <mach-o/dyld.h>
6#include <sys/stackshot.h>
7#include <kdd.h>
8#include <signal.h>
9
10#define RECURSIONS 25
11#define FIRST_RECURSIVE_FRAME 3
12
13T_GLOBAL_META(
14		T_META_NAMESPACE("xnu.stackshot.accuracy"),
15		T_META_RADAR_COMPONENT_NAME("xnu"),
16		T_META_RADAR_COMPONENT_VERSION("stackshot"),
17		T_META_OWNER("jonathan_w_adams"),
18		T_META_CHECK_LEAKS(false),
19		T_META_ASROOT(true)
20		);
21
22
23void child_init(void);
24void parent_helper_singleproc(int);
25
26#define CHECK_FOR_FAULT_STATS         (1 << 0)
27#define WRITE_STACKSHOT_BUFFER_TO_TMP (1 << 1)
28#define CHECK_FOR_KERNEL_THREADS      (1 << 2)
29int check_stackshot(void *, int);
30
31/* used for WRITE_STACKSHOT_BUFFER_TO_TMP */
32static char const *current_scenario_name;
33static pid_t child_pid;
34
35/* helpers */
36
37static void __attribute__((noinline))
38child_recurse(int r, int spin, void (^cb)(void))
39{
40	if (r > 0) {
41		child_recurse(r - 1, spin, cb);
42	}
43
44	cb();
45
46	/* wait forever */
47	if (spin == 0) {
48		sleep(100000);
49	} else if (spin == 2) {
50		int v = 1;
51		/* ssh won't let the session die if we still have file handles open to its output. */
52		close(STDERR_FILENO);
53		close(STDOUT_FILENO);
54		T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.wedge_thread", NULL, NULL, &v, sizeof(v)),
55					"wedged thread in the kernel");
56	} else {
57		while (1) {
58			__asm__ volatile("" : : : "memory");
59		}
60	}
61}
62
63T_HELPER_DECL(simple_child_process, "child process that will be frozen and others")
64{
65	child_init();
66}
67
68T_HELPER_DECL(sid_child_process, "child process that setsid()s")
69{
70	pid_t ppid = getppid();
71
72	T_ASSERT_POSIX_SUCCESS(setsid(), "session id set");
73
74	child_recurse(RECURSIONS, 2, ^{
75		kill(ppid, SIGUSR1);
76	});
77
78	T_ASSERT_FAIL("child_init returned!");
79}
80
81static void
82kill_children(void)
83{
84	kill(child_pid, SIGKILL);
85}
86
87static void *
88take_stackshot(pid_t target_pid, uint64_t extra_flags, uint64_t since_timestamp)
89{
90	void *stackshot_config;
91	int err, retries = 5;
92	uint64_t stackshot_flags = STACKSHOT_KCDATA_FORMAT |
93								STACKSHOT_THREAD_WAITINFO |
94								STACKSHOT_GET_DQ;
95
96	/* we should be able to verify delta stackshots */
97	if (since_timestamp != 0) {
98		stackshot_flags |= STACKSHOT_COLLECT_DELTA_SNAPSHOT;
99	}
100
101	stackshot_flags |= extra_flags;
102
103	stackshot_config = stackshot_config_create();
104	T_ASSERT_NOTNULL(stackshot_config, "allocate stackshot config");
105
106	err = stackshot_config_set_flags(stackshot_config, stackshot_flags);
107	T_ASSERT_EQ(err, 0, "set flags on stackshot config");
108
109	err = stackshot_config_set_pid(stackshot_config, target_pid);
110	T_ASSERT_EQ(err, 0, "set target pid on stackshot config");
111
112	if (since_timestamp != 0) {
113		err = stackshot_config_set_delta_timestamp(stackshot_config, since_timestamp);
114		T_ASSERT_EQ(err, 0, "set prev snapshot time on stackshot config");
115	}
116
117	while (retries > 0) {
118		err = stackshot_capture_with_config(stackshot_config);
119		if (err == 0) {
120			break;
121		} else if (err == EBUSY || err == ETIMEDOUT) {
122			T_LOG("stackshot capture returned %d (%s)\n", err, strerror(err));
123			if (retries == 0) {
124				T_ASSERT_FAIL("failed to take stackshot with error after retries: %d: %s\n", err, strerror(err));
125			}
126
127			retries--;
128			continue;
129		} else {
130			T_ASSERT_FAIL("failed to take stackshot with error: %d: %s\n", err, strerror(err));
131		}
132	}
133
134	return stackshot_config;
135}
136
137int
138check_stackshot(void *stackshot_config, int flags)
139{
140	void *buf;
141	uint32_t buflen, kcdata_type;
142	kcdata_iter_t iter;
143	NSError *nserror = nil;
144	pid_t target_pid;
145	int ret = 0;
146	uint64_t expected_return_addr = 0;
147	bool found_fault_stats = false;
148	struct stackshot_fault_stats fault_stats = {0};
149
150	buf = stackshot_config_get_stackshot_buffer(stackshot_config);
151	T_ASSERT_NOTNULL(buf, "stackshot buffer is not null");
152	buflen = stackshot_config_get_stackshot_size(stackshot_config);
153	T_ASSERT_GT(buflen, 0, "valid stackshot buffer length");
154	target_pid = ((struct stackshot_config*)stackshot_config)->sc_pid;
155	T_ASSERT_GT(target_pid, 0, "valid target_pid");
156
157	/* if need to write it to fs, do it now */
158	if (flags & WRITE_STACKSHOT_BUFFER_TO_TMP) {
159		char sspath[MAXPATHLEN];
160		strlcpy(sspath, current_scenario_name, sizeof(sspath));
161		strlcat(sspath, ".kcdata", sizeof(sspath));
162		T_QUIET; T_ASSERT_POSIX_ZERO(dt_resultfile(sspath, sizeof(sspath)),
163				"create result file path");
164
165		FILE *f = fopen(sspath, "w");
166		T_WITH_ERRNO; T_QUIET; T_ASSERT_NOTNULL(f,
167				"open stackshot output file");
168
169		size_t written = fwrite(buf, buflen, 1, f);
170		T_QUIET; T_ASSERT_POSIX_SUCCESS(written, "wrote stackshot to file");
171
172		fclose(f);
173	}
174
175	/* begin iterating */
176	iter = kcdata_iter(buf, buflen);
177	T_ASSERT_EQ(kcdata_iter_type(iter), KCDATA_BUFFER_BEGIN_STACKSHOT, "buffer is a stackshot");
178
179	/* time to iterate */
180	iter = kcdata_iter_next(iter);
181	KCDATA_ITER_FOREACH(iter) {
182		kcdata_type = kcdata_iter_type(iter);
183		NSNumber *parsedPid;
184		NSMutableDictionary *parsedContainer, *parsedThreads;
185
186		if ((flags & CHECK_FOR_FAULT_STATS) != 0 &&
187				kcdata_type == STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS) {
188			memcpy(&fault_stats, kcdata_iter_payload(iter), sizeof(fault_stats));
189			found_fault_stats = true;
190		}
191
192		if (kcdata_type != KCDATA_TYPE_CONTAINER_BEGIN) {
193			continue;
194		}
195
196		if (kcdata_iter_container_type(iter) != STACKSHOT_KCCONTAINER_TASK) {
197			continue;
198		}
199
200		parsedContainer = parseKCDataContainer(&iter, &nserror);
201		T_ASSERT_NOTNULL(parsedContainer, "parsedContainer is not null");
202		T_ASSERT_NULL(nserror, "no NSError occured while parsing the kcdata container");
203
204		/*
205		 * given that we've targetted the pid, we can be sure that this
206		 * ts_pid will be the pid we expect
207		 */
208		parsedPid = parsedContainer[@"task_snapshots"][@"task_snapshot"][@"ts_pid"];
209		T_ASSERT_EQ([parsedPid intValue], target_pid, "found correct pid");
210
211		/* start parsing the threads */
212		parsedThreads = parsedContainer[@"task_snapshots"][@"thread_snapshots"];
213		for (id th_key in parsedThreads) {
214			uint32_t frame_index = 0;
215
216			if ((flags & CHECK_FOR_KERNEL_THREADS) == 0) {
217				/* skip threads that don't have enough frames */
218				if ([parsedThreads[th_key][@"user_stack_frames"] count] < RECURSIONS) {
219					continue;
220				}
221
222				for (id frame in parsedThreads[th_key][@"user_stack_frames"]) {
223					if ((frame_index >= FIRST_RECURSIVE_FRAME) && (frame_index < (RECURSIONS - FIRST_RECURSIVE_FRAME))) {
224						if (expected_return_addr == 0ull) {
225							expected_return_addr = [frame[@"lr"] unsignedLongLongValue];
226						} else {
227							T_QUIET;
228							T_ASSERT_EQ(expected_return_addr, [frame[@"lr"] unsignedLongLongValue], "expected return address found");
229						}
230					}
231					frame_index ++;
232				}
233			} else {
234				T_ASSERT_NOTNULL(parsedThreads[th_key][@"kernel_stack_frames"],
235						"found kernel stack frames");
236			}
237
238		}
239	}
240
241	if (found_fault_stats) {
242		T_LOG("number of pages faulted in: %d", fault_stats.sfs_pages_faulted_in);
243		T_LOG("MATUs spent faulting: %lld", fault_stats.sfs_time_spent_faulting);
244		T_LOG("MATUS fault time limit: %lld", fault_stats.sfs_system_max_fault_time);
245		T_LOG("did we stop because of the limit?: %s", fault_stats.sfs_stopped_faulting ? "yes" : "no");
246		if (expected_return_addr != 0ull) {
247			T_ASSERT_GT(fault_stats.sfs_pages_faulted_in, 0, "faulted at least one page in");
248			T_LOG("NOTE: successfully faulted in the pages");
249		} else {
250			T_LOG("NOTE: We were not able to fault the stack's pages back in");
251
252			/* if we couldn't fault the pages back in, then at least verify that we tried */
253			T_ASSERT_GT(fault_stats.sfs_time_spent_faulting, 0ull, "spent time trying to fault");
254		}
255	} else if ((flags & CHECK_FOR_KERNEL_THREADS) == 0) {
256		T_ASSERT_NE(expected_return_addr, 0ull, "found child thread with recursions");
257	}
258
259	if (flags & CHECK_FOR_FAULT_STATS) {
260		T_ASSERT_EQ(found_fault_stats, true, "found fault stats");
261	}
262
263	return ret;
264}
265
266void
267child_init(void)
268{
269#if !TARGET_OS_OSX
270	int freeze_state;
271#endif /* !TARGET_OS_OSX */
272	pid_t pid = getpid();
273	char padding[16 * 1024];
274	__asm__ volatile(""::"r"(padding));
275
276	T_LOG("child pid: %d\n", pid);
277
278#if !TARGET_OS_OSX
279	/* allow us to be frozen */
280	freeze_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, pid, 0, NULL, 0);
281	if (freeze_state == 0) {
282		T_LOG("CHILD was found to be UNFREEZABLE, enabling freezing.");
283		memorystatus_control(MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE, pid, 1, NULL, 0);
284		freeze_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, pid, 0, NULL, 0);
285		T_ASSERT_EQ(freeze_state, 1, "successfully set freezeability");
286	}
287#else
288	T_LOG("Cannot change freezeability as freezing is only available on embedded devices");
289#endif /* !TARGET_OS_OSX */
290
291	/*
292	 * recurse a bunch of times to generate predictable data in the stackshot,
293	 * then send SIGUSR1 to the parent to let it know that we are done.
294	 */
295	child_recurse(RECURSIONS, 0, ^{
296		kill(getppid(), SIGUSR1);
297	});
298
299	T_ASSERT_FAIL("child_recurse returned, but it must not?");
300}
301
302void
303parent_helper_singleproc(int spin)
304{
305	dispatch_semaphore_t child_done_sema = dispatch_semaphore_create(0);
306	dispatch_queue_t dq = dispatch_queue_create("com.apple.stackshot_accuracy.basic_sp", NULL);
307	void *stackshot_config;
308
309	dispatch_async(dq, ^{
310		char padding[16 * 1024];
311		__asm__ volatile(""::"r"(padding));
312
313		child_recurse(RECURSIONS, spin, ^{
314			dispatch_semaphore_signal(child_done_sema);
315		});
316	});
317
318	dispatch_semaphore_wait(child_done_sema, DISPATCH_TIME_FOREVER);
319	T_LOG("done waiting for child");
320
321	/* take the stackshot and parse it */
322	stackshot_config = take_stackshot(getpid(), 0, 0);
323
324	/* check that the stackshot has the stack frames */
325	check_stackshot(stackshot_config, 0);
326
327	T_LOG("done!");
328}
329
330T_DECL(basic, "test that no-fault stackshot works correctly")
331{
332	char path[PATH_MAX];
333	uint32_t path_size = sizeof(path);
334	char *args[] = { path, "-n", "simple_child_process", NULL };
335	dispatch_queue_t dq = dispatch_queue_create("com.apple.stackshot_accuracy.basic", NULL);
336	dispatch_semaphore_t child_done_sema = dispatch_semaphore_create(0);
337	dispatch_source_t child_sig_src;
338	void *stackshot_config;
339
340	current_scenario_name = __func__;
341
342	T_LOG("parent pid: %d\n", getpid());
343	T_QUIET; T_ASSERT_POSIX_ZERO(_NSGetExecutablePath(path, &path_size), "_NSGetExecutablePath");
344
345	/* check if we can run the child successfully */
346#if !TARGET_OS_OSX
347	int freeze_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, getpid(), 0, NULL, 0);
348	if (freeze_state == -1) {
349		T_SKIP("This device doesn't have CONFIG_FREEZE enabled.");
350	}
351#endif
352
353	/* setup signal handling */
354	signal(SIGUSR1, SIG_IGN);
355	child_sig_src = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dq);
356	dispatch_source_set_event_handler(child_sig_src, ^{
357		dispatch_semaphore_signal(child_done_sema);
358	});
359	dispatch_activate(child_sig_src);
360
361	/* create the child process */
362	T_ASSERT_POSIX_SUCCESS(dt_launch_tool(&child_pid, args, false, NULL, NULL), "child launched");
363	T_ATEND(kill_children);
364
365	/* wait until the child has recursed enough */
366	dispatch_semaphore_wait(child_done_sema, dispatch_time(DISPATCH_TIME_NOW, 10 /*seconds*/ * 1000000000ULL));
367
368	T_LOG("child finished, parent executing");
369
370	/* take the stackshot and parse it */
371	stackshot_config = take_stackshot(child_pid, 0, 0);
372
373	/* check that the stackshot has the stack frames */
374	check_stackshot(stackshot_config, 0);
375
376	T_LOG("all done, killing child");
377
378	/* tell the child to quit */
379	T_ASSERT_POSIX_SUCCESS(kill(child_pid, SIGTERM), "killed child");
380}
381
382T_DECL(basic_singleproc, "test that no-fault stackshot works correctly in single process setting")
383{
384	current_scenario_name = __func__;
385	parent_helper_singleproc(0);
386}
387
388T_DECL(basic_singleproc_spin, "test that no-fault stackshot works correctly in single process setting with spinning")
389{
390	current_scenario_name = __func__;
391	parent_helper_singleproc(1);
392}
393
394T_DECL(fault, "test that faulting stackshots work correctly")
395{
396	dispatch_queue_t dq = dispatch_queue_create("com.apple.stackshot_fault_accuracy", NULL);
397	dispatch_source_t child_sig_src;
398	dispatch_semaphore_t child_done_sema = dispatch_semaphore_create(0);
399	void *stackshot_config;
400	int oldftm, newval = 1, freeze_enabled, oldratio, newratio = 0;
401	size_t oldlen = sizeof(oldftm), fe_len = sizeof(freeze_enabled), ratiolen = sizeof(oldratio);
402	char path[PATH_MAX];
403	uint32_t path_size = sizeof(path);
404	char *args[] = { path, "-n", "simple_child_process", NULL };
405
406	current_scenario_name = __func__;
407	T_QUIET; T_ASSERT_POSIX_ZERO(_NSGetExecutablePath(path, &path_size), "_NSGetExecutablePath");
408
409#if TARGET_OS_OSX
410	T_SKIP("freezing is not available on macOS");
411#endif /* TARGET_OS_OSX */
412
413	/* Try checking if freezing is enabled at all */
414	if (sysctlbyname("vm.freeze_enabled", &freeze_enabled, &fe_len, NULL, 0) == -1) {
415		if (errno == ENOENT) {
416			T_SKIP("This device doesn't have CONFIG_FREEZE enabled.");
417		} else {
418			T_FAIL("failed to query vm.freeze_enabled, errno: %d", errno);
419		}
420	}
421
422	if (!freeze_enabled) {
423		T_SKIP("Freeze is not enabled, skipping test.");
424	}
425
426	/* signal handling */
427	signal(SIGUSR1, SIG_IGN);
428	child_sig_src = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dq);
429	dispatch_source_set_event_handler(child_sig_src, ^{
430		dispatch_semaphore_signal(child_done_sema);
431	});
432	dispatch_activate(child_sig_src);
433
434	T_ASSERT_POSIX_SUCCESS(dt_launch_tool(&child_pid, args, false, NULL, NULL), "child launched");
435	T_ATEND(kill_children);
436
437	dispatch_semaphore_wait(child_done_sema, DISPATCH_TIME_FOREVER);
438
439	/* keep processes in memory */
440	T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.memorystatus_freeze_to_memory", &oldftm, &oldlen, &newval, sizeof(newval)),
441			"disabled freezing to disk");
442
443	/* set the ratio to zero */
444	T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.memorystatus_freeze_private_shared_pages_ratio", &oldratio, &ratiolen, &newratio, sizeof(newratio)), "disabled private:shared ratio checking");
445
446	/* freeze the child */
447	T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.memorystatus_freeze", NULL, 0, &child_pid, sizeof(child_pid)),
448			"froze child");
449
450	/* Sleep to allow the compressor to finish compressing the child */
451	sleep(5);
452
453	/* take the stackshot and parse it */
454	stackshot_config = take_stackshot(child_pid, STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING, 0);
455
456	/* check that the stackshot has the stack frames */
457	check_stackshot(stackshot_config, CHECK_FOR_FAULT_STATS);
458
459	T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.memorystatus_freeze_to_memory", NULL, 0, &oldftm, sizeof(oldftm)),
460			"reset freezing to disk");
461
462	/* reset the private:shared ratio */
463	T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.memorystatus_freeze_private_shared_pages_ratio", NULL, 0, &oldratio, sizeof(oldratio)), "reset private:shared ratio");
464
465	T_LOG("all done, killing child");
466
467	/* tell the child to quit */
468	T_ASSERT_POSIX_SUCCESS(kill(child_pid, SIGTERM), "killed child");
469}
470
471T_DECL(fault_singleproc, "test that faulting stackshots work correctly in a single process setting")
472{
473	dispatch_semaphore_t child_done_sema = dispatch_semaphore_create(0);
474	dispatch_queue_t dq = dispatch_queue_create("com.apple.stackshot_accuracy.fault_sp", NULL);
475	void *stackshot_config;
476	__block pthread_t child_thread;
477	char *child_stack;
478	size_t child_stacklen;
479
480#if !TARGET_OS_OSX
481	T_SKIP("madvise(..., ..., MADV_PAGEOUT) is not available on embedded platforms");
482#endif /* !TARGET_OS_OSX */
483
484	dispatch_async(dq, ^{
485		char padding[16 * 1024];
486		__asm__ volatile(""::"r"(padding));
487
488		child_recurse(RECURSIONS, 0, ^{
489			child_thread = pthread_self();
490			dispatch_semaphore_signal(child_done_sema);
491		});
492	});
493
494	dispatch_semaphore_wait(child_done_sema, DISPATCH_TIME_FOREVER);
495	T_LOG("done waiting for child");
496
497	child_stack = pthread_get_stackaddr_np(child_thread);
498	child_stacklen = pthread_get_stacksize_np(child_thread);
499	child_stack -= child_stacklen;
500	T_LOG("child stack: [0x%p - 0x%p]: 0x%zu bytes", (void *)child_stack,
501			(void *)(child_stack + child_stacklen), child_stacklen);
502
503	/* paging out the child */
504	T_ASSERT_POSIX_SUCCESS(madvise(child_stack, child_stacklen, MADV_PAGEOUT), "paged out via madvise(2) the child stack");
505
506	/* take the stackshot and parse it */
507	stackshot_config = take_stackshot(getpid(), STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING, 0);
508
509	/* check that the stackshot has the stack frames */
510	check_stackshot(stackshot_config, CHECK_FOR_FAULT_STATS);
511
512	T_LOG("done!");
513}
514
515T_DECL(zombie, "test that threads wedged in the kernel can be stackshot'd")
516{
517	dispatch_queue_t dq = dispatch_queue_create("com.apple.stackshot_accuracy.zombie", NULL);
518	dispatch_semaphore_t child_done_sema = dispatch_semaphore_create(0);
519	dispatch_source_t child_sig_src;
520	void *stackshot_config;
521	char path[PATH_MAX];
522	uint32_t path_size = sizeof(path);
523	char *args[] = { path, "-n", "sid_child_process", NULL };
524
525	current_scenario_name = __func__;
526	T_QUIET; T_ASSERT_POSIX_ZERO(_NSGetExecutablePath(path, &path_size), "_NSGetExecutablePath");
527
528	T_LOG("parent pid: %d\n", getpid());
529
530	/* setup signal handling */
531	signal(SIGUSR1, SIG_IGN);
532	child_sig_src = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dq);
533	dispatch_source_set_event_handler(child_sig_src, ^{
534		dispatch_semaphore_signal(child_done_sema);
535	});
536	dispatch_activate(child_sig_src);
537
538	/* create the child process */
539	T_ASSERT_POSIX_SUCCESS(dt_launch_tool(&child_pid, args, false, NULL, NULL), "child launched");
540	T_ATEND(kill_children);
541
542	/* wait until the child has recursed enough */
543	dispatch_semaphore_wait(child_done_sema, DISPATCH_TIME_FOREVER);
544
545	T_LOG("child finished, parent executing. invoking jetsam");
546
547	T_ASSERT_POSIX_SUCCESS(memorystatus_control(MEMORYSTATUS_CMD_TEST_JETSAM, child_pid, 0, 0, 0),
548			"jetsam'd the child");
549
550	/* Sleep to allow the target process to become zombified */
551	sleep(1);
552
553	/* take the stackshot and parse it */
554	stackshot_config = take_stackshot(child_pid, 0, 0);
555
556	/* check that the stackshot has the stack frames */
557	check_stackshot(stackshot_config, CHECK_FOR_KERNEL_THREADS);
558
559	T_LOG("all done, unwedging and killing child");
560
561	int v = 1;
562	T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.unwedge_thread", NULL, NULL, &v, sizeof(v)),
563			"unwedged child");
564
565	/* tell the child to quit */
566	T_ASSERT_POSIX_SUCCESS(kill(child_pid, SIGTERM), "killed child");
567}
568