1 #include <stdio.h>
2 #include <signal.h>
3 #include <sys/sysctl.h>
4 #include <sys/kern_memorystatus.h>
5 #include <mach-o/dyld.h>
6 #include <perfcheck_keys.h>
7
8 #ifdef T_NAMESPACE
9 #undef T_NAMESPACE
10 #endif
11 #include <darwintest.h>
12 #include <darwintest_utils.h>
13
14 T_GLOBAL_META(
15 T_META_NAMESPACE("xnu.vm.perf"),
16 T_META_RADAR_COMPONENT_NAME("xnu"),
17 T_META_RADAR_COMPONENT_VERSION("VM"),
18 T_META_CHECK_LEAKS(false),
19 T_META_TAG_PERF
20 );
21
22 enum {
23 ALL_ZEROS,
24 MOSTLY_ZEROS,
25 RANDOM,
26 TYPICAL
27 };
28
29 #define CREATE_LIST(X) \
30 X(SUCCESS) \
31 X(TOO_FEW_ARGUMENTS) \
32 X(SYSCTL_VM_PAGESIZE_FAILED) \
33 X(VM_PAGESIZE_IS_ZERO) \
34 X(UNKNOWN_PAGE_TYPE) \
35 X(DISPATCH_SOURCE_CREATE_FAILED) \
36 X(INITIAL_SIGNAL_TO_PARENT_FAILED) \
37 X(SIGNAL_TO_PARENT_FAILED) \
38 X(MEMORYSTATUS_CONTROL_FAILED) \
39 X(IS_FREEZABLE_NOT_AS_EXPECTED) \
40 X(EXIT_CODE_MAX)
41
42 #define EXIT_CODES_ENUM(VAR) VAR,
43 enum exit_codes_num {
44 CREATE_LIST(EXIT_CODES_ENUM)
45 };
46
47 #define EXIT_CODES_STRING(VAR) #VAR,
48 static const char *exit_codes_str[] = {
49 CREATE_LIST(EXIT_CODES_STRING)
50 };
51
52 #define SYSCTL_FREEZE_TO_MEMORY "kern.memorystatus_freeze_to_memory=1"
53
54 static pid_t pid = -1;
55 static dt_stat_t ratio;
56 static dt_stat_time_t compr_time;
57 static dt_stat_time_t decompr_time;
58
59 void allocate_zero_pages(char **buf, int num_pages, int vmpgsize);
60 void allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize);
61 void allocate_random_pages(char **buf, int num_pages, int vmpgsize);
62 void allocate_representative_pages(char **buf, int num_pages, int vmpgsize);
63 void run_compressor_test(int size_mb, int page_type);
64 void freeze_helper_process(void);
65 void cleanup(void);
66
67 void
allocate_zero_pages(char ** buf,int num_pages,int vmpgsize)68 allocate_zero_pages(char **buf, int num_pages, int vmpgsize)
69 {
70 int i;
71
72 for (i = 0; i < num_pages; i++) {
73 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
74 memset(buf[i], 0, vmpgsize);
75 }
76 }
77
78 void
allocate_mostly_zero_pages(char ** buf,int num_pages,int vmpgsize)79 allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize)
80 {
81 int i, j;
82
83 for (i = 0; i < num_pages; i++) {
84 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
85 memset(buf[i], 0, vmpgsize);
86 for (j = 0; j < 40; j++) {
87 buf[i][j] = (char)(j + 1);
88 }
89 }
90 }
91
92 void
allocate_random_pages(char ** buf,int num_pages,int vmpgsize)93 allocate_random_pages(char **buf, int num_pages, int vmpgsize)
94 {
95 int i;
96
97 for (i = 0; i < num_pages; i++) {
98 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
99 arc4random_buf((void*)buf[i], (size_t)vmpgsize);
100 }
101 }
102
103 // Gives us the compression ratio we see in the typical case (~2.7)
104 void
allocate_representative_pages(char ** buf,int num_pages,int vmpgsize)105 allocate_representative_pages(char **buf, int num_pages, int vmpgsize)
106 {
107 int i, j;
108 char val;
109
110 for (j = 0; j < num_pages; j++) {
111 buf[j] = (char*)malloc((size_t)vmpgsize * sizeof(char));
112 val = 0;
113 for (i = 0; i < vmpgsize; i += 16) {
114 memset(&buf[j][i], val, 16);
115 if (i < 3400 * (vmpgsize / 4096)) {
116 val++;
117 }
118 }
119 }
120 }
121
122 void
freeze_helper_process(void)123 freeze_helper_process(void)
124 {
125 int ret, freeze_enabled;
126 int64_t compressed_before, compressed_after, input_before, input_after;
127 size_t length;
128 int errno_sysctl_freeze;
129
130 length = sizeof(compressed_before);
131 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_before, &length, NULL, 0),
132 "failed to query vm.compressor_compressed_bytes");
133 length = sizeof(input_before);
134 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_before, &length, NULL, 0),
135 "failed to query vm.compressor_input_bytes");
136
137 T_STAT_MEASURE(compr_time) {
138 ret = sysctlbyname("kern.memorystatus_freeze", NULL, NULL, &pid, sizeof(pid));
139 errno_sysctl_freeze = errno;
140 };
141
142 length = sizeof(compressed_after);
143 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_after, &length, NULL, 0),
144 "failed to query vm.compressor_compressed_bytes");
145 length = sizeof(input_after);
146 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_after, &length, NULL, 0),
147 "failed to query vm.compressor_input_bytes");
148
149 length = sizeof(freeze_enabled);
150 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
151 "failed to query vm.freeze_enabled");
152 if (freeze_enabled) {
153 errno = errno_sysctl_freeze;
154 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_freeze failed");
155 } else {
156 /* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
157 T_LOG("Freeze has been disabled. Terminating early.");
158 T_END;
159 }
160
161 dt_stat_add(ratio, (double)(input_after - input_before) / (double)(compressed_after - compressed_before));
162
163 ret = sysctlbyname("kern.memorystatus_thaw", NULL, NULL, &pid, sizeof(pid));
164 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_thaw failed");
165
166 T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGUSR1), "failed to send SIGUSR1 to child process");
167 }
168
169 void
cleanup(void)170 cleanup(void)
171 {
172 /* No helper process. */
173 if (pid == -1) {
174 return;
175 }
176 /* Kill the helper process. */
177 kill(pid, SIGKILL);
178 }
179
180 void
run_compressor_test(int size_mb,int page_type)181 run_compressor_test(int size_mb, int page_type)
182 {
183 int ret;
184 char sz_str[50];
185 char pt_str[50];
186 char **launch_tool_args;
187 char testpath[PATH_MAX];
188 uint32_t testpath_buf_size;
189 dispatch_source_t ds_freeze, ds_proc, ds_decompr;
190 int freeze_enabled;
191 size_t length;
192 __block bool decompr_latency_is_stable = false;
193
194 length = sizeof(freeze_enabled);
195 T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
196 "failed to query vm.freeze_enabled");
197 if (!freeze_enabled) {
198 /* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
199 T_SKIP("Freeze has been disabled. Skipping test.");
200 }
201
202 T_ATEND(cleanup);
203
204 ratio = dt_stat_create("(input bytes / compressed bytes)", "compression_ratio");
205 compr_time = dt_stat_time_create("compressor_latency");
206
207 // This sets the A/B failure threshold at 50% of baseline for compressor_latency
208 dt_stat_set_variable((struct dt_stat *)compr_time, kPCFailureThresholdPctVar, 50.0);
209
210 signal(SIGUSR2, SIG_IGN);
211 ds_decompr = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR2, 0, dispatch_get_main_queue());
212 T_QUIET; T_ASSERT_NOTNULL(ds_decompr, "dispatch_source_create (ds_decompr)");
213
214 dispatch_source_set_event_handler(ds_decompr, ^{
215 decompr_latency_is_stable = true;
216 });
217 dispatch_activate(ds_decompr);
218
219 signal(SIGUSR1, SIG_IGN);
220 ds_freeze = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
221 T_QUIET; T_ASSERT_NOTNULL(ds_freeze, "dispatch_source_create (ds_freeze)");
222
223 dispatch_source_set_event_handler(ds_freeze, ^{
224 if (!(dt_stat_stable(compr_time) && decompr_latency_is_stable)) {
225 freeze_helper_process();
226 } else {
227 dt_stat_finalize(compr_time);
228 dt_stat_finalize(ratio);
229
230 kill(pid, SIGKILL);
231 dispatch_source_cancel(ds_freeze);
232 dispatch_source_cancel(ds_decompr);
233 }
234 });
235 dispatch_activate(ds_freeze);
236
237 testpath_buf_size = sizeof(testpath);
238 ret = _NSGetExecutablePath(testpath, &testpath_buf_size);
239 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "_NSGetExecutablePath");
240 T_LOG("Executable path: %s", testpath);
241
242 sprintf(sz_str, "%d", size_mb);
243 sprintf(pt_str, "%d", page_type);
244 launch_tool_args = (char *[]){
245 testpath,
246 "-n",
247 "allocate_pages",
248 "--",
249 sz_str,
250 pt_str,
251 NULL
252 };
253
254 /* Spawn the child process. Suspend after launch until the exit proc handler has been set up. */
255 ret = dt_launch_tool(&pid, launch_tool_args, true, NULL, NULL);
256 if (ret != 0) {
257 T_LOG("dt_launch tool returned %d with error code %d", ret, errno);
258 }
259 T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "dt_launch_tool");
260
261 ds_proc = dispatch_source_create(DISPATCH_SOURCE_TYPE_PROC, (uintptr_t)pid, DISPATCH_PROC_EXIT, dispatch_get_main_queue());
262 T_QUIET; T_ASSERT_NOTNULL(ds_proc, "dispatch_source_create (ds_proc)");
263
264 dispatch_source_set_event_handler(ds_proc, ^{
265 int status = 0, code = 0;
266 pid_t rc = waitpid(pid, &status, 0);
267 T_QUIET; T_ASSERT_EQ(rc, pid, "waitpid");
268 code = WEXITSTATUS(status);
269
270 if (code == 0) {
271 T_END;
272 } else if (code > 0 && code < EXIT_CODE_MAX) {
273 T_ASSERT_FAIL("Child exited with %s", exit_codes_str[code]);
274 } else {
275 T_ASSERT_FAIL("Child exited with unknown exit code %d", code);
276 }
277 });
278 dispatch_activate(ds_proc);
279
280 T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGCONT), "failed to send SIGCONT to child process");
281 dispatch_main();
282 }
283
284 T_HELPER_DECL(allocate_pages, "allocates pages to compress") {
285 int i, j, ret, size_mb, page_type, vmpgsize, freezable_state;
286 size_t vmpgsize_length;
287 __block int num_pages;
288 __block char **buf;
289 dispatch_source_t ds_signal;
290
291 vmpgsize_length = sizeof(vmpgsize);
292 ret = sysctlbyname("vm.pagesize", &vmpgsize, &vmpgsize_length, NULL, 0);
293 if (ret != 0) {
294 exit(SYSCTL_VM_PAGESIZE_FAILED);
295 }
296 if (vmpgsize == 0) {
297 exit(VM_PAGESIZE_IS_ZERO);
298 }
299
300 if (argc < 2) {
301 exit(TOO_FEW_ARGUMENTS);
302 }
303
304 size_mb = atoi(argv[0]);
305 page_type = atoi(argv[1]);
306 num_pages = size_mb * 1024 * 1024 / vmpgsize;
307 buf = (char**)malloc(sizeof(char*) * (size_t)num_pages);
308
309 // Switch on the type of page requested
310 switch (page_type) {
311 case ALL_ZEROS:
312 allocate_zero_pages(buf, num_pages, vmpgsize);
313 break;
314 case MOSTLY_ZEROS:
315 allocate_mostly_zero_pages(buf, num_pages, vmpgsize);
316 break;
317 case RANDOM:
318 allocate_random_pages(buf, num_pages, vmpgsize);
319 break;
320 case TYPICAL:
321 allocate_representative_pages(buf, num_pages, vmpgsize);
322 break;
323 default:
324 exit(UNKNOWN_PAGE_TYPE);
325 }
326
327 for (j = 0; j < num_pages; j++) {
328 i = buf[j][0];
329 }
330
331 decompr_time = dt_stat_time_create("decompression_latency");
332
333 /* Opt in to freezing. */
334 printf("[%d] Setting state to freezable\n", getpid());
335 if (memorystatus_control(MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE, getpid(), 1, NULL, 0) != KERN_SUCCESS) {
336 exit(MEMORYSTATUS_CONTROL_FAILED);
337 }
338
339 /* Verify that the state has been set correctly */
340 freezable_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, getpid(), 0, NULL, 0);
341 if (freezable_state != 1) {
342 exit(IS_FREEZABLE_NOT_AS_EXPECTED);
343 }
344
345 dispatch_after(dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC), dispatch_get_main_queue(), ^{
346 /* Signal to the parent that we're done allocating and it's ok to freeze us */
347 printf("[%d] Sending initial signal to parent to begin freezing\n", getpid());
348 if (kill(getppid(), SIGUSR1) != 0) {
349 exit(INITIAL_SIGNAL_TO_PARENT_FAILED);
350 }
351 });
352
353 signal(SIGUSR1, SIG_IGN);
354 ds_signal = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
355 if (ds_signal == NULL) {
356 exit(DISPATCH_SOURCE_CREATE_FAILED);
357 }
358
359 __block bool collect_dt_stat_measurements = true;
360
361 dispatch_source_set_event_handler(ds_signal, ^{
362 volatile int tmp;
363 uint64_t decompr_start_time, decompr_end_time;
364
365 decompr_start_time = mach_absolute_time();
366
367 /* Make sure all the pages are accessed before trying to freeze again */
368 for (int x = 0; x < num_pages; x++) {
369 tmp = buf[x][0];
370 }
371
372 decompr_end_time = mach_absolute_time();
373
374 if (collect_dt_stat_measurements) {
375 if (dt_stat_stable(decompr_time)) {
376 collect_dt_stat_measurements = false;
377 dt_stat_finalize(decompr_time);
378 if (kill(getppid(), SIGUSR2) != 0) {
379 exit(SIGNAL_TO_PARENT_FAILED);
380 }
381 } else {
382 dt_stat_mach_time_add(decompr_time, decompr_end_time - decompr_start_time);
383 }
384 }
385
386 if (kill(getppid(), SIGUSR1) != 0) {
387 exit(SIGNAL_TO_PARENT_FAILED);
388 }
389 });
390 dispatch_activate(ds_signal);
391
392 dispatch_main();
393 }
394
395 // Numbers for 10MB and above are fairly reproducible. Anything smaller shows a lot of variation.
396
397 // Keeping just the 100MB version for iOSMark
398 #ifndef DT_IOSMARK
399 T_DECL(compr_10MB_zero,
400 "Compression latency for 10MB - zero pages",
401 T_META_ASROOT(true),
402 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
403 run_compressor_test(10, ALL_ZEROS);
404 }
405
406 T_DECL(compr_10MB_mostly_zero,
407 "Compression latency for 10MB - mostly zero pages",
408 T_META_ASROOT(true),
409 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
410 run_compressor_test(10, MOSTLY_ZEROS);
411 }
412
413 T_DECL(compr_10MB_random,
414 "Compression latency for 10MB - random pages",
415 T_META_ASROOT(true),
416 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
417 run_compressor_test(10, RANDOM);
418 }
419
420 T_DECL(compr_10MB_typical,
421 "Compression latency for 10MB - typical pages",
422 T_META_ASROOT(true),
423 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
424 run_compressor_test(10, TYPICAL);
425 }
426
427 T_DECL(compr_100MB_zero,
428 "Compression latency for 100MB - zero pages",
429 T_META_ASROOT(true),
430 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
431 run_compressor_test(100, ALL_ZEROS);
432 }
433
434 T_DECL(compr_100MB_mostly_zero,
435 "Compression latency for 100MB - mostly zero pages",
436 T_META_ASROOT(true),
437 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
438 run_compressor_test(100, MOSTLY_ZEROS);
439 }
440
441 T_DECL(compr_100MB_random,
442 "Compression latency for 100MB - random pages",
443 T_META_ASROOT(true),
444 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
445 run_compressor_test(100, RANDOM);
446 }
447 #endif
448
449 T_DECL(compr_100MB_typical,
450 "Compression latency for 100MB - typical pages",
451 T_META_ASROOT(true),
452 T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
453 run_compressor_test(100, TYPICAL);
454 }
455