1 /**
2 * neural_footprint.c
3 * Neural composite footprint ledger test
4 *
5 * Test various memory settings to ensure correct accounting
6 * Copyright (c) 2023 Apple Inc. All rights reserved.
7 */
8
9 #include <sys/mman.h>
10 #include <mach/vm_map.h>
11 #include <mach/mach_vm.h>
12 #include <mach/mach_port.h>
13 #include <mach/mach_init.h>
14 #include <mach/mach_error.h>
15 #include <darwintest_utils.h>
16 #include <libproc_internal.h>
17 #include <mach/memory_entry.h>
18 #include <Kernel/kern/ledger.h>
19
20
21 extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);
22
23 #define ALLOCATION_SIZE (10 * vm_kernel_page_size) /* 10 pages */
24
25 T_GLOBAL_META(
26 T_META_RUN_CONCURRENTLY(true),
27 T_META_RADAR_COMPONENT_NAME("xnu"),
28 T_META_RADAR_COMPONENT_VERSION("VM"));
29
30 static int neural_nofootprint_index = -1;
31 static int neural_nofootprint_compressed_index = -1;
32 static int neural_total_index = -1;
33 static int neural_footprint_index = -1;
34 static int neural_footprint_compressed_index = -1;
35
36 static uint64_t neural_nofootprint_before;
37 static uint64_t neural_nofootprint_compressed_before;
38 static uint64_t neural_footprint_before;
39 static uint64_t neural_footprint_compressed_before;
40 static uint64_t neural_total_before;
41 static uint64_t neural_nofootprint_after;
42 static uint64_t neural_nofootprint_compressed_after;
43 static uint64_t neural_footprint_after;
44 static uint64_t neural_footprint_compressed_after;
45 static uint64_t neural_total_after;
46
47 static int64_t ledger_count = -1;
48 static struct ledger_entry_info *lei = NULL;
49
50 static uint64_t phys_max_before, phys_max_after;
51 static struct rusage_info_v6 ru;
52 static uint64_t neural_lifetime_max;
53
54
55
56 static void
ledger_init(void)57 ledger_init(void)
58 {
59 kern_return_t kr;
60 static int ledger_inited = 0;
61 struct ledger_template_info *templateInfo;
62 int64_t templateCnt;
63 struct ledger_info li;
64 if (ledger_inited) {
65 return;
66 }
67 ledger_inited = 1;
68
69 T_SETUPBEGIN;
70
71 kr = ledger(LEDGER_INFO,
72 (caddr_t)(uintptr_t)getpid(),
73 (caddr_t)&li,
74 NULL);
75
76 T_ASSERT_MACH_SUCCESS(kr, "ledger() 0x%x (%s)",
77 kr, mach_error_string(kr));
78
79 templateCnt = li.li_entries;
80 templateInfo = malloc((size_t)li.li_entries *
81 sizeof(struct ledger_template_info));
82 T_QUIET;
83 T_WITH_ERRNO;
84 T_ASSERT_NOTNULL(templateInfo, "malloc()");
85
86 ledger_count = li.li_entries;
87
88 T_QUIET;
89 T_WITH_ERRNO;
90 T_ASSERT_POSIX_SUCCESS(ledger(LEDGER_TEMPLATE_INFO,
91 (caddr_t)templateInfo,
92 (caddr_t)&templateCnt,
93 NULL),
94 "ledger(LEDGER_TEMPLATE_INFO)");
95 for (int i = 0; i < templateCnt; i++) {
96 if (!strncmp(templateInfo[i].lti_name,
97 "neural_nofootprint_compressed",
98 strlen("neural_nofootprint_compressed"))) {
99 neural_nofootprint_compressed_index = i;
100 T_LOG("Acquired index of neural_nofootprint_compressed");
101 } else if (!strncmp(templateInfo[i].lti_name,
102 "neural_footprint_compressed",
103 strlen("neural_footprint_compressed"))) {
104 neural_footprint_compressed_index = i;
105 T_LOG("Acquired index of neural_footprint_compressed");
106 } else if (!strncmp(templateInfo[i].lti_name,
107 "neural_nofootprint_total",
108 strlen("neural_nofootprint_total"))) {
109 neural_total_index = i;
110 T_LOG("Acquired index of neural_nofootprint_total");
111 } else if (!strncmp(templateInfo[i].lti_name,
112 "neural_nofootprint",
113 strlen("neural_nofootprint"))) {
114 neural_nofootprint_index = i;
115 T_LOG("Acquired index of neural_nofootprint");
116 } else if (!strncmp(templateInfo[i].lti_name,
117 "neural_footprint",
118 strlen("neural_footprint"))) {
119 neural_footprint_index = i;
120 T_LOG("Acquired index of neural_footprint");
121 }
122 }
123 free(templateInfo);
124
125 lei = (struct ledger_entry_info *)
126 malloc((size_t)ledger_count * sizeof(*lei));
127 T_QUIET;
128 T_WITH_ERRNO;
129 T_ASSERT_NE(lei, NULL,
130 "malloc(ledger_entry_info)");
131
132 T_QUIET;
133 T_ASSERT_NE(neural_nofootprint_compressed_index, -1,
134 "no nofootprint_compressed_index");
135 T_QUIET;
136 T_ASSERT_NE(neural_footprint_compressed_index, -1,
137 "no footprint_compressed_index");
138 T_QUIET;
139 T_ASSERT_NE(neural_total_index, -1,
140 "no nofootprint_total_index");
141 T_QUIET;
142 T_ASSERT_NE(neural_nofootprint_index, -1,
143 "no nofootprint_index");
144 T_QUIET;
145 T_ASSERT_NE(neural_footprint_index, -1,
146 "no footprint_index");
147
148 T_SETUPEND;
149 }
150
151 static void
get_ledger_info(uint64_t * neural_nofootprint,uint64_t * neural_nofootprint_compressed,uint64_t * neural_footprint,uint64_t * neural_footprint_compressed,uint64_t * neural_total)152 get_ledger_info(
153 uint64_t *neural_nofootprint,
154 uint64_t *neural_nofootprint_compressed,
155 uint64_t *neural_footprint,
156 uint64_t *neural_footprint_compressed,
157 uint64_t *neural_total)
158 {
159 int64_t count;
160
161 count = ledger_count;
162 T_QUIET;
163 T_WITH_ERRNO;
164 T_ASSERT_POSIX_SUCCESS(ledger(LEDGER_ENTRY_INFO,
165 (caddr_t)(uintptr_t)getpid(),
166 (caddr_t)lei,
167 (caddr_t)&count),
168 "ledger(LEDGER_ENTRY_INFO)");
169 T_QUIET;
170 T_ASSERT_GT(count, (int64_t)neural_nofootprint_index,
171 "no entry for neural_nofootprint");
172 T_QUIET;
173 T_ASSERT_GT(count, (int64_t)neural_nofootprint_compressed_index,
174 "no entry for neural_nofootprint_compressed");
175 T_QUIET;
176 T_ASSERT_GT(count, (int64_t)neural_total_index,
177 "no entry for neural_total");
178 T_QUIET;
179 T_ASSERT_GT(count, (int64_t)neural_footprint_compressed_index,
180 "no entry for neural_footprint_compressed");
181 T_QUIET;
182 T_ASSERT_GT(count, (int64_t)neural_footprint_index,
183 "no entry for neural_footprint");
184 if (neural_footprint_index) {
185 *neural_footprint = (uint64_t)(lei[neural_footprint_index].lei_balance);
186 }
187 if (neural_footprint_compressed_index) {
188 *neural_footprint_compressed = (uint64_t)(
189 lei[neural_footprint_compressed_index].lei_balance);
190 }
191 if (neural_nofootprint_index) {
192 *neural_nofootprint = (uint64_t)(lei[neural_nofootprint_index].lei_balance);
193 }
194 if (neural_nofootprint_compressed_index) {
195 *neural_nofootprint_compressed = (uint64_t)(
196 lei[neural_nofootprint_compressed_index].lei_balance);
197 }
198 if (neural_total_index) {
199 *neural_total = (uint64_t)(lei[neural_total_index].lei_balance);
200 }
201 }
202
203 static void
get_ledger_before(void)204 get_ledger_before(void)
205 {
206 get_ledger_info(
207 &neural_nofootprint_before,
208 &neural_nofootprint_compressed_before,
209 &neural_footprint_before,
210 &neural_footprint_compressed_before,
211 &neural_total_before);
212 T_LOG(
213 "*** pages before: footprint:%llu compr:%llu nofootprint:%llu nofootprint compr: %llu, total: %llu",
214 neural_footprint_before / vm_kernel_page_size,
215 neural_footprint_compressed_before / vm_kernel_page_size,
216 neural_nofootprint_before / vm_kernel_page_size,
217 neural_nofootprint_compressed_before / vm_kernel_page_size,
218 neural_total_before / vm_kernel_page_size);
219 }
220
221 static void
get_ledger_after(void)222 get_ledger_after(void)
223 {
224 get_ledger_info(
225 &neural_nofootprint_after,
226 &neural_nofootprint_compressed_after,
227 &neural_footprint_after,
228 &neural_footprint_compressed_after,
229 &neural_total_after);
230
231 T_LOG(
232 "*** pages after: footprint:%llu compr:%llu nofootprint:%llu nofootprint compr: %llu, total: %llu",
233 neural_footprint_after / vm_kernel_page_size,
234 neural_footprint_compressed_after / vm_kernel_page_size,
235 neural_nofootprint_after / vm_kernel_page_size,
236 neural_nofootprint_compressed_after / vm_kernel_page_size,
237 neural_total_after / vm_kernel_page_size);
238 }
239
240 static void
compress_pages(mach_vm_address_t vm_addr,mach_vm_size_t vm_size)241 compress_pages(mach_vm_address_t vm_addr, mach_vm_size_t vm_size)
242 {
243 int ret;
244 T_LOG(">>>> Compress all pages");
245 unsigned char *cp;
246 cp = (unsigned char *)(uintptr_t)vm_addr;
247 T_ASSERT_POSIX_SUCCESS(
248 madvise(cp,
249 (size_t)vm_size,
250 MADV_PAGEOUT),
251 "page out all with madvise");
252
253 T_LOG("...> Wait for pages to be compressed");
254 unsigned char vec;
255
256 for (size_t idx = 0; idx < vm_size; idx += vm_kernel_page_size) {
257 do {
258 ret = mincore(&cp[idx], 1, (char *)&vec);
259 if (ret != 0) {
260 T_ASSERT_POSIX_SUCCESS(ret, "failed on mincore check");
261 }
262 } while (vec & MINCORE_INCORE);
263 }
264 }
265
266 static void
uncompress_pages(mach_vm_address_t vm_addr,mach_vm_size_t vm_size)267 uncompress_pages(mach_vm_address_t vm_addr, mach_vm_size_t vm_size)
268 {
269 T_LOG("<<<< Reading pages to bring back from compressor");
270 unsigned char *cp;
271 cp = (unsigned char *)(uintptr_t)vm_addr;
272 memset(cp, 0xff, vm_size);
273 }
274
275 static void
check_phys_footprint_rusage(void)276 check_phys_footprint_rusage(void)
277 {
278 int ret;
279 T_LOG("---? Check phys footprint lifetime max and max interval");
280
281 ret = proc_pid_rusage(getpid(), RUSAGE_INFO_V6, (rusage_info_t *)&ru);
282 T_QUIET;
283 T_ASSERT_POSIX_SUCCESS(ret, "proc_pid_rusage");
284 phys_max_before = ru.ri_lifetime_max_phys_footprint;
285 T_LOG("Phys max: %llu", ru.ri_lifetime_max_phys_footprint);
286 }
287
288 static void
check_phys_footprint_rusage_after(void)289 check_phys_footprint_rusage_after(void)
290 {
291 int ret;
292
293 T_LOG("---? Check phys footprint lifetime max and max interval");
294
295 ret = proc_pid_rusage(getpid(), RUSAGE_INFO_V6, (rusage_info_t *)&ru);
296 T_QUIET;
297 T_ASSERT_POSIX_SUCCESS(ret, "proc_pid_rusage");
298 phys_max_after = ru.ri_lifetime_max_phys_footprint;;
299 T_LOG("Phys_footprint max: %llu before: %llu diff: %llu",
300 phys_max_after,
301 phys_max_before,
302 phys_max_after - phys_max_before);
303
304 T_ASSERT_EQ(0ULL, phys_max_after - phys_max_before,
305 "Phys footprint lifetime max shouldn't change");
306 }
307
308 static void
make_volatile(mach_vm_address_t vm_addr)309 make_volatile(mach_vm_address_t vm_addr)
310 {
311 kern_return_t kr;
312 vm_purgable_t state;
313 char *vm_purgable_state[4] = {
314 "nonvolatile",
315 "volatile",
316 "empty",
317 "deny"
318 };
319 state = VM_PURGABLE_VOLATILE;
320 kr = mach_vm_purgable_control(
321 mach_task_self(),
322 vm_addr,
323 VM_PURGABLE_SET_STATE,
324 &state);
325 T_ASSERT_MACH_SUCCESS(kr, "vm_purgable_control(volatile)");
326 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
327 "nonvolatile -> volatile: state was %s",
328 vm_purgable_state[state]);
329 }
330
331 static void
make_nonvolatile(mach_vm_address_t vm_addr)332 make_nonvolatile(mach_vm_address_t vm_addr)
333 {
334 kern_return_t kr;
335 vm_purgable_t state;
336 char *vm_purgable_state[4] = {
337 "nonvolatile",
338 "volatile",
339 "empty",
340 "deny"
341 };
342 state = VM_PURGABLE_NONVOLATILE;
343 kr = mach_vm_purgable_control(
344 mach_task_self(),
345 vm_addr,
346 VM_PURGABLE_SET_STATE,
347 &state);
348 T_ASSERT_MACH_SUCCESS(kr, "vm_purgable_control(nonvolatile)");
349 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
350 "volatile -> nonvolatile: state was %s",
351 vm_purgable_state[state]);
352 }
353
354 static void
reset_max_interval(uint64_t check)355 reset_max_interval(uint64_t check)
356 {
357 int ret;
358 T_LOG("---> Reset interval max");
359 ret = proc_reset_footprint_interval(getpid());
360 T_ASSERT_POSIX_SUCCESS(ret, "proc_reset_footprint_interval()");
361
362 ret = proc_pid_rusage(getpid(), RUSAGE_INFO_V6, (rusage_info_t *)&ru);
363 T_QUIET;
364 T_ASSERT_POSIX_SUCCESS(ret, "proc_pid_rusage");
365 T_EXPECT_EQ(check, ru.ri_interval_max_neural_footprint,
366 "Neural max interval footprint is %llu pages", check / vm_kernel_page_size);
367 }
368
369 T_DECL(check_neural_total_ledger,
370 "Check neural totall ledger",
371 T_META_LTEPHASE(LTE_POSTINIT))
372 {
373 int ret;
374 kern_return_t kr;
375 mach_vm_size_t vm_size, me_size;
376 mach_port_t footprint_port, nofootprint_port;
377 mach_port_t footprint_port2, nofootprint_port2;
378 mach_port_t file_mem_entry;
379 vm_prot_t permissions;
380
381 mach_vm_address_t footprint_vm_addr = 0, nofootprint_vm_addr = 0;
382 mach_vm_address_t footprint_vm_addr2 = 0, nofootprint_vm_addr2 = 0;
383 mach_vm_size_t dirty_size = ALLOCATION_SIZE;
384
385 ledger_init();
386
387 get_ledger_before();
388
389 vm_size = ALLOCATION_SIZE;
390 me_size = vm_size;
391 footprint_port = MACH_PORT_NULL;
392 nofootprint_port = MACH_PORT_NULL;
393
394 permissions = MAP_MEM_NAMED_CREATE |
395 MAP_MEM_PURGABLE |
396 MAP_MEM_LEDGER_TAGGED |
397 VM_PROT_DEFAULT;
398
399 T_LOG("---> Allocate for footprint");
400 kr = mach_make_memory_entry_64(
401 mach_task_self(),
402 &me_size,
403 0,
404 permissions,
405 &footprint_port,
406 MACH_PORT_NULL);
407 T_ASSERT_MACH_SUCCESS(kr, "make_memory_entry()");
408 T_ASSERT_EQ(me_size, vm_size, "checking memory entry size mismatch");
409
410 T_LOG("---> Allocate for nofootprint");
411 kr = mach_make_memory_entry_64(
412 mach_task_self(),
413 &me_size,
414 0,
415 permissions,
416 &nofootprint_port,
417 MACH_PORT_NULL);
418 T_ASSERT_MACH_SUCCESS(kr, "make_memory_entry()");
419 T_ASSERT_EQ(me_size, vm_size, "checking memory entry size mismatch");
420
421 T_LOG("---> Allocate for secondary footprint");
422 kr = mach_make_memory_entry_64(
423 mach_task_self(),
424 &me_size,
425 0,
426 permissions,
427 &footprint_port2,
428 MACH_PORT_NULL);
429 T_ASSERT_MACH_SUCCESS(kr, "make_memory_entry()");
430 T_ASSERT_EQ(me_size, vm_size, "checking memory entry size mismatch");
431
432 T_LOG("---> Allocate for secondary nofootprint");
433 kr = mach_make_memory_entry_64(
434 mach_task_self(),
435 &me_size,
436 0,
437 permissions,
438 &nofootprint_port2,
439 MACH_PORT_NULL);
440 T_ASSERT_MACH_SUCCESS(kr, "make_memory_entry()");
441 T_ASSERT_EQ(me_size, vm_size, "checking memory entry size mismatch");
442
443 T_LOG("---> Allocate for file footprint");
444 char const *tmp_dir = dt_tmpdir();
445 char filepath[MAXPATHLEN];
446 int fd = -1;
447 char *file_addr = NULL;
448 snprintf(filepath, sizeof(filepath), "%s/file.XXXXXX", tmp_dir);
449 T_ASSERT_POSIX_SUCCESS(fd = mkstemp(filepath), NULL);
450 T_ASSERT_POSIX_SUCCESS(unlink(filepath), NULL);
451 T_ASSERT_POSIX_SUCCESS(ftruncate(fd, (off_t)vm_size), NULL);
452 T_ASSERT_POSIX_SUCCESS(file_addr = mmap(NULL, (size_t)vm_size, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0), NULL);
453 me_size = vm_size;
454 T_ASSERT_MACH_SUCCESS(mach_make_memory_entry_64(
455 mach_task_self(),
456 &me_size,
457 (memory_object_offset_t)(uintptr_t)file_addr,
458 /* MAP_MEM_LEDGER_TAGGED | */ VM_PROT_READ,
459 &file_mem_entry,
460 MACH_PORT_NULL), NULL);
461 T_ASSERT_EQ(me_size, vm_size, "checking memory entry size mismatch");
462 T_ASSERT_POSIX_SUCCESS(mlock(file_addr, (size_t)vm_size), NULL);
463
464
465 get_ledger_after();
466 T_LOG("Change is not expected on neural ledgers");
467 T_ASSERT_EQ(neural_nofootprint_before, neural_nofootprint_after,
468 "neural entry size changed %llu -> %llu",
469 neural_nofootprint_before, neural_nofootprint_after);
470 T_ASSERT_EQ(neural_nofootprint_compressed_before, neural_nofootprint_compressed_after,
471 "neural entry size changed %llu -> %llu",
472 neural_nofootprint_compressed_before, neural_nofootprint_compressed_after);
473 T_ASSERT_EQ(neural_footprint_before, neural_footprint_after,
474 "neural entry size changed %llu -> %llu",
475 neural_footprint_before, neural_footprint_after);
476 T_ASSERT_EQ(neural_footprint_compressed_before, neural_footprint_compressed_after,
477 "neural entry size changed %llu -> %llu",
478 neural_footprint_compressed_before, neural_footprint_compressed_after);
479 T_ASSERT_EQ(neural_total_before, neural_total_after,
480 "neural entry size changed %llu -> %llu",
481 neural_total_before, neural_total_after);
482
483
484 T_ASSERT_MACH_SUCCESS(
485 mach_vm_map(
486 mach_task_self(),
487 &footprint_vm_addr,
488 dirty_size,
489 0, /* mask */
490 VM_FLAGS_ANYWHERE,
491 footprint_port,
492 0, /* offset */
493 false, /* copy */
494 VM_PROT_DEFAULT,
495 VM_PROT_DEFAULT,
496 VM_INHERIT_NONE),
497 "mach_vm_map() for primary neural footprint"
498 );
499
500 T_ASSERT_MACH_SUCCESS(
501 mach_vm_map(
502 mach_task_self(),
503 &nofootprint_vm_addr,
504 dirty_size,
505 0, /* mask */
506 VM_FLAGS_ANYWHERE,
507 nofootprint_port,
508 0, /* offset */
509 false, /* copy */
510 VM_PROT_DEFAULT,
511 VM_PROT_DEFAULT,
512 VM_INHERIT_NONE),
513 "mach_vm_map() for primary neural nofootprint"
514 );
515
516 T_ASSERT_MACH_SUCCESS(
517 mach_vm_map(
518 mach_task_self(),
519 &footprint_vm_addr2,
520 dirty_size,
521 0, /* mask */
522 VM_FLAGS_ANYWHERE,
523 footprint_port2,
524 0, /* offset */
525 false, /* copy */
526 VM_PROT_DEFAULT,
527 VM_PROT_DEFAULT,
528 VM_INHERIT_NONE),
529 "mach_vm_map() for secondary neural footprint"
530 );
531
532 T_ASSERT_MACH_SUCCESS(
533 mach_vm_map(
534 mach_task_self(),
535 &nofootprint_vm_addr2,
536 dirty_size,
537 0, /* mask */
538 VM_FLAGS_ANYWHERE,
539 nofootprint_port2,
540 0, /* offset */
541 false, /* copy */
542 VM_PROT_DEFAULT,
543 VM_PROT_DEFAULT,
544 VM_INHERIT_NONE),
545 "mach_vm_map() for secondary neural nofootprint"
546 );
547
548 T_LOG("Dirtying pages");
549 memset((char *)(uintptr_t)footprint_vm_addr, 0xaa, (size_t)dirty_size);
550 memset((char *)(uintptr_t)footprint_vm_addr2, 0xbb, (size_t)dirty_size);
551 memset((char *)(uintptr_t)nofootprint_vm_addr, 0xcc, (size_t)dirty_size);
552 memset((char *)(uintptr_t)nofootprint_vm_addr2, 0xdd, (size_t)dirty_size);
553
554 T_LOG("Checking if compression works correctly with phys_footprint");
555 check_phys_footprint_rusage();
556
557 compress_pages(footprint_vm_addr, vm_size);
558
559 check_phys_footprint_rusage_after();
560
561 uncompress_pages(footprint_vm_addr, vm_size);
562
563 get_ledger_before();
564
565 T_LOG("---> Move primary footprint to neural");
566 kr = mach_memory_entry_ownership(
567 footprint_port, /* entry port */
568 TASK_NULL, /* owner remains unchanged */
569 VM_LEDGER_TAG_NEURAL, /* ledger-tag */
570 0); /* ledger flags */
571 T_ASSERT_MACH_SUCCESS(kr,
572 "mach_memory_entry_ownership() primary neural footprint 0x%x (%s)",
573 kr, mach_error_string(kr));
574
575 get_ledger_after();
576
577 T_ASSERT_EQ(
578 neural_footprint_before + vm_size,
579 neural_footprint_after,
580 "neural footprint increased %llu -> %llu pages",
581 neural_footprint_before / vm_kernel_page_size,
582 neural_footprint_after / vm_kernel_page_size);
583 T_ASSERT_EQ(
584 0ULL,
585 neural_total_after,
586 "neural total is zero %llu -> %llu pages",
587 neural_total_before / vm_kernel_page_size,
588 neural_total_after / vm_kernel_page_size);
589
590 T_LOG("---> Move secondary footprint to neural");
591 get_ledger_before();
592 kr = mach_memory_entry_ownership(
593 footprint_port2, /* entry port */
594 TASK_NULL, /* owner remains unchanged */
595 VM_LEDGER_TAG_NEURAL, /* ledger-tag */
596 0); /* ledger flags */
597 T_ASSERT_MACH_SUCCESS(kr,
598 "mach_memory_entry_ownership() primary neural footprint 0x%x (%s)",
599 kr, mach_error_string(kr));
600 get_ledger_after();
601
602 T_ASSERT_EQ(
603 neural_footprint_before + vm_size,
604 neural_footprint_after,
605 "neural footprint increased %llu -> %llu pages",
606 neural_footprint_before / vm_kernel_page_size,
607 neural_footprint_after / vm_kernel_page_size);
608 T_ASSERT_EQ(
609 0ULL,
610 neural_total_after,
611 "neural total is zero %llu -> %llu pages",
612 neural_total_before / vm_kernel_page_size,
613 neural_total_after / vm_kernel_page_size);
614
615 T_LOG("---> Move primary nofootprint to neural");
616 get_ledger_before();
617 kr = mach_memory_entry_ownership(
618 nofootprint_port, /* entry port */
619 TASK_NULL, /* owner remains unchanged */
620 VM_LEDGER_TAG_NEURAL, /* ledger-tag */
621 VM_LEDGER_FLAG_NO_FOOTPRINT); /* ledger flags */
622 T_ASSERT_MACH_SUCCESS(kr,
623 "mach_memory_entry_ownership() primary neural footprint 0x%x (%s)",
624 kr, mach_error_string(kr));
625
626 get_ledger_after();
627 T_ASSERT_EQ(
628 neural_nofootprint_before + vm_size,
629 neural_nofootprint_after,
630 "neural nofootprint increased %llu -> %llu pages",
631 neural_nofootprint_before / vm_kernel_page_size,
632 neural_nofootprint_after / vm_kernel_page_size);
633 T_ASSERT_EQ(
634 neural_total_before + vm_size,
635 neural_total_after,
636 "neural total increased %llu -> %llu pages",
637 neural_total_before / vm_kernel_page_size,
638 neural_total_after / vm_kernel_page_size);
639
640 T_LOG("---> Move secondary nofootprint to neural");
641 get_ledger_before();
642 kr = mach_memory_entry_ownership(
643 nofootprint_port2, /* entry port */
644 TASK_NULL, /* owner remains unchanged */
645 VM_LEDGER_TAG_NEURAL, /* ledger-tag */
646 VM_LEDGER_FLAG_NO_FOOTPRINT); /* ledger flags */
647 T_ASSERT_MACH_SUCCESS(kr,
648 "mach_memory_entry_ownership() primary neural footprint 0x%x (%s)",
649 kr, mach_error_string(kr));
650
651 get_ledger_after();
652 T_ASSERT_EQ(
653 neural_nofootprint_before + vm_size,
654 neural_nofootprint_after,
655 "neural nofootprint increased %llu -> %llu pages",
656 neural_nofootprint_before / vm_kernel_page_size,
657 neural_nofootprint_after / vm_kernel_page_size);
658 T_ASSERT_EQ(
659 neural_total_before + vm_size,
660 neural_total_after,
661 "neural total increased %llu -> %llu pages",
662 neural_total_before / vm_kernel_page_size,
663 neural_total_after / vm_kernel_page_size);
664
665 neural_lifetime_max = neural_total_after;
666
667 T_LOG("---? Check neural total lifetime max and max interval");
668 ret = proc_pid_rusage(getpid(), RUSAGE_INFO_V6, (rusage_info_t *)&ru);
669 T_QUIET;
670 T_ASSERT_POSIX_SUCCESS(ret, "proc_pid_rusage");
671 T_EXPECT_EQ(ru.ri_lifetime_max_neural_footprint,
672 neural_total_after,
673 "Neural max footprint is equal to 4 x vm_size (in pages): %llu = %llu",
674 ru.ri_lifetime_max_neural_footprint / vm_kernel_page_size,
675 4 * vm_size / vm_kernel_page_size);
676 T_EXPECT_EQ(ru.ri_interval_max_neural_footprint,
677 neural_total_after,
678 "Neural max interval footprint is equal to current neural total: %llu = %llu",
679 ru.ri_interval_max_neural_footprint / vm_kernel_page_size,
680 neural_total_after / vm_kernel_page_size);
681
682 T_LOG("---> compress primary footprint");
683 get_ledger_before();
684 compress_pages(footprint_vm_addr, vm_size);
685 get_ledger_after();
686 T_ASSERT_EQ(
687 neural_footprint_before - vm_size,
688 neural_footprint_after,
689 "neural footprint decreased %llu -> %llu pages",
690 neural_footprint_before / vm_kernel_page_size,
691 neural_footprint_after / vm_kernel_page_size);
692 T_ASSERT_EQ(
693 neural_footprint_compressed_before + vm_size,
694 neural_footprint_compressed_after,
695 "neural footprint compressed increased %llu -> %llu pages",
696 neural_footprint_compressed_before / vm_kernel_page_size,
697 neural_footprint_compressed_after / vm_kernel_page_size);
698 T_ASSERT_EQ(
699 neural_total_before,
700 neural_total_after,
701 "neural total did not change %llu -> %llu pages",
702 neural_total_before / vm_kernel_page_size,
703 neural_total_after / vm_kernel_page_size);
704
705
706 T_LOG("---> compress primary nofootprint");
707 get_ledger_before();
708 compress_pages(nofootprint_vm_addr, vm_size);
709 get_ledger_after();
710 T_ASSERT_EQ(
711 neural_nofootprint_before - vm_size,
712 neural_nofootprint_after,
713 "neural nofootprint decreased %llu -> %llu pages",
714 neural_nofootprint_before / vm_kernel_page_size,
715 neural_nofootprint_after / vm_kernel_page_size);
716 T_ASSERT_EQ(
717 neural_nofootprint_compressed_before + vm_size,
718 neural_nofootprint_compressed_after,
719 "neural nofootprint compressed increased %llu -> %llu pages",
720 neural_nofootprint_compressed_before / vm_kernel_page_size,
721 neural_nofootprint_compressed_after / vm_kernel_page_size);
722 T_ASSERT_EQ(
723 neural_total_before,
724 neural_total_after,
725 "neural total did not change %llu -> %llu pages",
726 neural_total_before / vm_kernel_page_size,
727 neural_total_after / vm_kernel_page_size);
728
729 T_LOG("---> Decompress primary footprint");
730 get_ledger_before();
731 uncompress_pages(footprint_vm_addr, vm_size);
732 get_ledger_after();
733 T_ASSERT_EQ(
734 neural_footprint_before + vm_size,
735 neural_footprint_after,
736 "neural footprint increased %llu -> %llu pages",
737 neural_footprint_before / vm_kernel_page_size,
738 neural_footprint_after / vm_kernel_page_size);
739 T_ASSERT_EQ(
740 neural_footprint_compressed_before - vm_size,
741 neural_footprint_compressed_after,
742 "neural footprint compressed decreased %llu -> %llu pages",
743 neural_footprint_compressed_before / vm_kernel_page_size,
744 neural_footprint_compressed_after / vm_kernel_page_size);
745 T_ASSERT_EQ(
746 neural_total_before,
747 neural_total_after,
748 "neural total did not change %llu -> %llu pages",
749 neural_total_before / vm_kernel_page_size,
750 neural_total_after / vm_kernel_page_size);
751
752 T_LOG("---> Make primary footprint volatile");
753 get_ledger_before();
754 make_volatile(footprint_vm_addr);
755 get_ledger_after();
756 T_ASSERT_EQ(
757 neural_footprint_before - vm_size,
758 neural_footprint_after,
759 "neural footprint decreased %llu -> %llu pages",
760 neural_footprint_before / vm_kernel_page_size,
761 neural_footprint_after / vm_kernel_page_size);
762 T_ASSERT_EQ(
763 neural_nofootprint_before + vm_size,
764 neural_nofootprint_after,
765 "neural nofootprint increased %llu -> %llu pages",
766 neural_nofootprint_before / vm_kernel_page_size,
767 neural_nofootprint_after / vm_kernel_page_size);
768 T_ASSERT_EQ(
769 neural_total_before,
770 neural_total_after,
771 "neural total did not change %llu -> %llu pages",
772 neural_total_before / vm_kernel_page_size,
773 neural_total_after / vm_kernel_page_size);
774
775 T_LOG("---> Make primary nofootprint (compressed) volatile");
776 get_ledger_before();
777 make_volatile(nofootprint_vm_addr);
778 get_ledger_after();
779 T_ASSERT_EQ(
780 neural_nofootprint_before,
781 neural_nofootprint_after,
782 "neural nofootprint did not change %llu -> %llu pages",
783 neural_nofootprint_before / vm_kernel_page_size,
784 neural_nofootprint_after / vm_kernel_page_size);
785 T_ASSERT_EQ(
786 neural_nofootprint_compressed_before,
787 neural_nofootprint_compressed_after,
788 "neural nofootprint_compressed did not change (volatile now) %llu -> %llu pages",
789 neural_nofootprint_compressed_before / vm_kernel_page_size,
790 neural_nofootprint_compressed_after / vm_kernel_page_size);
791 T_ASSERT_EQ(
792 neural_footprint_before,
793 neural_footprint_after,
794 "neural footprint did not change %llu -> %llu pages",
795 neural_footprint_before / vm_kernel_page_size,
796 neural_footprint_after / vm_kernel_page_size);
797 T_ASSERT_EQ(
798 neural_total_before - vm_size,
799 neural_total_after,
800 "neural total decreased %llu -> %llu pages",
801 neural_total_before / vm_kernel_page_size,
802 neural_total_after / vm_kernel_page_size);
803
804 reset_max_interval(neural_total_after);
805
806 T_LOG("---> Make primary footprint non-volatile");
807 get_ledger_before();
808 make_nonvolatile(footprint_vm_addr);
809 get_ledger_after();
810 T_ASSERT_EQ(
811 neural_footprint_before + vm_size,
812 neural_footprint_after,
813 "neural footprint increased %llu -> %llu pages",
814 neural_footprint_before / vm_kernel_page_size,
815 neural_footprint_after / vm_kernel_page_size);
816 T_ASSERT_EQ(
817 neural_nofootprint_before - vm_size,
818 neural_nofootprint_after,
819 "neural nofootprint decreased %llu -> %llu pages",
820 neural_nofootprint_before / vm_kernel_page_size,
821 neural_nofootprint_after / vm_kernel_page_size);
822 T_ASSERT_EQ(
823 neural_total_before,
824 neural_total_after,
825 "neural total did not change %llu -> %llu pages",
826 neural_total_before / vm_kernel_page_size,
827 neural_total_after / vm_kernel_page_size);
828
829 T_LOG("---> Make primary nofootprint (compressed) non-volatile");
830 get_ledger_before();
831 make_nonvolatile(nofootprint_vm_addr);
832 get_ledger_after();
833 T_ASSERT_EQ(
834 neural_footprint_before,
835 neural_footprint_after,
836 "neural footprint did not change %llu -> %llu pages",
837 neural_footprint_before / vm_kernel_page_size,
838 neural_footprint_after / vm_kernel_page_size);
839 T_ASSERT_EQ(
840 neural_nofootprint_before,
841 neural_nofootprint_after,
842 "neural nofootprint did not change %llu -> %llu pages",
843 neural_nofootprint_before / vm_kernel_page_size,
844 neural_nofootprint_after / vm_kernel_page_size);
845 T_ASSERT_EQ(
846 neural_nofootprint_compressed_before,
847 neural_nofootprint_compressed_after,
848 "neural nofootprint_compressed did not change (now non-volatile) %llu -> %llu pages",
849 neural_nofootprint_compressed_before / vm_kernel_page_size,
850 neural_nofootprint_compressed_after / vm_kernel_page_size);
851 T_ASSERT_EQ(
852 neural_total_before + vm_size,
853 neural_total_after,
854 "neural total increased %llu -> %llu pages",
855 neural_total_before / vm_kernel_page_size,
856 neural_total_after / vm_kernel_page_size);
857
858 T_LOG("---? Check neural total max interval");
859 ret = proc_pid_rusage(getpid(), RUSAGE_INFO_V6, (rusage_info_t *)&ru);
860 T_QUIET;
861 T_ASSERT_POSIX_SUCCESS(ret, "proc_pid_rusage");
862 T_ASSERT_EQ(ru.ri_interval_max_neural_footprint,
863 neural_total_after,
864 "Neural max interval footprint is equal to total: %llu = %llu",
865 ru.ri_interval_max_neural_footprint / vm_kernel_page_size,
866 neural_total_after / vm_kernel_page_size);
867
868
869 T_LOG("---> Take no-footprint ownership of the file");
870 get_ledger_before();
871 kr = mach_memory_entry_ownership(
872 file_mem_entry, /* entry port */
873 mach_task_self(), /* claim ownership */
874 VM_LEDGER_TAG_NEURAL, /* ledger-tag */
875 VM_LEDGER_FLAG_NO_FOOTPRINT); /* ledger flags */
876 T_ASSERT_MACH_SUCCESS(kr,
877 "mach_memory_entry_ownership() file neural no-footprint 0x%x (%s)",
878 kr, mach_error_string(kr));
879 get_ledger_after();
880 T_ASSERT_EQ(
881 neural_nofootprint_before + vm_size,
882 neural_nofootprint_after,
883 "neural nofootprint increased %llu -> %llu pages",
884 neural_nofootprint_before / vm_kernel_page_size,
885 neural_nofootprint_after / vm_kernel_page_size);
886 T_ASSERT_EQ(
887 neural_total_before + vm_size,
888 neural_total_after,
889 "neural total increase %llu -> %llu pages",
890 neural_total_before / vm_kernel_page_size,
891 neural_total_after / vm_kernel_page_size);
892 // fprintf(stdout, "pausing...\n"); fflush(stdout); getchar();
893
894
895 T_LOG("---> Take footprint ownership of the file");
896 get_ledger_before();
897 kr = mach_memory_entry_ownership(
898 file_mem_entry, /* entry port */
899 MACH_PORT_NULL, /* owner remains unchanged */
900 VM_LEDGER_TAG_NEURAL, /* ledger-tag */
901 0); /* ledger flags */
902 T_ASSERT_MACH_SUCCESS(kr,
903 "mach_memory_entry_ownership() file neural footprint 0x%x (%s)",
904 kr, mach_error_string(kr));
905 get_ledger_after();
906 T_ASSERT_EQ(
907 neural_nofootprint_before - vm_size,
908 neural_nofootprint_after,
909 "neural nofootprint decreased %llu -> %llu pages",
910 neural_nofootprint_before / vm_kernel_page_size,
911 neural_nofootprint_after / vm_kernel_page_size);
912 T_ASSERT_EQ(
913 neural_footprint_before + vm_size,
914 neural_footprint_after,
915 "neural footprint increased %llu -> %llu pages",
916 neural_footprint_before / vm_kernel_page_size,
917 neural_footprint_after / vm_kernel_page_size);
918 T_ASSERT_EQ(
919 neural_total_before - vm_size,
920 neural_total_after,
921 "neural total decreased %llu -> %llu pages",
922 neural_total_before / vm_kernel_page_size,
923 neural_total_after / vm_kernel_page_size);
924 // fprintf(stdout, "pausing...\n"); fflush(stdout); getchar();
925
926 T_LOG("---> Unlock the file");
927 get_ledger_before();
928 T_ASSERT_POSIX_SUCCESS(munlock(file_addr, (size_t)vm_size), NULL);
929 get_ledger_after();
930 T_ASSERT_EQ(
931 neural_nofootprint_before,
932 neural_nofootprint_after,
933 "neural nofootprint unchanged %llu -> %llu pages",
934 neural_nofootprint_before / vm_kernel_page_size,
935 neural_nofootprint_after / vm_kernel_page_size);
936 T_ASSERT_EQ(
937 neural_footprint_before - vm_size,
938 neural_footprint_after,
939 "neural footprint decreased %llu -> %llu pages",
940 neural_footprint_before / vm_kernel_page_size,
941 neural_footprint_after / vm_kernel_page_size);
942 T_ASSERT_EQ(
943 neural_total_before,
944 neural_total_after,
945 "neural total unchanged %llu -> %llu pages",
946 neural_total_before / vm_kernel_page_size,
947 neural_total_after / vm_kernel_page_size);
948 // fprintf(stdout, "pausing...\n"); fflush(stdout); getchar();
949
950 T_LOG("---> Relock the file");
951 get_ledger_before();
952 T_ASSERT_POSIX_SUCCESS(mlock(file_addr, (size_t)vm_size), NULL);
953 get_ledger_after();
954 T_ASSERT_EQ(
955 neural_nofootprint_before,
956 neural_nofootprint_after,
957 "neural nofootprint unchanged %llu -> %llu pages",
958 neural_nofootprint_before / vm_kernel_page_size,
959 neural_nofootprint_after / vm_kernel_page_size);
960 T_ASSERT_EQ(
961 neural_footprint_before + vm_size,
962 neural_footprint_after,
963 "neural footprint increased %llu -> %llu pages",
964 neural_footprint_before / vm_kernel_page_size,
965 neural_footprint_after / vm_kernel_page_size);
966 T_ASSERT_EQ(
967 neural_total_before,
968 neural_total_after,
969 "neural total unchanged %llu -> %llu pages",
970 neural_total_before / vm_kernel_page_size,
971 neural_total_after / vm_kernel_page_size);
972 // fprintf(stdout, "pausing...\n"); fflush(stdout); getchar();
973 T_ASSERT_POSIX_SUCCESS(munlock(file_addr, (size_t)vm_size), NULL);
974 // fprintf(stdout, "pausing...\n"); fflush(stdout); getchar();
975
976 T_LOG("<--- Deallocate");
977 get_ledger_before();
978 /* deallocating memory while holding memory entry... */
979 kr = mach_vm_deallocate(mach_task_self(), footprint_vm_addr, vm_size);
980 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
981 kr = mach_vm_deallocate(mach_task_self(), nofootprint_vm_addr, vm_size);
982 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
983 kr = mach_vm_deallocate(mach_task_self(), footprint_vm_addr2, vm_size);
984 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
985 kr = mach_vm_deallocate(mach_task_self(), nofootprint_vm_addr2, vm_size);
986 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
987 /* releasing the memory entry... */
988 kr = mach_port_deallocate(mach_task_self(), footprint_port);
989 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
990 kr = mach_port_deallocate(mach_task_self(), footprint_port2);
991 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
992 kr = mach_port_deallocate(mach_task_self(), nofootprint_port);
993 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
994 kr = mach_port_deallocate(mach_task_self(), nofootprint_port2);
995 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "vm_deallocate() ");
996 get_ledger_after();
997
998 T_ASSERT_EQ(
999 0ULL,
1000 neural_footprint_after,
1001 "neural footprint zero %llu -> %llu pages",
1002 neural_footprint_before / vm_kernel_page_size,
1003 neural_footprint_after / vm_kernel_page_size);
1004 T_ASSERT_EQ(
1005 0ULL,
1006 neural_footprint_compressed_after,
1007 "neural footprint compressed zero %llu -> %llu pages",
1008 neural_footprint_compressed_before / vm_kernel_page_size,
1009 neural_footprint_compressed_after / vm_kernel_page_size);
1010 T_ASSERT_EQ(
1011 0ULL,
1012 neural_nofootprint_after,
1013 "neural nofootprint zero %llu -> %llu pages",
1014 neural_nofootprint_before / vm_kernel_page_size,
1015 neural_nofootprint_after / vm_kernel_page_size);
1016 T_ASSERT_EQ(
1017 0ULL,
1018 neural_nofootprint_compressed_after,
1019 "neural nofootprint compressed to zero %llu -> %llu pages",
1020 neural_nofootprint_compressed_before / vm_kernel_page_size,
1021 neural_nofootprint_compressed_after / vm_kernel_page_size);
1022 T_ASSERT_EQ(
1023 0ULL,
1024 neural_total_after,
1025 "neural total zero %llu -> %llu pages",
1026 neural_total_before / vm_kernel_page_size,
1027 neural_total_after / vm_kernel_page_size);
1028 }
1029