1 #include <darwintest.h>
2 #include <darwintest_utils.h>
3
4 #include <mach/mach_error.h>
5 #include <mach/mach_init.h>
6 #include <mach/memory_entry.h>
7 #include <mach/mach_port.h>
8 #include <mach/mach_vm.h>
9 #include <mach/memory_entry.h>
10 #include <mach/task.h>
11 #include <mach/task_info.h>
12 #include <mach/vm_map.h>
13
14 #include <sys/kern_memorystatus.h>
15 #include <sys/mman.h>
16 #include <sys/types.h>
17 #include <sys/sysctl.h>
18
19 #include <TargetConditionals.h>
20
21 #include <Kernel/kern/ledger.h>
22 extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);
23
24 T_GLOBAL_META(T_META_RUN_CONCURRENTLY(true));
25 static boolean_t legacy_footprint = FALSE;
26
27 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
28 #define TEST_VM_NAMESPACE "xnu.vm_legacy"
29 #else /* ENTITLED && __arm64__ */
30 #define TEST_VM_NAMESPACE "xnu.vm"
31 #endif /* ENTITLED && __arm64__ */
32
33 T_GLOBAL_META(
34 T_META_NAMESPACE(TEST_VM_NAMESPACE),
35 T_META_RADAR_COMPONENT_NAME("xnu"),
36 T_META_RADAR_COMPONENT_VERSION("VM"),
37 T_META_RUN_CONCURRENTLY(true));
38
39 #define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */
40
41 static int64_t ledger_count = -1;
42 static int footprint_index = -1;
43 static int pagetable_index = -1;
44 static struct ledger_entry_info *lei = NULL;
45
46 static void
ledger_init(void)47 ledger_init(void)
48 {
49 static int ledger_inited = 0;
50 struct ledger_info li;
51 struct ledger_template_info *templateInfo;
52 int64_t templateCnt;
53 int i;
54
55 if (ledger_inited) {
56 return;
57 }
58 ledger_inited = 1;
59
60 T_SETUPBEGIN;
61
62 legacy_footprint = FALSE;
63 #if LEGACY_FOOTPRINT_ENTITLED
64 int legacy_footprint_entitlement_mode;
65 size_t oldlen;
66 int ret;
67
68 T_QUIET;
69 T_WITH_ERRNO;
70 oldlen = sizeof(legacy_footprint_entitlement_mode);
71 ret = sysctlbyname("kern.legacy_footprint_entitlement_mode",
72 &legacy_footprint_entitlement_mode,
73 &oldlen,
74 NULL,
75 0);
76 if (ret == 0 && legacy_footprint_entitlement_mode == 2) {
77 legacy_footprint = TRUE;
78 }
79 #endif /* LEGACY_FOOTPRINT_ENTITLED */
80
81 T_QUIET;
82 T_WITH_ERRNO;
83 T_ASSERT_EQ(ledger(LEDGER_INFO,
84 (caddr_t)(uintptr_t)getpid(),
85 (caddr_t)&li,
86 NULL),
87 0,
88 "ledger(LEDGER_INFO)");
89
90 templateCnt = li.li_entries;
91 templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
92 T_QUIET;
93 T_WITH_ERRNO;
94 T_ASSERT_NE(templateInfo, NULL, "malloc()");
95
96 ledger_count = li.li_entries;
97 footprint_index = -1;
98 pagetable_index = -1;
99 T_QUIET;
100 T_WITH_ERRNO;
101 T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
102 (caddr_t)templateInfo,
103 (caddr_t)&templateCnt,
104 NULL),
105 0,
106 "ledger(LEDGER_TEMPLATE_INFO)");
107 for (i = 0; i < templateCnt; i++) {
108 if (!strncmp(templateInfo[i].lti_name,
109 "phys_footprint",
110 strlen("phys_footprint"))) {
111 footprint_index = i;
112 } else if (!strncmp(templateInfo[i].lti_name,
113 "page_table",
114 strlen("page_table"))) {
115 pagetable_index = i;
116 }
117 }
118 free(templateInfo);
119
120 lei = (struct ledger_entry_info *)
121 malloc((size_t)ledger_count * sizeof(*lei));
122 T_QUIET;
123 T_WITH_ERRNO;
124 T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");
125
126 T_QUIET;
127 T_ASSERT_NE(footprint_index, -1, "no footprint_index");
128 T_QUIET;
129 T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");
130
131 T_SETUPEND;
132 }
133
134 static void
get_ledger_info(uint64_t * phys_footprint,uint64_t * page_table)135 get_ledger_info(
136 uint64_t *phys_footprint,
137 uint64_t *page_table)
138 {
139 int64_t count;
140
141 count = ledger_count;
142 T_QUIET;
143 T_WITH_ERRNO;
144 T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
145 (caddr_t)(uintptr_t)getpid(),
146 (caddr_t)lei,
147 (caddr_t)&count),
148 0,
149 "ledger(LEDGER_ENTRY_INFO)");
150 T_QUIET;
151 T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
152 T_QUIET;
153 T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
154 if (phys_footprint) {
155 *phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
156 }
157 if (page_table) {
158 *page_table = (uint64_t)(lei[pagetable_index].lei_balance);
159 }
160 }
161
162 static mach_vm_address_t
pre_warm_with_tag(mach_vm_size_t vm_size,int tag)163 pre_warm_with_tag(
164 mach_vm_size_t vm_size,
165 int tag)
166 {
167 kern_return_t kr;
168 mach_vm_address_t vm_addr;
169 unsigned char BigBufOnStack[100 * 1024];
170 uint64_t footprint, page_table;
171
172 /* make sure ledgers are ready to be queried */
173 ledger_init();
174
175 T_SETUPBEGIN;
176
177 /*
178 * Touch a few pages ahead on the stack, to make
179 * sure we don't see a footprint increase due to
180 * an extra stack page later.
181 */
182 memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
183 T_QUIET;
184 T_EXPECT_EQ(BigBufOnStack[0], 0xb,
185 "BigBufOnStack[0] == 0x%x",
186 BigBufOnStack[0]);
187 T_QUIET;
188 T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
189 "BigBufOnStack[%lu] == 0x%x",
190 sizeof(BigBufOnStack),
191 BigBufOnStack[sizeof(BigBufOnStack) - 1]);
192
193 /*
194 * Pre-allocate, touch and then release the same amount
195 * of memory we'll be allocating later during the test,
196 * to account for any memory overhead (page tables, global
197 * variables, ...). Supplied VM tag should ensure memory
198 * is allocated in the appropriate VM range.
199 */
200 vm_addr = 0;
201 kr = mach_vm_allocate(mach_task_self(),
202 &vm_addr,
203 vm_size,
204 VM_FLAGS_ANYWHERE | VM_MAKE_TAG(tag));
205 T_QUIET;
206 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
207 vm_size, kr, mach_error_string(kr));
208 memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
209 kr = mach_vm_deallocate(mach_task_self(),
210 vm_addr,
211 vm_size);
212 T_QUIET;
213 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
214 kr, mach_error_string(kr));
215
216 /*
217 * Exercise the ledger code to make sure it's ready to run
218 * without any extra memory overhead later.
219 */
220 get_ledger_info(&footprint, &page_table);
221
222 T_SETUPEND;
223
224 /*
225 * Return the start of the virtual range we pre-warmed, so that the
226 * test can check that it's using the same range.
227 */
228 return vm_addr;
229 }
230
231 static mach_vm_address_t
pre_warm(mach_vm_size_t vm_size)232 pre_warm(
233 mach_vm_size_t vm_size)
234 {
235 /*
236 * Default to not tagging pre_warm memory.
237 */
238 return pre_warm_with_tag(vm_size, 0);
239 }
240
241 T_DECL(phys_footprint_anonymous,
242 "phys_footprint for anonymous memory",
243 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
244 {
245 uint64_t footprint_before, pagetable_before;
246 uint64_t footprint_after, pagetable_after;
247 uint64_t footprint_expected;
248 kern_return_t kr;
249 mach_vm_address_t pre_vm_addr, vm_addr;
250 mach_vm_size_t vm_size, dirty_size;
251
252 /* pre-warm to account for page table expansion */
253 pre_vm_addr = pre_warm(MEM_SIZE);
254
255 /* allocating virtual memory... */
256 get_ledger_info(&footprint_before, &pagetable_before);
257 vm_addr = 0;
258 vm_size = MEM_SIZE;
259 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
260 VM_FLAGS_ANYWHERE);
261 T_QUIET;
262 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
263 kr, mach_error_string(kr));
264 T_QUIET;
265 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
266 /* ... should not change footprint */
267 get_ledger_info(&footprint_after, &pagetable_after);
268 footprint_expected = footprint_before;
269 footprint_expected += (pagetable_after - pagetable_before);
270 T_LOG("virtual allocation does not change phys_footprint");
271 T_EXPECT_EQ(footprint_after, footprint_expected,
272 "virtual allocation of %lld bytes: "
273 "footprint %lld -> %lld expected %lld delta %lld",
274 vm_size, footprint_before, footprint_after,
275 footprint_expected, footprint_after - footprint_expected);
276
277 /* touching memory... */
278 get_ledger_info(&footprint_before, &pagetable_before);
279 dirty_size = vm_size / 2;
280 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
281 /* ... should increase footprint */
282 get_ledger_info(&footprint_after, &pagetable_after);
283 footprint_expected = footprint_before + dirty_size;
284 footprint_expected += (pagetable_after - pagetable_before);
285 T_LOG("modifying anonymous memory increases phys_footprint");
286 T_EXPECT_EQ(footprint_after, footprint_expected,
287 "touched %lld bytes: "
288 "footprint %lld -> %lld expected %lld delta %lld",
289 dirty_size, footprint_before, footprint_after,
290 footprint_expected, footprint_after - footprint_expected);
291
292 /* deallocating memory... */
293 get_ledger_info(&footprint_before, &pagetable_before);
294 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
295 T_QUIET;
296 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
297 kr, mach_error_string(kr));
298 /* ... should decrease footprint */
299 get_ledger_info(&footprint_after, &pagetable_after);
300 footprint_expected = footprint_before - dirty_size;
301 footprint_expected += (pagetable_after - pagetable_before);
302 T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
303 T_EXPECT_EQ(footprint_after, footprint_expected,
304 "deallocated %lld dirty bytes: "
305 "footprint %lld -> %lld expected %lld delta %lld",
306 dirty_size, footprint_before, footprint_after,
307 footprint_expected, footprint_after - footprint_expected);
308 }
309
310 #define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
311 #define TEMP_FILE_SIZE (1 * 1024 * 1024)
312
313 T_DECL(phys_footprint_file,
314 "phys_footprint for mapped file",
315 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
316 {
317 uint64_t footprint_before, pagetable_before;
318 uint64_t footprint_after, pagetable_after;
319 uint64_t footprint_expected;
320 mach_vm_address_t pre_vm_addr;
321 int fd;
322 char *map_addr;
323 size_t map_size, dirty_size;
324 ssize_t nbytes;
325 char tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
326 char *buf;
327 size_t buf_size;
328
329 T_SETUPBEGIN;
330 buf_size = TEMP_FILE_SIZE;
331 T_QUIET;
332 T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
333 "allocate %zu-byte buffer", buf_size);
334 memset(buf, 'f', buf_size);
335 T_WITH_ERRNO;
336 T_QUIET;
337 T_ASSERT_NOTNULL(mktemp(tmp_file_name),
338 "create temporary file name");
339 T_WITH_ERRNO;
340 T_QUIET;
341 T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
342 0,
343 "create temp file");
344 T_WITH_ERRNO;
345 T_QUIET;
346 T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
347 (ssize_t)buf_size,
348 "write %zu bytes", buf_size);
349 free(buf);
350 T_SETUPEND;
351
352 /* pre-warm to account for page table expansion */
353 #if TARGET_OS_OSX
354 pre_vm_addr = pre_warm_with_tag(TEMP_FILE_SIZE, VM_MEMORY_MALLOC);
355 #else
356 pre_vm_addr = pre_warm_with_tag(TEMP_FILE_SIZE, VM_MEMORY_MALLOC_TINY);
357 #endif
358
359 /* mapping a file does not impact footprint... */
360 get_ledger_info(&footprint_before, &pagetable_before);
361 map_size = TEMP_FILE_SIZE;
362 T_WITH_ERRNO;
363 T_QUIET;
364 T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
365 PROT_READ | PROT_WRITE,
366 MAP_FILE | MAP_SHARED, fd, 0),
367 "mmap()");
368 T_QUIET;
369 T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
370 "pre-warm mishap");
371 /* ... should not change footprint */
372 get_ledger_info(&footprint_after, &pagetable_after);
373 footprint_expected = footprint_before;
374 footprint_expected += (pagetable_after - pagetable_before);
375 T_LOG("mapping file does not change phys_footprint");
376 T_EXPECT_EQ(footprint_after, footprint_expected,
377 "mapping file with %zu bytes: "
378 "footprint %lld -> %lld expected %lld delta %lld",
379 map_size, footprint_before, footprint_after,
380 footprint_expected, footprint_after - footprint_expected);
381
382 /* touching file-backed memory... */
383 get_ledger_info(&footprint_before, &pagetable_before);
384 dirty_size = map_size / 2;
385 memset(map_addr, 'F', dirty_size);
386 /* ... should not impact footprint */
387 get_ledger_info(&footprint_after, &pagetable_after);
388 footprint_expected = footprint_before;
389 footprint_expected += (pagetable_after - pagetable_before);
390 T_LOG("modifying file-backed memory does not impact phys_footprint");
391 T_EXPECT_EQ(footprint_after, footprint_expected,
392 "touched %zu bytes: "
393 "footprint %lld -> %lld expected %lld delta %lld",
394 dirty_size, footprint_before, footprint_after,
395 footprint_expected, footprint_after - footprint_expected);
396
397 /* deallocating file-backed memory... */
398 get_ledger_info(&footprint_before, &pagetable_before);
399 T_WITH_ERRNO;
400 T_QUIET;
401 T_ASSERT_EQ(munmap(map_addr, map_size),
402 0,
403 "unmap file");
404 /* ... should not impact footprint */
405 get_ledger_info(&footprint_after, &pagetable_after);
406 footprint_expected = footprint_before;
407 footprint_expected += (pagetable_after - pagetable_before);
408 T_LOG("unmapping file-backed memory does not impact phys_footprint");
409 T_EXPECT_EQ(footprint_after, footprint_expected,
410 "unmapped %zu dirty bytes: "
411 "footprint %lld -> %lld expected %lld delta %lld",
412 dirty_size, footprint_before, footprint_after,
413 footprint_expected, footprint_after - footprint_expected);
414 }
415
416 T_DECL(phys_footprint_purgeable,
417 "phys_footprint for purgeable memory",
418 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
419 {
420 uint64_t footprint_before, pagetable_before;
421 uint64_t footprint_after, pagetable_after;
422 uint64_t footprint_expected;
423 kern_return_t kr;
424 mach_vm_address_t pre_vm_addr, vm_addr;
425 mach_vm_size_t vm_size, dirty_size;
426 int state;
427
428 /* pre-warm to account for page table expansion */
429 pre_vm_addr = pre_warm(MEM_SIZE);
430
431 /* allocating purgeable virtual memory... */
432 get_ledger_info(&footprint_before, &pagetable_before);
433 vm_addr = 0;
434 vm_size = MEM_SIZE;
435 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
436 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
437 T_QUIET;
438 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
439 kr, mach_error_string(kr));
440 T_QUIET;
441 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
442 /* ... should not change footprint */
443 get_ledger_info(&footprint_after, &pagetable_after);
444 footprint_expected = footprint_before;
445 footprint_expected += (pagetable_after - pagetable_before);
446 T_LOG("purgeable virtual allocation does not change phys_footprint");
447 T_EXPECT_EQ(footprint_after, footprint_expected,
448 "purgeable virtual allocation of %lld bytes: "
449 "footprint %lld -> %lld expected %lld delta %lld",
450 vm_size, footprint_before, footprint_after,
451 footprint_expected, footprint_after - footprint_expected);
452
453 /* touching memory... */
454 get_ledger_info(&footprint_before, &pagetable_before);
455 dirty_size = vm_size / 2;
456 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
457 /* ... should increase footprint */
458 get_ledger_info(&footprint_after, &pagetable_after);
459 footprint_expected = footprint_before + dirty_size;
460 footprint_expected += (pagetable_after - pagetable_before);
461 T_LOG("modifying anonymous memory increases phys_footprint");
462 T_EXPECT_EQ(footprint_after, footprint_expected,
463 "touched %lld bytes: "
464 "footprint %lld -> %lld expected %lld delta %lld",
465 dirty_size, footprint_before, footprint_after,
466 footprint_expected, footprint_after - footprint_expected);
467
468 /* making it volatile... */
469 get_ledger_info(&footprint_before, &pagetable_before);
470 state = VM_PURGABLE_VOLATILE;
471 T_QUIET;
472 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
473 vm_addr,
474 VM_PURGABLE_SET_STATE,
475 &state),
476 KERN_SUCCESS,
477 "vm_purgable_control(VOLATILE)");
478 T_QUIET;
479 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
480 "memory was non-volatile");
481 /* ... should decrease footprint */
482 get_ledger_info(&footprint_after, &pagetable_after);
483 footprint_expected = footprint_before - dirty_size;
484 footprint_expected += (pagetable_after - pagetable_before);
485 T_LOG("making volatile decreases phys_footprint");
486 T_EXPECT_EQ(footprint_after, footprint_expected,
487 "made volatile %lld dirty bytes: "
488 "footprint %lld -> %lld expected %lld delta %lld",
489 dirty_size, footprint_before, footprint_after,
490 footprint_expected, footprint_after - footprint_expected);
491
492 /* making it non-volatile... */
493 get_ledger_info(&footprint_before, &pagetable_before);
494 state = VM_PURGABLE_NONVOLATILE;
495 T_QUIET;
496 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
497 vm_addr,
498 VM_PURGABLE_SET_STATE,
499 &state),
500 KERN_SUCCESS,
501 "vm_purgable_control(NONVOLATILE)");
502 T_QUIET;
503 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
504 "memory was volatile");
505 /* ... should increase footprint */
506 get_ledger_info(&footprint_after, &pagetable_after);
507 footprint_expected = footprint_before + dirty_size;
508 footprint_expected += (pagetable_after - pagetable_before);
509 T_LOG("making non-volatile increases phys_footprint");
510 T_EXPECT_EQ(footprint_after, footprint_expected,
511 "made non-volatile %lld dirty bytes: "
512 "footprint %lld -> %lld expected %lld delta %lld",
513 dirty_size, footprint_before, footprint_after,
514 footprint_expected, footprint_after - footprint_expected);
515
516 /* deallocating memory... */
517 get_ledger_info(&footprint_before, &pagetable_before);
518 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
519 T_QUIET;
520 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
521 kr, mach_error_string(kr));
522 /* ... should decrease footprint */
523 get_ledger_info(&footprint_after, &pagetable_after);
524 footprint_expected = footprint_before - dirty_size;
525 footprint_expected += (pagetable_after - pagetable_before);
526 T_LOG("deallocating memory decreases phys_footprint");
527 T_EXPECT_EQ(footprint_after, footprint_expected,
528 "deallocated %lld dirty bytes: "
529 "footprint %lld -> %lld expected %lld delta %lld",
530 dirty_size, footprint_before, footprint_after,
531 footprint_expected, footprint_after - footprint_expected);
532 }
533
534 T_DECL(phys_footprint_purgeable_ownership,
535 "phys_footprint for owned purgeable memory",
536 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
537 {
538 uint64_t footprint_before, pagetable_before;
539 uint64_t footprint_after, pagetable_after;
540 uint64_t footprint_expected;
541 kern_return_t kr;
542 mach_vm_address_t pre_vm_addr, vm_addr;
543 mach_vm_size_t vm_size, dirty_size, me_size;
544 int state;
545 mach_port_t me_port;
546
547 /* pre-warm to account for page table expansion */
548 pre_vm_addr = pre_warm(MEM_SIZE);
549
550 /* allocating purgeable virtual memory... */
551 get_ledger_info(&footprint_before, &pagetable_before);
552 vm_addr = 0;
553 vm_size = MEM_SIZE;
554 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
555 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
556 T_QUIET;
557 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
558 kr, mach_error_string(kr));
559 T_QUIET;
560 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
561 /* ... should not change footprint */
562 get_ledger_info(&footprint_after, &pagetable_after);
563 footprint_expected = footprint_before;
564 footprint_expected += (pagetable_after - pagetable_before);
565 T_LOG("purgeable virtual allocation does not change phys_footprint");
566 T_EXPECT_EQ(footprint_after, footprint_expected,
567 "purgeable virtual allocation of %lld bytes: "
568 "footprint %lld -> %lld expected %lld delta %lld",
569 vm_size, footprint_before, footprint_after,
570 footprint_expected, footprint_after - footprint_expected);
571
572 /* touching memory... */
573 get_ledger_info(&footprint_before, &pagetable_before);
574 dirty_size = vm_size / 2;
575 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
576 /* ... should increase footprint */
577 get_ledger_info(&footprint_after, &pagetable_after);
578 footprint_expected = footprint_before + dirty_size;
579 footprint_expected += (pagetable_after - pagetable_before);
580 T_LOG("modifying anonymous memory increases phys_footprint");
581 T_EXPECT_EQ(footprint_after, footprint_expected,
582 "touched %lld bytes: "
583 "footprint %lld -> %lld expected %lld delta %lld",
584 dirty_size, footprint_before, footprint_after,
585 footprint_expected, footprint_after - footprint_expected);
586
587 /* making it volatile... */
588 get_ledger_info(&footprint_before, &pagetable_before);
589 state = VM_PURGABLE_VOLATILE;
590 T_QUIET;
591 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
592 vm_addr,
593 VM_PURGABLE_SET_STATE,
594 &state),
595 KERN_SUCCESS,
596 "vm_purgable_control(VOLATILE)");
597 T_QUIET;
598 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
599 "memory was non-volatile");
600 /* ... should decrease footprint */
601 get_ledger_info(&footprint_after, &pagetable_after);
602 footprint_expected = footprint_before - dirty_size;
603 footprint_expected += (pagetable_after - pagetable_before);
604 T_LOG("making volatile decreases phys_footprint");
605 T_EXPECT_EQ(footprint_after, footprint_expected,
606 "made volatile %lld dirty bytes: "
607 "footprint %lld -> %lld expected %lld delta %lld",
608 dirty_size, footprint_before, footprint_after,
609 footprint_expected, footprint_after - footprint_expected);
610
611 /* making it non-volatile... */
612 get_ledger_info(&footprint_before, &pagetable_before);
613 state = VM_PURGABLE_NONVOLATILE;
614 T_QUIET;
615 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
616 vm_addr,
617 VM_PURGABLE_SET_STATE,
618 &state),
619 KERN_SUCCESS,
620 "vm_purgable_control(NONVOLATILE)");
621 T_QUIET;
622 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
623 "memory was volatile");
624 /* ... should increase footprint */
625 get_ledger_info(&footprint_after, &pagetable_after);
626 footprint_expected = footprint_before + dirty_size;
627 footprint_expected += (pagetable_after - pagetable_before);
628 T_LOG("making non-volatile increases phys_footprint");
629 T_EXPECT_EQ(footprint_after, footprint_expected,
630 "made non-volatile %lld dirty bytes: "
631 "footprint %lld -> %lld expected %lld delta %lld",
632 dirty_size, footprint_before, footprint_after,
633 footprint_expected, footprint_after - footprint_expected);
634
635 /* making a memory entry... */
636 get_ledger_info(&footprint_before, &pagetable_before);
637 me_size = vm_size;
638 me_port = MACH_PORT_NULL;
639 kr = mach_make_memory_entry_64(mach_task_self(),
640 &me_size,
641 vm_addr,
642 VM_PROT_READ | VM_PROT_WRITE,
643 &me_port,
644 MACH_PORT_NULL);
645 T_QUIET;
646 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
647 kr, mach_error_string(kr));
648 T_QUIET;
649 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
650 /* ... should not change footprint */
651 get_ledger_info(&footprint_after, &pagetable_after);
652 footprint_expected = footprint_before;
653 footprint_expected += (pagetable_after - pagetable_before);
654 T_LOG("making a memory entry does not change phys_footprint");
655 T_EXPECT_EQ(footprint_after, footprint_expected,
656 "making a memory entry of %lld bytes: "
657 "footprint %lld -> %lld expected %lld delta %lld",
658 vm_size, footprint_before, footprint_after,
659 footprint_expected, footprint_after - footprint_expected);
660
661 /* deallocating memory while holding memory entry... */
662 get_ledger_info(&footprint_before, &pagetable_before);
663 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
664 T_QUIET;
665 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
666 kr, mach_error_string(kr));
667 /* ... should not change footprint */
668 get_ledger_info(&footprint_after, &pagetable_after);
669 footprint_expected = footprint_before;
670 footprint_expected += (pagetable_after - pagetable_before);
671 T_LOG("deallocating owned memory while holding memory entry "
672 "does not change phys_footprint");
673 T_EXPECT_EQ(footprint_after, footprint_expected,
674 "deallocated %lld dirty bytes: "
675 "footprint %lld -> %lld expected %lld delta %lld",
676 dirty_size, footprint_before, footprint_after,
677 footprint_expected, footprint_after - footprint_expected);
678
679 get_ledger_info(&footprint_after, &pagetable_after);
680 footprint_expected = footprint_before;
681 footprint_expected += (pagetable_after - pagetable_before);
682 T_LOG("deallocating owned memory while holding memory entry "
683 "does not change phys_footprint");
684 T_EXPECT_EQ(footprint_after, footprint_expected,
685 "deallocated %lld dirty bytes: "
686 "footprint %lld -> %lld expected %lld delta %lld",
687 dirty_size, footprint_before, footprint_after,
688 footprint_expected, footprint_after - footprint_expected);
689
690 /* changing ownership of memory entry (from self to self)... */
691 get_ledger_info(&footprint_before, &pagetable_before);
692 kr = mach_memory_entry_ownership(me_port,
693 #if VM_LEDGER_TAG_UNCHANGED
694 MACH_PORT_NULL, /* owner remains unchanged */
695 VM_LEDGER_TAG_UNCHANGED, /* ledger-tag remains unchanged */
696 #else /* VM_LEDGER_TAG_UNCHANGED */
697 mach_task_self(),
698 VM_LEDGER_TAG_DEFAULT,
699 #endif /* VM_LEDGER_TAG_UNCHANGED */
700 0);
701 T_QUIET;
702 T_EXPECT_EQ(kr, KERN_SUCCESS,
703 "mach_memory_entry_ownership() error 0x%x (%s)",
704 kr, mach_error_string(kr));
705 /* ... should not change footprint */
706 get_ledger_info(&footprint_after, &pagetable_after);
707 footprint_expected = footprint_before;
708 footprint_expected += (pagetable_after - pagetable_before);
709 T_LOG("changing ownership from self to self "
710 "does not change phys_footprint");
711 T_EXPECT_EQ(footprint_after, footprint_expected,
712 "changed ownership of %lld dirty bytes: "
713 "footprint %lld -> %lld expected %lld delta %lld",
714 dirty_size, footprint_before, footprint_after,
715 footprint_expected, footprint_after - footprint_expected);
716
717 /* releasing the memory entry... */
718 kr = mach_port_deallocate(mach_task_self(), me_port);
719 T_QUIET;
720 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
721 kr, mach_error_string(kr));
722 /* ... should decrease footprint */
723 get_ledger_info(&footprint_after, &pagetable_after);
724 footprint_expected = footprint_before - dirty_size;
725 footprint_expected += (pagetable_after - pagetable_before);
726 T_LOG("releasing memory entry decreases phys_footprint");
727 T_EXPECT_EQ(footprint_after, footprint_expected,
728 "made volatile %lld dirty bytes: "
729 "footprint %lld -> %lld expected %lld delta %lld",
730 dirty_size, footprint_before, footprint_after,
731 footprint_expected, footprint_after - footprint_expected);
732 }
733
734 #if VM_LEDGER_TAG_UNCHANGED
735 T_DECL(phys_footprint_multiple_purgeable_ownership,
736 "phys_footprint for owned purgeable memory (MAP_MEM_VM_SHARE)",
737 T_META_NAMESPACE(TEST_VM_NAMESPACE),
738 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
739 {
740 uint64_t footprint_before, pagetable_before;
741 uint64_t footprint_after, pagetable_after;
742 uint64_t footprint_expected;
743 kern_return_t kr;
744 mach_vm_address_t pre_vm_addr, vm_addr;
745 mach_vm_size_t vm_size, dirty_size, me_size;
746 int state;
747 mach_port_t me_port;
748
749 /* pre-warm to account for page table expansion */
750 pre_vm_addr = pre_warm(MEM_SIZE);
751
752 /* allocating purgeable virtual memory... */
753 get_ledger_info(&footprint_before, &pagetable_before);
754 vm_addr = 0;
755 vm_size = MEM_SIZE;
756 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
757 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
758 T_QUIET;
759 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
760 kr, mach_error_string(kr));
761 T_QUIET;
762 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
763 /* ... should not change footprint */
764 get_ledger_info(&footprint_after, &pagetable_after);
765 footprint_expected = footprint_before;
766 footprint_expected += (pagetable_after - pagetable_before);
767 T_LOG("purgeable virtual allocation does not change phys_footprint");
768 T_EXPECT_EQ(footprint_after, footprint_expected,
769 "purgeable virtual allocation of %lld bytes: "
770 "footprint %lld -> %lld expected %lld delta %lld",
771 vm_size, footprint_before, footprint_after,
772 footprint_expected, footprint_after - footprint_expected);
773
774 /* touching memory... */
775 get_ledger_info(&footprint_before, &pagetable_before);
776 dirty_size = vm_size / 2;
777 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
778 /* ... should increase footprint */
779 get_ledger_info(&footprint_after, &pagetable_after);
780 footprint_expected = footprint_before + dirty_size;
781 footprint_expected += (pagetable_after - pagetable_before);
782 T_LOG("modifying anonymous memory increases phys_footprint");
783 T_EXPECT_EQ(footprint_after, footprint_expected,
784 "touched %lld bytes: "
785 "footprint %lld -> %lld expected %lld delta %lld",
786 dirty_size, footprint_before, footprint_after,
787 footprint_expected, footprint_after - footprint_expected);
788
789 /* making it volatile... */
790 get_ledger_info(&footprint_before, &pagetable_before);
791 state = VM_PURGABLE_VOLATILE;
792 T_QUIET;
793 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
794 vm_addr,
795 VM_PURGABLE_SET_STATE,
796 &state),
797 KERN_SUCCESS,
798 "vm_purgable_control(VOLATILE)");
799 T_QUIET;
800 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
801 "memory was non-volatile");
802 /* ... should decrease footprint */
803 get_ledger_info(&footprint_after, &pagetable_after);
804 footprint_expected = footprint_before - dirty_size;
805 footprint_expected += (pagetable_after - pagetable_before);
806 T_LOG("making volatile decreases phys_footprint");
807 T_EXPECT_EQ(footprint_after, footprint_expected,
808 "made volatile %lld dirty bytes: "
809 "footprint %lld -> %lld expected %lld delta %lld",
810 dirty_size, footprint_before, footprint_after,
811 footprint_expected, footprint_after - footprint_expected);
812
813 /* making it non-volatile... */
814 get_ledger_info(&footprint_before, &pagetable_before);
815 state = VM_PURGABLE_NONVOLATILE;
816 T_QUIET;
817 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
818 vm_addr,
819 VM_PURGABLE_SET_STATE,
820 &state),
821 KERN_SUCCESS,
822 "vm_purgable_control(NONVOLATILE)");
823 T_QUIET;
824 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
825 "memory was volatile");
826 /* ... should increase footprint */
827 get_ledger_info(&footprint_after, &pagetable_after);
828 footprint_expected = footprint_before + dirty_size;
829 footprint_expected += (pagetable_after - pagetable_before);
830 T_LOG("making non-volatile increases phys_footprint");
831 T_EXPECT_EQ(footprint_after, footprint_expected,
832 "made non-volatile %lld dirty bytes: "
833 "footprint %lld -> %lld expected %lld delta %lld",
834 dirty_size, footprint_before, footprint_after,
835 footprint_expected, footprint_after - footprint_expected);
836
837 /* making a memory entry... */
838 get_ledger_info(&footprint_before, &pagetable_before);
839 me_size = vm_size;
840 me_port = MACH_PORT_NULL;
841 kr = mach_make_memory_entry_64(mach_task_self(),
842 &me_size,
843 vm_addr,
844 MAP_MEM_VM_SHARE | VM_PROT_READ | VM_PROT_WRITE,
845 &me_port,
846 MACH_PORT_NULL);
847 T_QUIET;
848 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
849 kr, mach_error_string(kr));
850 T_QUIET;
851 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
852 /* ... should not change footprint */
853 get_ledger_info(&footprint_after, &pagetable_after);
854 footprint_expected = footprint_before;
855 footprint_expected += (pagetable_after - pagetable_before);
856 T_LOG("making a memory entry does not change phys_footprint");
857 T_EXPECT_EQ(footprint_after, footprint_expected,
858 "making a memory entry of %lld bytes: "
859 "footprint %lld -> %lld expected %lld delta %lld",
860 vm_size, footprint_before, footprint_after,
861 footprint_expected, footprint_after - footprint_expected);
862
863 /* deallocating memory while holding memory entry... */
864 get_ledger_info(&footprint_before, &pagetable_before);
865 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
866 T_QUIET;
867 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
868 kr, mach_error_string(kr));
869 /* ... should not change footprint */
870 get_ledger_info(&footprint_after, &pagetable_after);
871 footprint_expected = footprint_before;
872 footprint_expected += (pagetable_after - pagetable_before);
873 T_LOG("deallocating owned memory while holding memory entry "
874 "does not change phys_footprint");
875 T_EXPECT_EQ(footprint_after, footprint_expected,
876 "deallocated %lld dirty bytes: "
877 "footprint %lld -> %lld expected %lld delta %lld",
878 dirty_size, footprint_before, footprint_after,
879 footprint_expected, footprint_after - footprint_expected);
880
881 /* changing ownership of memory entry (from self to self)... */
882 get_ledger_info(&footprint_before, &pagetable_before);
883 kr = mach_memory_entry_ownership(me_port,
884 MACH_PORT_NULL, /* owner remains unchanged */
885 VM_LEDGER_TAG_UNCHANGED, /* ledger-tag remains unchanged */
886 0);
887 T_QUIET;
888 T_EXPECT_EQ(kr, KERN_SUCCESS,
889 "mach_memory_entry_ownership() error 0x%x (%s)",
890 kr, mach_error_string(kr));
891 /* ... should not change footprint */
892 get_ledger_info(&footprint_after, &pagetable_after);
893 footprint_expected = footprint_before;
894 footprint_expected += (pagetable_after - pagetable_before);
895 T_LOG("changing ownership from self to self "
896 "does not change phys_footprint");
897 T_EXPECT_EQ(footprint_after, footprint_expected,
898 "changed ownership of %lld dirty bytes: "
899 "footprint %lld -> %lld expected %lld delta %lld",
900 dirty_size, footprint_before, footprint_after,
901 footprint_expected, footprint_after - footprint_expected);
902
903 /* releasing the memory entry... */
904 kr = mach_port_deallocate(mach_task_self(), me_port);
905 T_QUIET;
906 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
907 kr, mach_error_string(kr));
908 /* ... should decrease footprint */
909 get_ledger_info(&footprint_after, &pagetable_after);
910 footprint_expected = footprint_before - dirty_size;
911 footprint_expected += (pagetable_after - pagetable_before);
912 T_LOG("releasing memory entry decreases phys_footprint");
913 T_EXPECT_EQ(footprint_after, footprint_expected,
914 "made volatile %lld dirty bytes: "
915 "footprint %lld -> %lld expected %lld delta %lld",
916 dirty_size, footprint_before, footprint_after,
917 footprint_expected, footprint_after - footprint_expected);
918 }
919 #endif /* VM_LEDGER_TAG_UNCHANGED */
920
921 #ifdef MAP_MEM_LEDGER_TAGGED
922 T_DECL(phys_footprint_ledger_purgeable_owned,
923 "phys_footprint for ledger-tagged purgeable memory ownership",
924 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
925 {
926 uint64_t footprint_before, pagetable_before;
927 uint64_t footprint_after, pagetable_after;
928 uint64_t footprint_expected;
929 kern_return_t kr;
930 mach_vm_address_t pre_vm_addr, vm_addr;
931 mach_vm_size_t vm_size, dirty_size, me_size;
932 int state;
933 mach_port_t me_port;
934
935 /* pre-warm to account for page table expansion */
936 pre_vm_addr = pre_warm(MEM_SIZE);
937
938 /* making a memory entry... */
939 get_ledger_info(&footprint_before, &pagetable_before);
940 vm_size = MEM_SIZE;
941 me_size = vm_size;
942 me_port = MACH_PORT_NULL;
943 kr = mach_make_memory_entry_64(mach_task_self(),
944 &me_size,
945 0,
946 (MAP_MEM_NAMED_CREATE |
947 MAP_MEM_LEDGER_TAGGED |
948 MAP_MEM_PURGABLE |
949 VM_PROT_READ | VM_PROT_WRITE),
950 &me_port,
951 MACH_PORT_NULL);
952 T_QUIET;
953 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
954 kr, mach_error_string(kr));
955 T_QUIET;
956 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
957 /* ... should not change footprint */
958 get_ledger_info(&footprint_after, &pagetable_after);
959 footprint_expected = footprint_before;
960 footprint_expected += (pagetable_after - pagetable_before);
961 T_LOG("making a memory entry does not change phys_footprint");
962 T_EXPECT_EQ(footprint_after, footprint_expected,
963 "making a memory entry of %lld bytes: "
964 "footprint %lld -> %lld expected %lld delta %lld",
965 vm_size, footprint_before, footprint_after,
966 footprint_expected, footprint_after - footprint_expected);
967
968 /* mapping ledger-tagged virtual memory... */
969 get_ledger_info(&footprint_before, &pagetable_before);
970 vm_addr = 0;
971 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
972 0, /* mask */
973 VM_FLAGS_ANYWHERE,
974 me_port,
975 0, /* offset */
976 FALSE, /* copy */
977 VM_PROT_READ | VM_PROT_WRITE,
978 VM_PROT_READ | VM_PROT_WRITE,
979 VM_INHERIT_DEFAULT);
980 T_QUIET;
981 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
982 kr, mach_error_string(kr));
983 T_QUIET;
984 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
985 /* ... should not change footprint */
986 get_ledger_info(&footprint_after, &pagetable_after);
987 footprint_expected = footprint_before;
988 footprint_expected += (pagetable_after - pagetable_before);
989 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
990 T_EXPECT_EQ(footprint_after, footprint_expected,
991 "ledger-tagged mapping of %lld bytes: "
992 "footprint %lld -> %lld expected %lld delta %lld",
993 vm_size, footprint_before, footprint_after,
994 footprint_expected, footprint_after - footprint_expected);
995
996 /* touching memory... */
997 get_ledger_info(&footprint_before, &pagetable_before);
998 dirty_size = vm_size / 2;
999 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1000 /* ... should increase footprint */
1001 get_ledger_info(&footprint_after, &pagetable_after);
1002 footprint_expected = footprint_before + dirty_size;
1003 footprint_expected += (pagetable_after - pagetable_before);
1004 T_LOG("modifying ledger-tagged memory increases phys_footprint");
1005 T_EXPECT_EQ(footprint_after, footprint_expected,
1006 "touched %lld bytes: "
1007 "footprint %lld -> %lld expected %lld delta %lld",
1008 dirty_size, footprint_before, footprint_after,
1009 footprint_expected, footprint_after - footprint_expected);
1010
1011 /* making it volatile... */
1012 get_ledger_info(&footprint_before, &pagetable_before);
1013 state = VM_PURGABLE_VOLATILE;
1014 T_QUIET;
1015 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
1016 vm_addr,
1017 VM_PURGABLE_SET_STATE,
1018 &state),
1019 KERN_SUCCESS,
1020 "vm_purgable_control(VOLATILE)");
1021 T_QUIET;
1022 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
1023 "memory was non-volatile");
1024 /* ... should decrease footprint */
1025 get_ledger_info(&footprint_after, &pagetable_after);
1026 footprint_expected = footprint_before - dirty_size;
1027 footprint_expected += (pagetable_after - pagetable_before);
1028 T_LOG("making volatile decreases phys_footprint");
1029 T_EXPECT_EQ(footprint_after, footprint_expected,
1030 "made volatile %lld dirty bytes: "
1031 "footprint %lld -> %lld expected %lld delta %lld",
1032 dirty_size, footprint_before, footprint_after,
1033 footprint_expected, footprint_after - footprint_expected);
1034
1035 /* making it non-volatile... */
1036 get_ledger_info(&footprint_before, &pagetable_before);
1037 state = VM_PURGABLE_NONVOLATILE;
1038 T_QUIET;
1039 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
1040 vm_addr,
1041 VM_PURGABLE_SET_STATE,
1042 &state),
1043 KERN_SUCCESS,
1044 "vm_purgable_control(NONVOLATILE)");
1045 T_QUIET;
1046 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
1047 "memory was volatile");
1048 /* ... should increase footprint */
1049 get_ledger_info(&footprint_after, &pagetable_after);
1050 footprint_expected = footprint_before + dirty_size;
1051 footprint_expected += (pagetable_after - pagetable_before);
1052 T_LOG("making non-volatile increases phys_footprint");
1053 T_EXPECT_EQ(footprint_after, footprint_expected,
1054 "made non-volatile %lld dirty bytes: "
1055 "footprint %lld -> %lld expected %lld delta %lld",
1056 dirty_size, footprint_before, footprint_after,
1057 footprint_expected, footprint_after - footprint_expected);
1058
1059 /* deallocating memory while holding memory entry... */
1060 get_ledger_info(&footprint_before, &pagetable_before);
1061 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1062 T_QUIET;
1063 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1064 kr, mach_error_string(kr));
1065 /* ... should not change footprint */
1066 get_ledger_info(&footprint_after, &pagetable_after);
1067 footprint_expected = footprint_before;
1068 footprint_expected += (pagetable_after - pagetable_before);
1069 T_LOG("deallocating owned memory while holding memory entry "
1070 "does not change phys_footprint");
1071 T_EXPECT_EQ(footprint_after, footprint_expected,
1072 "deallocated %lld dirty bytes: "
1073 "footprint %lld -> %lld expected %lld delta %lld",
1074 dirty_size, footprint_before, footprint_after,
1075 footprint_expected, footprint_after - footprint_expected);
1076
1077 /* releasing the memory entry... */
1078 kr = mach_port_deallocate(mach_task_self(), me_port);
1079 T_QUIET;
1080 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1081 kr, mach_error_string(kr));
1082 /* ... should decrease footprint */
1083 get_ledger_info(&footprint_after, &pagetable_after);
1084 footprint_expected = footprint_before - dirty_size;
1085 footprint_expected += (pagetable_after - pagetable_before);
1086 T_LOG("releasing memory entry decreases phys_footprint");
1087 T_EXPECT_EQ(footprint_after, footprint_expected,
1088 "made volatile %lld dirty bytes: "
1089 "footprint %lld -> %lld expected %lld delta %lld",
1090 dirty_size, footprint_before, footprint_after,
1091 footprint_expected, footprint_after - footprint_expected);
1092 }
1093
1094 T_DECL(phys_footprint_ledger_owned,
1095 "phys_footprint for ledger-tagged memory ownership",
1096 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
1097 {
1098 uint64_t footprint_before, pagetable_before;
1099 uint64_t footprint_after, pagetable_after;
1100 uint64_t footprint_expected;
1101 kern_return_t kr;
1102 mach_vm_address_t pre_vm_addr, vm_addr;
1103 mach_vm_size_t vm_size, dirty_size, me_size;
1104 mach_port_t me_port;
1105
1106 /* pre-warm to account for page table expansion */
1107 pre_vm_addr = pre_warm(MEM_SIZE);
1108
1109 /* making a memory entry... */
1110 get_ledger_info(&footprint_before, &pagetable_before);
1111 vm_size = MEM_SIZE;
1112 me_size = vm_size;
1113 me_port = MACH_PORT_NULL;
1114 kr = mach_make_memory_entry_64(mach_task_self(),
1115 &me_size,
1116 0,
1117 (MAP_MEM_NAMED_CREATE |
1118 MAP_MEM_LEDGER_TAGGED |
1119 VM_PROT_READ | VM_PROT_WRITE),
1120 &me_port,
1121 MACH_PORT_NULL);
1122 T_QUIET;
1123 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1124 kr, mach_error_string(kr));
1125 T_QUIET;
1126 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1127 /* ... should not change footprint */
1128 get_ledger_info(&footprint_after, &pagetable_after);
1129 footprint_expected = footprint_before;
1130 footprint_expected += (pagetable_after - pagetable_before);
1131 T_LOG("making a memory entry does not change phys_footprint");
1132 T_EXPECT_EQ(footprint_after, footprint_expected,
1133 "making a memory entry of %lld bytes: "
1134 "footprint %lld -> %lld expected %lld delta %lld",
1135 vm_size, footprint_before, footprint_after,
1136 footprint_expected, footprint_after - footprint_expected);
1137
1138 /* mapping ledger-tagged virtual memory... */
1139 get_ledger_info(&footprint_before, &pagetable_before);
1140 vm_addr = 0;
1141 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1142 0, /* mask */
1143 VM_FLAGS_ANYWHERE,
1144 me_port,
1145 0, /* offset */
1146 FALSE, /* copy */
1147 VM_PROT_READ | VM_PROT_WRITE,
1148 VM_PROT_READ | VM_PROT_WRITE,
1149 VM_INHERIT_DEFAULT);
1150 T_QUIET;
1151 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1152 kr, mach_error_string(kr));
1153 T_QUIET;
1154 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1155 /* ... should not change footprint */
1156 get_ledger_info(&footprint_after, &pagetable_after);
1157 footprint_expected = footprint_before;
1158 footprint_expected += (pagetable_after - pagetable_before);
1159 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1160 T_EXPECT_EQ(footprint_after, footprint_expected,
1161 "ledger-tagged mapping of %lld bytes: "
1162 "footprint %lld -> %lld expected %lld delta %lld",
1163 vm_size, footprint_before, footprint_after,
1164 footprint_expected, footprint_after - footprint_expected);
1165
1166 /* touching memory... */
1167 get_ledger_info(&footprint_before, &pagetable_before);
1168 dirty_size = vm_size / 2;
1169 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1170 /* ... should increase footprint */
1171 get_ledger_info(&footprint_after, &pagetable_after);
1172 footprint_expected = footprint_before + dirty_size;
1173 footprint_expected += (pagetable_after - pagetable_before);
1174 T_LOG("modifying ledger-tagged memory increases phys_footprint");
1175 T_EXPECT_EQ(footprint_after, footprint_expected,
1176 "touched %lld bytes: "
1177 "footprint %lld -> %lld expected %lld delta %lld",
1178 dirty_size, footprint_before, footprint_after,
1179 footprint_expected, footprint_after - footprint_expected);
1180
1181 /* deallocating memory while holding memory entry... */
1182 get_ledger_info(&footprint_before, &pagetable_before);
1183 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1184 T_QUIET;
1185 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1186 kr, mach_error_string(kr));
1187 /* ... should not change footprint */
1188 get_ledger_info(&footprint_after, &pagetable_after);
1189 footprint_expected = footprint_before;
1190 footprint_expected += (pagetable_after - pagetable_before);
1191 T_LOG("deallocating owned memory while holding memory entry "
1192 "does not change phys_footprint");
1193 T_EXPECT_EQ(footprint_after, footprint_expected,
1194 "deallocated %lld dirty bytes: "
1195 "footprint %lld -> %lld expected %lld delta %lld",
1196 dirty_size, footprint_before, footprint_after,
1197 footprint_expected, footprint_after - footprint_expected);
1198
1199 /* releasing the memory entry... */
1200 kr = mach_port_deallocate(mach_task_self(), me_port);
1201 T_QUIET;
1202 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1203 kr, mach_error_string(kr));
1204 /* ... should decrease footprint */
1205 get_ledger_info(&footprint_after, &pagetable_after);
1206 footprint_expected = footprint_before - dirty_size;
1207 footprint_expected += (pagetable_after - pagetable_before);
1208 T_LOG("releasing memory entry decreases phys_footprint");
1209 T_EXPECT_EQ(footprint_after, footprint_expected,
1210 "made volatile %lld dirty bytes: "
1211 "footprint %lld -> %lld expected %lld delta %lld",
1212 dirty_size, footprint_before, footprint_after,
1213 footprint_expected, footprint_after - footprint_expected);
1214 }
1215
1216 T_DECL(phys_footprint_no_footprint_for_debug,
1217 "phys_footprint for no_footprint_for_debug",
1218 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
1219 {
1220 uint64_t footprint_before, pagetable_before;
1221 uint64_t footprint_after, pagetable_after;
1222 uint64_t footprint_expected;
1223 kern_return_t kr;
1224 mach_vm_address_t pre_vm_addr, vm_addr;
1225 mach_vm_size_t vm_size, dirty_size, me_size;
1226 mach_port_t me_port;
1227 int new_value, ret;
1228
1229 /* pre-warm to account for page table expansion */
1230 pre_vm_addr = pre_warm(MEM_SIZE);
1231
1232 /* making a memory entry... */
1233 get_ledger_info(&footprint_before, &pagetable_before);
1234 vm_size = MEM_SIZE;
1235 me_size = vm_size;
1236 me_port = MACH_PORT_NULL;
1237 kr = mach_make_memory_entry_64(mach_task_self(),
1238 &me_size,
1239 0,
1240 (MAP_MEM_NAMED_CREATE |
1241 MAP_MEM_LEDGER_TAGGED |
1242 VM_PROT_READ | VM_PROT_WRITE),
1243 &me_port,
1244 MACH_PORT_NULL);
1245 T_QUIET;
1246 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1247 kr, mach_error_string(kr));
1248 T_QUIET;
1249 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1250 /* ... should not change footprint */
1251 get_ledger_info(&footprint_after, &pagetable_after);
1252 footprint_expected = footprint_before;
1253 footprint_expected += (pagetable_after - pagetable_before);
1254 T_LOG("making a memory entry does not change phys_footprint");
1255 T_EXPECT_EQ(footprint_after, footprint_expected,
1256 "making a memory entry of %lld bytes: "
1257 "footprint %lld -> %lld expected %lld delta %lld",
1258 vm_size, footprint_before, footprint_after,
1259 footprint_expected, footprint_after - footprint_expected);
1260
1261 /* pretending to call from the kernel should fail */
1262 kr = mach_memory_entry_ownership(me_port,
1263 mach_task_self(),
1264 VM_LEDGER_TAG_DEFAULT,
1265 VM_LEDGER_FLAG_FROM_KERNEL);
1266 T_QUIET;
1267 T_EXPECT_EQ(kr, KERN_INVALID_ARGUMENT,
1268 "mach_memory_entry_ownership(FROM_KERNEL) fails");
1269
1270 /* trying to hide debug memory from footprint while not allowed */
1271 kr = mach_memory_entry_ownership(me_port,
1272 mach_task_self(),
1273 VM_LEDGER_TAG_DEFAULT,
1274 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1275 T_QUIET;
1276 T_EXPECT_EQ(kr, KERN_NO_ACCESS,
1277 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) fails without sysctl");
1278
1279 /* let's get permission to hide debug memory */
1280 new_value = 1;
1281 ret = sysctlbyname("vm.task_no_footprint_for_debug", NULL, NULL, &new_value, sizeof(new_value));
1282 if (ret == -1 && errno == ENOENT) {
1283 T_SKIP("sysctlbyname(vm.task_no_footprint_for_debug) ENOENT");
1284 }
1285 T_QUIET;
1286 T_ASSERT_POSIX_SUCCESS(ret, "sysctlbyname(vm.task_no_footprint_for_debug)");
1287
1288 /* trying to hide debug memory from footprint while allowed */
1289 task_id_token_t token;
1290 kr = task_create_identity_token(mach_task_self(), &token);
1291 T_QUIET;
1292 T_ASSERT_EQ(kr, KERN_SUCCESS,
1293 "task_create_identity_token");
1294
1295 kr = mach_memory_entry_ownership(me_port,
1296 token, /* this time also try calling it with id token */
1297 VM_LEDGER_TAG_DEFAULT,
1298 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1299 T_QUIET;
1300 T_EXPECT_EQ(kr, KERN_SUCCESS,
1301 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) succeeds after sysctl");
1302 mach_port_deallocate(mach_task_self(), token);
1303
1304 /* mapping ledger-tagged virtual memory... */
1305 get_ledger_info(&footprint_before, &pagetable_before);
1306 vm_addr = 0;
1307 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1308 0, /* mask */
1309 VM_FLAGS_ANYWHERE,
1310 me_port,
1311 0, /* offset */
1312 FALSE, /* copy */
1313 VM_PROT_READ | VM_PROT_WRITE,
1314 VM_PROT_READ | VM_PROT_WRITE,
1315 VM_INHERIT_DEFAULT);
1316 T_QUIET;
1317 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1318 kr, mach_error_string(kr));
1319 T_QUIET;
1320 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1321 /* ... should not change footprint */
1322 get_ledger_info(&footprint_after, &pagetable_after);
1323 footprint_expected = footprint_before;
1324 footprint_expected += (pagetable_after - pagetable_before);
1325 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1326 T_EXPECT_EQ(footprint_after, footprint_expected,
1327 "ledger-tagged mapping of %lld bytes: "
1328 "footprint %lld -> %lld expected %lld delta %lld",
1329 vm_size, footprint_before, footprint_after,
1330 footprint_expected, footprint_after - footprint_expected);
1331
1332 /* touching memory... */
1333 get_ledger_info(&footprint_before, &pagetable_before);
1334 dirty_size = vm_size / 2;
1335 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1336 /* ... should not increase footprint */
1337 get_ledger_info(&footprint_after, &pagetable_after);
1338 footprint_expected = footprint_before;
1339 footprint_expected += (pagetable_after - pagetable_before);
1340 T_LOG("modifying no_footprint_for_debug memory does not increase phys_footprint");
1341 T_EXPECT_EQ(footprint_after, footprint_expected,
1342 "touched %lld bytes: "
1343 "footprint %lld -> %lld expected %lld delta %lld",
1344 dirty_size, footprint_before, footprint_after,
1345 footprint_expected, footprint_after - footprint_expected);
1346
1347 /* deallocating memory while holding memory entry... */
1348 get_ledger_info(&footprint_before, &pagetable_before);
1349 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1350 T_QUIET;
1351 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1352 kr, mach_error_string(kr));
1353 /* ... should not change footprint */
1354 get_ledger_info(&footprint_after, &pagetable_after);
1355 footprint_expected = footprint_before;
1356 footprint_expected += (pagetable_after - pagetable_before);
1357 T_LOG("deallocating owned memory while holding memory entry "
1358 "does not change phys_footprint");
1359 T_EXPECT_EQ(footprint_after, footprint_expected,
1360 "deallocated %lld dirty bytes: "
1361 "footprint %lld -> %lld expected %lld delta %lld",
1362 dirty_size, footprint_before, footprint_after,
1363 footprint_expected, footprint_after - footprint_expected);
1364
1365 /* releasing the memory entry... */
1366 kr = mach_port_deallocate(mach_task_self(), me_port);
1367 T_QUIET;
1368 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1369 kr, mach_error_string(kr));
1370 /* ... should not change footprint */
1371 get_ledger_info(&footprint_after, &pagetable_after);
1372 footprint_expected = footprint_before;
1373 footprint_expected += (pagetable_after - pagetable_before);
1374 T_LOG("releasing memory entry does not change phys_footprint");
1375 T_EXPECT_EQ(footprint_after, footprint_expected,
1376 "made volatile %lld dirty bytes: "
1377 "footprint %lld -> %lld expected %lld delta %lld",
1378 dirty_size, footprint_before, footprint_after,
1379 footprint_expected, footprint_after - footprint_expected);
1380 }
1381 #endif /* MAP_MEM_LEDGER_TAGGED */
1382
1383 /* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
1384 #include <CoreFoundation/CoreFoundation.h>
1385 #include <IOSurface/IOSurface.h>
1386 #include <IOSurface/IOSurfacePrivate.h>
1387 static size_t
bytes_per_element(uint32_t format)1388 bytes_per_element(uint32_t format)
1389 {
1390 size_t bpe = 0;
1391 switch (format) {
1392 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1393 bpe = 4;
1394 break;
1395 default:
1396 bpe = 0;
1397 break;
1398 }
1399 return bpe;
1400 }
1401 static size_t
bytes_per_pixel(uint32_t format)1402 bytes_per_pixel(uint32_t format)
1403 {
1404 size_t bpe = 0;
1405 switch (format) {
1406 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1407 bpe = 4;
1408 break;
1409 default:
1410 bpe = 0;
1411 break;
1412 }
1413 return bpe;
1414 }
1415 static inline size_t
roundSizeToMultiple(size_t size,size_t mult)1416 roundSizeToMultiple(size_t size, size_t mult)
1417 {
1418 return ((size + mult - 1) / mult) * mult;
1419 }
1420 static inline void
setIntValue(CFMutableDictionaryRef dict,const CFStringRef key,int value)1421 setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
1422 {
1423 CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
1424 CFDictionarySetValue(dict, key, number);
1425 CFRelease(number);
1426 }
1427 static inline void
setBoolValue(CFMutableDictionaryRef dict,const CFStringRef key,bool value)1428 setBoolValue(CFMutableDictionaryRef dict, const CFStringRef key, bool value)
1429 {
1430 CFDictionarySetValue(dict, key, value ? kCFBooleanTrue : kCFBooleanFalse);
1431 }
1432 typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
1433 static IOReturn
SurfaceApplyPlaneBlock(IOSurfaceRef surface,SurfacePlaneBlock block)1434 SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
1435 {
1436 if (surface == nil || block == nil) {
1437 return kIOReturnBadArgument;
1438 }
1439
1440 IOReturn result = kIOReturnSuccess;
1441 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1442
1443 if (planeCount == 0) {
1444 result = IOSurfaceLock(surface, 0, NULL);
1445 if (result != kIOReturnSuccess) {
1446 return result;
1447 }
1448
1449 void* base = IOSurfaceGetBaseAddress(surface);
1450 size_t rb = IOSurfaceGetBytesPerRow(surface);
1451 size_t w = IOSurfaceGetWidth(surface);
1452 size_t h = IOSurfaceGetHeight(surface);
1453
1454 if (base && rb && w && h) {
1455 block(base, 0, w, h, rb);
1456 }
1457
1458 IOSurfaceUnlock(surface, 0, NULL);
1459 } else if (planeCount == 2) {
1460 for (size_t i = 0; i < planeCount; i++) {
1461 result = IOSurfaceLock(surface, 0, NULL);
1462 if (result != kIOReturnSuccess) {
1463 return result;
1464 }
1465
1466 void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
1467 size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
1468 size_t w = IOSurfaceGetWidthOfPlane(surface, i);
1469 size_t h = IOSurfaceGetHeightOfPlane(surface, i);
1470
1471 if (base && rb && w && h) {
1472 block(base, i, w, h, rb);
1473 }
1474
1475 IOSurfaceUnlock(surface, 0, NULL);
1476 }
1477 }
1478 return result;
1479 }
1480 static void
ClearSurface(IOSurfaceRef surface)1481 ClearSurface(IOSurfaceRef surface)
1482 {
1483 const int zero = 0;
1484 (void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
1485 {
1486 if (i == 0) {
1487 memset(p, zero, rb * h);
1488 } else {
1489 memset(p, 128, rb * h);
1490 }
1491 });
1492 }
1493 static size_t
SurfaceGetMemorySize(IOSurfaceRef surface)1494 SurfaceGetMemorySize(IOSurfaceRef surface)
1495 {
1496 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1497
1498 if (planeCount == 0) {
1499 size_t rb = IOSurfaceGetBytesPerRow(surface);
1500 size_t h = IOSurfaceGetHeight(surface);
1501 return rb * h;
1502 } else if (planeCount == 2) {
1503 size_t rb0 = IOSurfaceGetBytesPerRowOfPlane(surface, 0);
1504 size_t h0 = IOSurfaceGetHeightOfPlane(surface, 0);
1505 size_t rb1 = IOSurfaceGetBytesPerRowOfPlane(surface, 1);
1506 size_t h1 = IOSurfaceGetHeightOfPlane(surface, 1);
1507 return rb0 * h0 + rb1 * h1;
1508 }
1509 return 0;
1510 }
1511 static IOSurfaceRef
CreateSurface(uint32_t pixelsWide,uint32_t pixelsHigh,uint32_t rowBytesAlignment,uint32_t fmt,bool purgeable,bool clear)1512 CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
1513 {
1514 IOSurfaceRef surface = nil;
1515
1516 if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
1517 return nil;
1518 }
1519
1520 size_t bpp = bytes_per_pixel(fmt);
1521 size_t bpe = bytes_per_element(fmt);
1522 if (bpp == 0 || bpe == 0) {
1523 return nil;
1524 }
1525
1526 size_t rowbytes = pixelsWide * bpp;
1527 if (rowBytesAlignment == 0) {
1528 rowBytesAlignment = 16;
1529 }
1530 rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);
1531
1532 CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1533 setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
1534 setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
1535 setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
1536 setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
1537 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1538 setBoolValue(props, kIOSurfaceNonPurgeable, !purgeable);
1539 #else /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1540 (void)purgeable;
1541 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1542 {
1543 if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
1544 setIntValue(props, kIOSurfaceElementWidth, 2);
1545 setIntValue(props, kIOSurfaceElementHeight, 1);
1546 }
1547 setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
1548 }
1549
1550 surface = IOSurfaceCreate(props);
1551
1552 if (clear) {
1553 ClearSurface(surface);
1554 }
1555
1556 CFRelease(props);
1557 return surface;
1558 }
1559 T_DECL(phys_footprint_purgeable_iokit,
1560 "phys_footprint for purgeable IOKit memory",
1561 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
1562 {
1563 uint64_t footprint_before, pagetable_before;
1564 uint64_t footprint_after, pagetable_after;
1565 uint64_t footprint_expected, footprint_delta_slop;
1566 int64_t footprint_delta;
1567 IOSurfaceRef init_surface, surface;
1568 uint32_t old_state;
1569 uint64_t surface_size;
1570
1571 T_SETUPBEGIN;
1572 footprint_delta_slop = 8 * vm_kernel_page_size;
1573 ledger_init();
1574 init_surface = CreateSurface(1024, 1024, 0, 32, true, true);
1575 IOSurfaceSetPurgeable(init_surface, kIOSurfacePurgeableVolatile, &old_state);
1576 IOSurfaceSetPurgeable(init_surface, kIOSurfacePurgeableNonVolatile, &old_state);
1577 T_SETUPEND;
1578
1579 surface_size = 1024 * 1024 * 4;
1580
1581 /* create IOsurface: footprint grows */
1582 get_ledger_info(&footprint_before, &pagetable_before);
1583 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1584 get_ledger_info(&footprint_after, &pagetable_after);
1585 if (legacy_footprint) {
1586 footprint_expected = footprint_before;
1587 footprint_expected += (pagetable_after - pagetable_before);
1588 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1589 T_LOG("LEGACY FOOTPRINT: creating purgeable IOSurface: no footprint impact");
1590 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1591 "create purgeable IOSurface %lld bytes: "
1592 "footprint %lld -> %lld expected %lld delta %lld",
1593 surface_size, footprint_before, footprint_after,
1594 footprint_expected, footprint_delta);
1595 } else {
1596 footprint_expected = footprint_before + surface_size;
1597 footprint_expected += (pagetable_after - pagetable_before);
1598 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1599 T_LOG("creating purgeable IOSurface increases phys_footprint");
1600 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1601 "create purgeable IOSurface %lld bytes: "
1602 "footprint %lld -> %lld expected %lld delta %lld",
1603 surface_size, footprint_before, footprint_after,
1604 footprint_expected, footprint_delta);
1605 }
1606
1607 /* make IOSurface volatile: footprint shrinks */
1608 get_ledger_info(&footprint_before, &pagetable_before);
1609 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1610 get_ledger_info(&footprint_after, &pagetable_after);
1611 if (legacy_footprint) {
1612 footprint_expected = footprint_before;
1613 footprint_expected += (pagetable_after - pagetable_before);
1614 T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
1615 T_EXPECT_EQ(footprint_after, footprint_expected,
1616 "volatile IOSurface %lld bytes: "
1617 "footprint %lld -> %lld expected %lld delta %lld",
1618 surface_size, footprint_before, footprint_after,
1619 footprint_expected, footprint_after - footprint_expected);
1620 } else {
1621 footprint_expected = footprint_before - surface_size;
1622 footprint_expected += (pagetable_after - pagetable_before);
1623 T_LOG("making IOSurface volatile decreases phys_footprint");
1624 T_EXPECT_EQ(footprint_after, footprint_expected,
1625 "made volatile %lld bytes: "
1626 "footprint %lld -> %lld expected %lld delta %lld",
1627 surface_size, footprint_before, footprint_after,
1628 footprint_expected, footprint_after - footprint_expected);
1629 }
1630
1631 /* make IOSurface non-volatile: footprint grows */
1632 get_ledger_info(&footprint_before, &pagetable_before);
1633 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1634 get_ledger_info(&footprint_after, &pagetable_after);
1635 if (legacy_footprint) {
1636 footprint_expected = footprint_before;
1637 footprint_expected += (pagetable_after - pagetable_before);
1638 T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
1639 T_EXPECT_EQ(footprint_after, footprint_expected,
1640 "non-volatile IOSurface %lld bytes: "
1641 "footprint %lld -> %lld expected %lld delta %lld",
1642 surface_size, footprint_before, footprint_after,
1643 footprint_expected, footprint_after - footprint_expected);
1644 } else {
1645 footprint_expected = footprint_before + surface_size;
1646 footprint_expected += (pagetable_after - pagetable_before);
1647 T_LOG("making IOSurface non-volatile increases phys_footprint");
1648 T_EXPECT_EQ(footprint_after, footprint_expected,
1649 "made non-volatile %lld bytes: "
1650 "footprint %lld -> %lld expected %lld delta %lld",
1651 surface_size, footprint_before, footprint_after,
1652 footprint_expected, footprint_after - footprint_expected);
1653 }
1654
1655 /* accessing IOSurface re-mapping: no footprint impact */
1656
1657 /* deallocating IOSurface re-mapping: no footprint impact */
1658
1659 /* release IOSurface: footprint shrinks */
1660 get_ledger_info(&footprint_before, &pagetable_before);
1661 CFRelease(surface);
1662 get_ledger_info(&footprint_after, &pagetable_after);
1663 if (legacy_footprint) {
1664 footprint_expected = footprint_before;
1665 footprint_expected += (pagetable_after - pagetable_before);
1666 T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
1667 T_EXPECT_EQ(footprint_after, footprint_expected,
1668 "releasing IOSurface %lld bytes: "
1669 "footprint %lld -> %lld expected %lld delta %lld",
1670 surface_size, footprint_before, footprint_after,
1671 footprint_expected, footprint_after - footprint_expected);
1672 } else {
1673 footprint_expected = footprint_before - surface_size;
1674 footprint_expected += (pagetable_after - pagetable_before);
1675 T_LOG("releasing IOSurface decreases phys_footprint");
1676 T_EXPECT_EQ(footprint_after, footprint_expected,
1677 "released IOSurface %lld bytes: "
1678 "footprint %lld -> %lld expected %lld delta %lld",
1679 surface_size, footprint_before, footprint_after,
1680 footprint_expected, footprint_after - footprint_expected);
1681 }
1682
1683 CFRelease(init_surface);
1684 }
1685
1686 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1687 T_DECL(phys_footprint_nonpurgeable_iokit,
1688 "phys_footprint for non-purgeable IOKit memory",
1689 T_META_LTEPHASE(LTE_POSTINIT), T_META_TAG_VM_PREFERRED)
1690 {
1691 uint64_t footprint_before, pagetable_before;
1692 uint64_t footprint_after, pagetable_after;
1693 uint64_t footprint_expected, footprint_delta_slop;
1694 int64_t footprint_delta;
1695 IOSurfaceRef init_surface, surface;
1696 uint64_t surface_size;
1697 void *map_base;
1698 size_t map_size;
1699 mach_vm_address_t remap_addr;
1700 kern_return_t kr;
1701 vm_prot_t cur_prot, max_prot;
1702 uint32_t old_state;
1703
1704
1705 T_SETUPBEGIN;
1706 ledger_init();
1707 init_surface = CreateSurface(1024, 1024, 0, 32, false, true);
1708 footprint_delta_slop = 8 * vm_kernel_page_size;
1709 T_SETUPEND;
1710
1711 surface_size = 1024 * 1024 * 4;
1712
1713 /* create IOsurface: footprint grows */
1714 get_ledger_info(&footprint_before, &pagetable_before);
1715 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1716 get_ledger_info(&footprint_after, &pagetable_after);
1717 footprint_expected = footprint_before + surface_size;
1718 footprint_expected += (pagetable_after - pagetable_before);
1719 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1720 T_LOG("creating non-purgeable IOSurface increases phys_footprint");
1721 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1722 "create non-purgeable IOSurface %lld bytes: "
1723 "footprint %lld -> %lld expected %lld delta %lld",
1724 surface_size, footprint_before, footprint_after,
1725 footprint_expected, footprint_delta);
1726
1727 /* make IOSurface volatile: fail and no footprint impact */
1728 get_ledger_info(&footprint_before, &pagetable_before);
1729 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1730 get_ledger_info(&footprint_after, &pagetable_after);
1731 footprint_expected = footprint_before;
1732 footprint_expected += (pagetable_after - pagetable_before);
1733 T_LOG("making non-purgeable IOSurface volatile: no footprint impact");
1734 T_EXPECT_EQ(footprint_after, footprint_expected,
1735 "made volatile %lld non-purgeable bytes: "
1736 "footprint %lld -> %lld expected %lld delta %lld",
1737 surface_size, footprint_before, footprint_after,
1738 footprint_expected, footprint_after - footprint_expected);
1739
1740 /* re-mapping IOSurface: no footprint impact */
1741 get_ledger_info(&footprint_before, &pagetable_before);
1742 map_base = IOSurfaceGetBaseAddress(surface);
1743 map_size = SurfaceGetMemorySize(surface);
1744 // T_EXPECT_EQ(map_size, surface_size, "map_size %lld surface_size %lld",
1745 // map_size, surface_size);
1746 remap_addr = 0;
1747 kr = mach_vm_remap(mach_task_self(),
1748 &remap_addr,
1749 (mach_vm_size_t)surface_size,
1750 0,
1751 VM_FLAGS_ANYWHERE,
1752 mach_task_self(),
1753 (mach_vm_address_t)map_base,
1754 FALSE, /* copy */
1755 &cur_prot,
1756 &max_prot,
1757 VM_INHERIT_DEFAULT);
1758 T_QUIET;
1759 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_remap() error 0x%x (%s)",
1760 kr, mach_error_string(kr));
1761 get_ledger_info(&footprint_after, &pagetable_after);
1762 footprint_expected = footprint_before;
1763 footprint_expected += (pagetable_after - pagetable_before);
1764 T_LOG("re-mapping IOSurface does not impact phys_footprint");
1765 T_EXPECT_EQ(footprint_after, footprint_expected,
1766 "remapping IOSurface %lld bytes: "
1767 "footprint %lld -> %lld expected %lld delta %lld",
1768 surface_size, footprint_before, footprint_after,
1769 footprint_expected, footprint_after - footprint_expected);
1770
1771 /* accessing IOSurface re-mapping: footprint grows */
1772 get_ledger_info(&footprint_before, &pagetable_before);
1773 memset((char *)(uintptr_t)remap_addr, 'p', (size_t)surface_size);
1774 get_ledger_info(&footprint_after, &pagetable_after);
1775 footprint_expected = footprint_before + surface_size;
1776 footprint_expected += (pagetable_after - pagetable_before);
1777 T_LOG("accessing re-mapped IOSurface grows phys_footprint");
1778 T_EXPECT_EQ(footprint_after, footprint_expected,
1779 "accessing remapped IOSurface %lld bytes: "
1780 "footprint %lld -> %lld expected %lld delta %lld",
1781 surface_size, footprint_before, footprint_after,
1782 footprint_expected, footprint_after - footprint_expected);
1783
1784 /* deallocating IOSurface re-mapping: footprint shrinks */
1785 get_ledger_info(&footprint_before, &pagetable_before);
1786 kr = mach_vm_deallocate(mach_task_self(),
1787 remap_addr,
1788 (mach_vm_size_t)surface_size);
1789 T_QUIET;
1790 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1791 kr, mach_error_string(kr));
1792 get_ledger_info(&footprint_after, &pagetable_after);
1793 footprint_expected = footprint_before - surface_size;
1794 footprint_expected += (pagetable_after - pagetable_before);
1795 T_LOG("deallocating re-mapping of IOSurface shrinks phys_footprint");
1796 T_EXPECT_EQ(footprint_after, footprint_expected,
1797 "deallocating remapped IOSurface %lld bytes: "
1798 "footprint %lld -> %lld expected %lld delta %lld",
1799 surface_size, footprint_before, footprint_after,
1800 footprint_expected, footprint_after - footprint_expected);
1801
1802 /* release IOSurface: footprint shrinks */
1803 get_ledger_info(&footprint_before, &pagetable_before);
1804 CFRelease(surface);
1805 get_ledger_info(&footprint_after, &pagetable_after);
1806 footprint_expected = footprint_before - surface_size;
1807 footprint_expected += (pagetable_after - pagetable_before);
1808 T_LOG("releasing IOSurface decreases phys_footprint");
1809 T_EXPECT_EQ(footprint_after, footprint_expected,
1810 "released IOSurface %lld bytes: "
1811 "footprint %lld -> %lld expected %lld delta %lld",
1812 surface_size, footprint_before, footprint_after,
1813 footprint_expected, footprint_after - footprint_expected);
1814
1815 CFRelease(init_surface);
1816 }
1817 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1818
1819 #if 0
1820 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
1821 T_DECL(legacy_footprint_entitled_set_and_query,
1822 "legacy footprint entitled is sticky with memlimit set properties and can be queried",
1823 T_META_BOOTARGS_SET("legacy_footprint_entitlement_mode=3"), T_META_TAG_VM_PREFERRED)
1824 {
1825 int ret;
1826 int32_t max_task_pmem = 0, legacy_footprint_bonus_mb = 0;
1827 size_t size_max_task_pmem = sizeof(max_task_pmem);
1828 size_t size_legacy_footprint_bonus_mb = sizeof(legacy_footprint_bonus_mb);
1829 memorystatus_memlimit_properties2_t mmprops;
1830
1831 /* Get the default limit */
1832 ret = sysctlbyname("kern.max_task_pmem", &max_task_pmem, &size_max_task_pmem, NULL, 0);
1833 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "call sysctlbyname to get max task physical memory.");
1834
1835 ret = sysctlbyname("kern.legacy_footprint_bonus_mb", &legacy_footprint_bonus_mb,
1836 &size_legacy_footprint_bonus_mb, NULL, 0);
1837 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "call sysctlbyname to get the bonus value.");
1838
1839 mmprops.v1.memlimit_active = -1;
1840 mmprops.v1.memlimit_inactive = -1;
1841 ret = memorystatus_control(MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES, getpid(), 0, &mmprops.v1, sizeof(mmprops.v1));
1842 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1843
1844 /* Check our memlimt */
1845 ret = memorystatus_control(MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES, getpid(), 0, &mmprops, sizeof(mmprops));
1846 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1847
1848 T_QUIET; T_ASSERT_EQ(mmprops.v1.memlimit_active, max_task_pmem + legacy_footprint_bonus_mb, "active limit");
1849 T_QUIET; T_ASSERT_EQ(mmprops.v1.memlimit_inactive, max_task_pmem + legacy_footprint_bonus_mb, "inactive limit");
1850
1851 /* Verify MEMORYSTATUS_CMD_CONVERT_MEMLIMIT_MB */
1852 ret = memorystatus_control(MEMORYSTATUS_CMD_CONVERT_MEMLIMIT_MB, getpid(), (uint32_t) -1, NULL, 0);
1853 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1854 T_QUIET; T_ASSERT_EQ(ret, max_task_pmem + legacy_footprint_bonus_mb, "got bonus adjustment");
1855 }
1856 #endif /* LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__) */
1857 #endif
1858