1 #include <darwintest.h>
2 #include <darwintest_utils.h>
3
4 #include <mach/mach_error.h>
5 #include <mach/mach_init.h>
6 #include <mach/memory_entry.h>
7 #include <mach/mach_port.h>
8 #include <mach/mach_vm.h>
9 #include <mach/memory_entry.h>
10 #include <mach/task.h>
11 #include <mach/task_info.h>
12 #include <mach/vm_map.h>
13
14 #include <sys/kern_memorystatus.h>
15 #include <sys/mman.h>
16 #include <sys/types.h>
17 #include <sys/sysctl.h>
18
19 #include <TargetConditionals.h>
20
21 #include <Kernel/kern/ledger.h>
22 extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);
23
24 T_GLOBAL_META(T_META_RUN_CONCURRENTLY(true));
25 static boolean_t legacy_footprint = FALSE;
26
27 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
28 #define TEST_VM_NAMESPACE "xnu.vm_legacy"
29 #else /* ENTITLED && __arm64__ */
30 #define TEST_VM_NAMESPACE "xnu.vm"
31 #endif /* ENTITLED && __arm64__ */
32
33 T_GLOBAL_META(
34 T_META_NAMESPACE(TEST_VM_NAMESPACE),
35 T_META_RADAR_COMPONENT_NAME("xnu"),
36 T_META_RADAR_COMPONENT_VERSION("VM"),
37 T_META_RUN_CONCURRENTLY(true));
38
39 #define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */
40
41 static int64_t ledger_count = -1;
42 static int footprint_index = -1;
43 static int pagetable_index = -1;
44 static struct ledger_entry_info *lei = NULL;
45
46 static void
ledger_init(void)47 ledger_init(void)
48 {
49 static int ledger_inited = 0;
50 struct ledger_info li;
51 struct ledger_template_info *templateInfo;
52 int64_t templateCnt;
53 int i;
54
55 if (ledger_inited) {
56 return;
57 }
58 ledger_inited = 1;
59
60 T_SETUPBEGIN;
61
62 legacy_footprint = FALSE;
63 #if LEGACY_FOOTPRINT_ENTITLED
64 int legacy_footprint_entitlement_mode;
65 size_t oldlen;
66 int ret;
67
68 T_QUIET;
69 T_WITH_ERRNO;
70 oldlen = sizeof(legacy_footprint_entitlement_mode);
71 ret = sysctlbyname("kern.legacy_footprint_entitlement_mode",
72 &legacy_footprint_entitlement_mode,
73 &oldlen,
74 NULL,
75 0);
76 if (ret == 0 && legacy_footprint_entitlement_mode == 2) {
77 legacy_footprint = TRUE;
78 }
79 #endif /* LEGACY_FOOTPRINT_ENTITLED */
80
81 T_QUIET;
82 T_WITH_ERRNO;
83 T_ASSERT_EQ(ledger(LEDGER_INFO,
84 (caddr_t)(uintptr_t)getpid(),
85 (caddr_t)&li,
86 NULL),
87 0,
88 "ledger(LEDGER_INFO)");
89
90 templateCnt = li.li_entries;
91 templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
92 T_QUIET;
93 T_WITH_ERRNO;
94 T_ASSERT_NE(templateInfo, NULL, "malloc()");
95
96 ledger_count = li.li_entries;
97 footprint_index = -1;
98 pagetable_index = -1;
99 T_QUIET;
100 T_WITH_ERRNO;
101 T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
102 (caddr_t)templateInfo,
103 (caddr_t)&templateCnt,
104 NULL),
105 0,
106 "ledger(LEDGER_TEMPLATE_INFO)");
107 for (i = 0; i < templateCnt; i++) {
108 if (!strncmp(templateInfo[i].lti_name,
109 "phys_footprint",
110 strlen("phys_footprint"))) {
111 footprint_index = i;
112 } else if (!strncmp(templateInfo[i].lti_name,
113 "page_table",
114 strlen("page_table"))) {
115 pagetable_index = i;
116 }
117 }
118 free(templateInfo);
119
120 lei = (struct ledger_entry_info *)
121 malloc((size_t)ledger_count * sizeof(*lei));
122 T_QUIET;
123 T_WITH_ERRNO;
124 T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");
125
126 T_QUIET;
127 T_ASSERT_NE(footprint_index, -1, "no footprint_index");
128 T_QUIET;
129 T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");
130
131 T_SETUPEND;
132 }
133
134 static void
get_ledger_info(uint64_t * phys_footprint,uint64_t * page_table)135 get_ledger_info(
136 uint64_t *phys_footprint,
137 uint64_t *page_table)
138 {
139 int64_t count;
140
141 count = ledger_count;
142 T_QUIET;
143 T_WITH_ERRNO;
144 T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
145 (caddr_t)(uintptr_t)getpid(),
146 (caddr_t)lei,
147 (caddr_t)&count),
148 0,
149 "ledger(LEDGER_ENTRY_INFO)");
150 T_QUIET;
151 T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
152 T_QUIET;
153 T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
154 if (phys_footprint) {
155 *phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
156 }
157 if (page_table) {
158 *page_table = (uint64_t)(lei[pagetable_index].lei_balance);
159 }
160 }
161
162 static mach_vm_address_t
pre_warm_with_tag(mach_vm_size_t vm_size,int tag)163 pre_warm_with_tag(
164 mach_vm_size_t vm_size,
165 int tag)
166 {
167 kern_return_t kr;
168 mach_vm_address_t vm_addr;
169 unsigned char BigBufOnStack[100 * 1024];
170 uint64_t footprint, page_table;
171
172 /* make sure ledgers are ready to be queried */
173 ledger_init();
174
175 T_SETUPBEGIN;
176
177 /*
178 * Touch a few pages ahead on the stack, to make
179 * sure we don't see a footprint increase due to
180 * an extra stack page later.
181 */
182 memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
183 T_QUIET;
184 T_EXPECT_EQ(BigBufOnStack[0], 0xb,
185 "BigBufOnStack[0] == 0x%x",
186 BigBufOnStack[0]);
187 T_QUIET;
188 T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
189 "BigBufOnStack[%lu] == 0x%x",
190 sizeof(BigBufOnStack),
191 BigBufOnStack[sizeof(BigBufOnStack) - 1]);
192
193 /*
194 * Pre-allocate, touch and then release the same amount
195 * of memory we'll be allocating later during the test,
196 * to account for any memory overhead (page tables, global
197 * variables, ...). Supplied VM tag should ensure memory
198 * is allocated in the appropriate VM range.
199 */
200 vm_addr = 0;
201 kr = mach_vm_allocate(mach_task_self(),
202 &vm_addr,
203 vm_size,
204 VM_FLAGS_ANYWHERE | VM_MAKE_TAG(tag));
205 T_QUIET;
206 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
207 vm_size, kr, mach_error_string(kr));
208 memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
209 kr = mach_vm_deallocate(mach_task_self(),
210 vm_addr,
211 vm_size);
212 T_QUIET;
213 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
214 kr, mach_error_string(kr));
215
216 /*
217 * Exercise the ledger code to make sure it's ready to run
218 * without any extra memory overhead later.
219 */
220 get_ledger_info(&footprint, &page_table);
221
222 T_SETUPEND;
223
224 /*
225 * Return the start of the virtual range we pre-warmed, so that the
226 * test can check that it's using the same range.
227 */
228 return vm_addr;
229 }
230
231 static mach_vm_address_t
pre_warm(mach_vm_size_t vm_size)232 pre_warm(
233 mach_vm_size_t vm_size)
234 {
235 /*
236 * Default to not tagging pre_warm memory.
237 */
238 return pre_warm_with_tag(vm_size, 0);
239 }
240
241 T_DECL(phys_footprint_anonymous,
242 "phys_footprint for anonymous memory",
243 T_META_LTEPHASE(LTE_POSTINIT))
244 {
245 uint64_t footprint_before, pagetable_before;
246 uint64_t footprint_after, pagetable_after;
247 uint64_t footprint_expected;
248 kern_return_t kr;
249 mach_vm_address_t pre_vm_addr, vm_addr;
250 mach_vm_size_t vm_size, dirty_size;
251
252 /* pre-warm to account for page table expansion */
253 pre_vm_addr = pre_warm(MEM_SIZE);
254
255 /* allocating virtual memory... */
256 get_ledger_info(&footprint_before, &pagetable_before);
257 vm_addr = 0;
258 vm_size = MEM_SIZE;
259 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
260 VM_FLAGS_ANYWHERE);
261 T_QUIET;
262 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
263 kr, mach_error_string(kr));
264 T_QUIET;
265 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
266 /* ... should not change footprint */
267 get_ledger_info(&footprint_after, &pagetable_after);
268 footprint_expected = footprint_before;
269 footprint_expected += (pagetable_after - pagetable_before);
270 T_LOG("virtual allocation does not change phys_footprint");
271 T_EXPECT_EQ(footprint_after, footprint_expected,
272 "virtual allocation of %lld bytes: "
273 "footprint %lld -> %lld expected %lld delta %lld",
274 vm_size, footprint_before, footprint_after,
275 footprint_expected, footprint_after - footprint_expected);
276
277 /* touching memory... */
278 get_ledger_info(&footprint_before, &pagetable_before);
279 dirty_size = vm_size / 2;
280 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
281 /* ... should increase footprint */
282 get_ledger_info(&footprint_after, &pagetable_after);
283 footprint_expected = footprint_before + dirty_size;
284 footprint_expected += (pagetable_after - pagetable_before);
285 T_LOG("modifying anonymous memory increases phys_footprint");
286 T_EXPECT_EQ(footprint_after, footprint_expected,
287 "touched %lld bytes: "
288 "footprint %lld -> %lld expected %lld delta %lld",
289 dirty_size, footprint_before, footprint_after,
290 footprint_expected, footprint_after - footprint_expected);
291
292 /* deallocating memory... */
293 get_ledger_info(&footprint_before, &pagetable_before);
294 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
295 T_QUIET;
296 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
297 kr, mach_error_string(kr));
298 /* ... should decrease footprint */
299 get_ledger_info(&footprint_after, &pagetable_after);
300 footprint_expected = footprint_before - dirty_size;
301 footprint_expected += (pagetable_after - pagetable_before);
302 T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
303 T_EXPECT_EQ(footprint_after, footprint_expected,
304 "deallocated %lld dirty bytes: "
305 "footprint %lld -> %lld expected %lld delta %lld",
306 dirty_size, footprint_before, footprint_after,
307 footprint_expected, footprint_after - footprint_expected);
308 }
309
310 #define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
311 #define TEMP_FILE_SIZE (1 * 1024 * 1024)
312
313 T_DECL(phys_footprint_file,
314 "phys_footprint for mapped file",
315 T_META_LTEPHASE(LTE_POSTINIT))
316 {
317 uint64_t footprint_before, pagetable_before;
318 uint64_t footprint_after, pagetable_after;
319 uint64_t footprint_expected;
320 mach_vm_address_t pre_vm_addr;
321 int fd;
322 char *map_addr;
323 size_t map_size, dirty_size;
324 ssize_t nbytes;
325 char tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
326 char *buf;
327 size_t buf_size;
328
329 T_SETUPBEGIN;
330 buf_size = TEMP_FILE_SIZE;
331 T_QUIET;
332 T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
333 "allocate %zu-byte buffer", buf_size);
334 memset(buf, 'f', buf_size);
335 T_WITH_ERRNO;
336 T_QUIET;
337 T_ASSERT_NOTNULL(mktemp(tmp_file_name),
338 "create temporary file name");
339 T_WITH_ERRNO;
340 T_QUIET;
341 T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
342 0,
343 "create temp file");
344 T_WITH_ERRNO;
345 T_QUIET;
346 T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
347 (ssize_t)buf_size,
348 "write %zu bytes", buf_size);
349 free(buf);
350 T_SETUPEND;
351
352 /* pre-warm to account for page table expansion */
353 #if TARGET_OS_OSX
354 pre_vm_addr = pre_warm_with_tag(TEMP_FILE_SIZE, VM_MEMORY_MALLOC);
355 #else
356 pre_vm_addr = pre_warm_with_tag(TEMP_FILE_SIZE, VM_MEMORY_MALLOC_TINY);
357 #endif
358
359 /* mapping a file does not impact footprint... */
360 get_ledger_info(&footprint_before, &pagetable_before);
361 map_size = TEMP_FILE_SIZE;
362 T_WITH_ERRNO;
363 T_QUIET;
364 T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
365 PROT_READ | PROT_WRITE,
366 MAP_FILE | MAP_SHARED, fd, 0),
367 "mmap()");
368 T_QUIET;
369 T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
370 "pre-warm mishap");
371 /* ... should not change footprint */
372 get_ledger_info(&footprint_after, &pagetable_after);
373 footprint_expected = footprint_before;
374 footprint_expected += (pagetable_after - pagetable_before);
375 T_LOG("mapping file does not change phys_footprint");
376 T_EXPECT_EQ(footprint_after, footprint_expected,
377 "mapping file with %zu bytes: "
378 "footprint %lld -> %lld expected %lld delta %lld",
379 map_size, footprint_before, footprint_after,
380 footprint_expected, footprint_after - footprint_expected);
381
382 /* touching file-backed memory... */
383 get_ledger_info(&footprint_before, &pagetable_before);
384 dirty_size = map_size / 2;
385 memset(map_addr, 'F', dirty_size);
386 /* ... should not impact footprint */
387 get_ledger_info(&footprint_after, &pagetable_after);
388 footprint_expected = footprint_before;
389 footprint_expected += (pagetable_after - pagetable_before);
390 T_LOG("modifying file-backed memory does not impact phys_footprint");
391 T_EXPECT_EQ(footprint_after, footprint_expected,
392 "touched %zu bytes: "
393 "footprint %lld -> %lld expected %lld delta %lld",
394 dirty_size, footprint_before, footprint_after,
395 footprint_expected, footprint_after - footprint_expected);
396
397 /* deallocating file-backed memory... */
398 get_ledger_info(&footprint_before, &pagetable_before);
399 T_WITH_ERRNO;
400 T_QUIET;
401 T_ASSERT_EQ(munmap(map_addr, map_size),
402 0,
403 "unmap file");
404 /* ... should not impact footprint */
405 get_ledger_info(&footprint_after, &pagetable_after);
406 footprint_expected = footprint_before;
407 footprint_expected += (pagetable_after - pagetable_before);
408 T_LOG("unmapping file-backed memory does not impact phys_footprint");
409 T_EXPECT_EQ(footprint_after, footprint_expected,
410 "unmapped %zu dirty bytes: "
411 "footprint %lld -> %lld expected %lld delta %lld",
412 dirty_size, footprint_before, footprint_after,
413 footprint_expected, footprint_after - footprint_expected);
414 }
415
416 T_DECL(phys_footprint_purgeable,
417 "phys_footprint for purgeable memory",
418 T_META_LTEPHASE(LTE_POSTINIT))
419 {
420 uint64_t footprint_before, pagetable_before;
421 uint64_t footprint_after, pagetable_after;
422 uint64_t footprint_expected;
423 kern_return_t kr;
424 mach_vm_address_t pre_vm_addr, vm_addr;
425 mach_vm_size_t vm_size, dirty_size;
426 int state;
427
428 /* pre-warm to account for page table expansion */
429 pre_vm_addr = pre_warm(MEM_SIZE);
430
431 /* allocating purgeable virtual memory... */
432 get_ledger_info(&footprint_before, &pagetable_before);
433 vm_addr = 0;
434 vm_size = MEM_SIZE;
435 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
436 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
437 T_QUIET;
438 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
439 kr, mach_error_string(kr));
440 T_QUIET;
441 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
442 /* ... should not change footprint */
443 get_ledger_info(&footprint_after, &pagetable_after);
444 footprint_expected = footprint_before;
445 footprint_expected += (pagetable_after - pagetable_before);
446 T_LOG("purgeable virtual allocation does not change phys_footprint");
447 T_EXPECT_EQ(footprint_after, footprint_expected,
448 "purgeable virtual allocation of %lld bytes: "
449 "footprint %lld -> %lld expected %lld delta %lld",
450 vm_size, footprint_before, footprint_after,
451 footprint_expected, footprint_after - footprint_expected);
452
453 /* touching memory... */
454 get_ledger_info(&footprint_before, &pagetable_before);
455 dirty_size = vm_size / 2;
456 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
457 /* ... should increase footprint */
458 get_ledger_info(&footprint_after, &pagetable_after);
459 footprint_expected = footprint_before + dirty_size;
460 footprint_expected += (pagetable_after - pagetable_before);
461 T_LOG("modifying anonymous memory increases phys_footprint");
462 T_EXPECT_EQ(footprint_after, footprint_expected,
463 "touched %lld bytes: "
464 "footprint %lld -> %lld expected %lld delta %lld",
465 dirty_size, footprint_before, footprint_after,
466 footprint_expected, footprint_after - footprint_expected);
467
468 /* making it volatile... */
469 get_ledger_info(&footprint_before, &pagetable_before);
470 state = VM_PURGABLE_VOLATILE;
471 T_QUIET;
472 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
473 vm_addr,
474 VM_PURGABLE_SET_STATE,
475 &state),
476 KERN_SUCCESS,
477 "vm_purgable_control(VOLATILE)");
478 T_QUIET;
479 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
480 "memory was non-volatile");
481 /* ... should decrease footprint */
482 get_ledger_info(&footprint_after, &pagetable_after);
483 footprint_expected = footprint_before - dirty_size;
484 footprint_expected += (pagetable_after - pagetable_before);
485 T_LOG("making volatile decreases phys_footprint");
486 T_EXPECT_EQ(footprint_after, footprint_expected,
487 "made volatile %lld dirty bytes: "
488 "footprint %lld -> %lld expected %lld delta %lld",
489 dirty_size, footprint_before, footprint_after,
490 footprint_expected, footprint_after - footprint_expected);
491
492 /* making it non-volatile... */
493 get_ledger_info(&footprint_before, &pagetable_before);
494 state = VM_PURGABLE_NONVOLATILE;
495 T_QUIET;
496 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
497 vm_addr,
498 VM_PURGABLE_SET_STATE,
499 &state),
500 KERN_SUCCESS,
501 "vm_purgable_control(NONVOLATILE)");
502 T_QUIET;
503 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
504 "memory was volatile");
505 /* ... should increase footprint */
506 get_ledger_info(&footprint_after, &pagetable_after);
507 footprint_expected = footprint_before + dirty_size;
508 footprint_expected += (pagetable_after - pagetable_before);
509 T_LOG("making non-volatile increases phys_footprint");
510 T_EXPECT_EQ(footprint_after, footprint_expected,
511 "made non-volatile %lld dirty bytes: "
512 "footprint %lld -> %lld expected %lld delta %lld",
513 dirty_size, footprint_before, footprint_after,
514 footprint_expected, footprint_after - footprint_expected);
515
516 /* deallocating memory... */
517 get_ledger_info(&footprint_before, &pagetable_before);
518 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
519 T_QUIET;
520 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
521 kr, mach_error_string(kr));
522 /* ... should decrease footprint */
523 get_ledger_info(&footprint_after, &pagetable_after);
524 footprint_expected = footprint_before - dirty_size;
525 footprint_expected += (pagetable_after - pagetable_before);
526 T_LOG("deallocating memory decreases phys_footprint");
527 T_EXPECT_EQ(footprint_after, footprint_expected,
528 "deallocated %lld dirty bytes: "
529 "footprint %lld -> %lld expected %lld delta %lld",
530 dirty_size, footprint_before, footprint_after,
531 footprint_expected, footprint_after - footprint_expected);
532 }
533
534 T_DECL(phys_footprint_purgeable_ownership,
535 "phys_footprint for owned purgeable memory",
536 T_META_LTEPHASE(LTE_POSTINIT))
537 {
538 uint64_t footprint_before, pagetable_before;
539 uint64_t footprint_after, pagetable_after;
540 uint64_t footprint_expected;
541 kern_return_t kr;
542 mach_vm_address_t pre_vm_addr, vm_addr;
543 mach_vm_size_t vm_size, dirty_size, me_size;
544 int state;
545 mach_port_t me_port;
546
547 /* pre-warm to account for page table expansion */
548 pre_vm_addr = pre_warm(MEM_SIZE);
549
550 /* allocating purgeable virtual memory... */
551 get_ledger_info(&footprint_before, &pagetable_before);
552 vm_addr = 0;
553 vm_size = MEM_SIZE;
554 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
555 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
556 T_QUIET;
557 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
558 kr, mach_error_string(kr));
559 T_QUIET;
560 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
561 /* ... should not change footprint */
562 get_ledger_info(&footprint_after, &pagetable_after);
563 footprint_expected = footprint_before;
564 footprint_expected += (pagetable_after - pagetable_before);
565 T_LOG("purgeable virtual allocation does not change phys_footprint");
566 T_EXPECT_EQ(footprint_after, footprint_expected,
567 "purgeable virtual allocation of %lld bytes: "
568 "footprint %lld -> %lld expected %lld delta %lld",
569 vm_size, footprint_before, footprint_after,
570 footprint_expected, footprint_after - footprint_expected);
571
572 /* touching memory... */
573 get_ledger_info(&footprint_before, &pagetable_before);
574 dirty_size = vm_size / 2;
575 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
576 /* ... should increase footprint */
577 get_ledger_info(&footprint_after, &pagetable_after);
578 footprint_expected = footprint_before + dirty_size;
579 footprint_expected += (pagetable_after - pagetable_before);
580 T_LOG("modifying anonymous memory increases phys_footprint");
581 T_EXPECT_EQ(footprint_after, footprint_expected,
582 "touched %lld bytes: "
583 "footprint %lld -> %lld expected %lld delta %lld",
584 dirty_size, footprint_before, footprint_after,
585 footprint_expected, footprint_after - footprint_expected);
586
587 /* making it volatile... */
588 get_ledger_info(&footprint_before, &pagetable_before);
589 state = VM_PURGABLE_VOLATILE;
590 T_QUIET;
591 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
592 vm_addr,
593 VM_PURGABLE_SET_STATE,
594 &state),
595 KERN_SUCCESS,
596 "vm_purgable_control(VOLATILE)");
597 T_QUIET;
598 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
599 "memory was non-volatile");
600 /* ... should decrease footprint */
601 get_ledger_info(&footprint_after, &pagetable_after);
602 footprint_expected = footprint_before - dirty_size;
603 footprint_expected += (pagetable_after - pagetable_before);
604 T_LOG("making volatile decreases phys_footprint");
605 T_EXPECT_EQ(footprint_after, footprint_expected,
606 "made volatile %lld dirty bytes: "
607 "footprint %lld -> %lld expected %lld delta %lld",
608 dirty_size, footprint_before, footprint_after,
609 footprint_expected, footprint_after - footprint_expected);
610
611 /* making it non-volatile... */
612 get_ledger_info(&footprint_before, &pagetable_before);
613 state = VM_PURGABLE_NONVOLATILE;
614 T_QUIET;
615 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
616 vm_addr,
617 VM_PURGABLE_SET_STATE,
618 &state),
619 KERN_SUCCESS,
620 "vm_purgable_control(NONVOLATILE)");
621 T_QUIET;
622 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
623 "memory was volatile");
624 /* ... should increase footprint */
625 get_ledger_info(&footprint_after, &pagetable_after);
626 footprint_expected = footprint_before + dirty_size;
627 footprint_expected += (pagetable_after - pagetable_before);
628 T_LOG("making non-volatile increases phys_footprint");
629 T_EXPECT_EQ(footprint_after, footprint_expected,
630 "made non-volatile %lld dirty bytes: "
631 "footprint %lld -> %lld expected %lld delta %lld",
632 dirty_size, footprint_before, footprint_after,
633 footprint_expected, footprint_after - footprint_expected);
634
635 /* making a memory entry... */
636 get_ledger_info(&footprint_before, &pagetable_before);
637 me_size = vm_size;
638 me_port = MACH_PORT_NULL;
639 kr = mach_make_memory_entry_64(mach_task_self(),
640 &me_size,
641 vm_addr,
642 VM_PROT_READ | VM_PROT_WRITE,
643 &me_port,
644 MACH_PORT_NULL);
645 T_QUIET;
646 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
647 kr, mach_error_string(kr));
648 T_QUIET;
649 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
650 /* ... should not change footprint */
651 get_ledger_info(&footprint_after, &pagetable_after);
652 footprint_expected = footprint_before;
653 footprint_expected += (pagetable_after - pagetable_before);
654 T_LOG("making a memory entry does not change phys_footprint");
655 T_EXPECT_EQ(footprint_after, footprint_expected,
656 "making a memory entry of %lld bytes: "
657 "footprint %lld -> %lld expected %lld delta %lld",
658 vm_size, footprint_before, footprint_after,
659 footprint_expected, footprint_after - footprint_expected);
660
661 /* deallocating memory while holding memory entry... */
662 get_ledger_info(&footprint_before, &pagetable_before);
663 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
664 T_QUIET;
665 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
666 kr, mach_error_string(kr));
667 /* ... should not change footprint */
668 get_ledger_info(&footprint_after, &pagetable_after);
669 footprint_expected = footprint_before;
670 footprint_expected += (pagetable_after - pagetable_before);
671 T_LOG("deallocating owned memory while holding memory entry "
672 "does not change phys_footprint");
673 T_EXPECT_EQ(footprint_after, footprint_expected,
674 "deallocated %lld dirty bytes: "
675 "footprint %lld -> %lld expected %lld delta %lld",
676 dirty_size, footprint_before, footprint_after,
677 footprint_expected, footprint_after - footprint_expected);
678
679 get_ledger_info(&footprint_after, &pagetable_after);
680 footprint_expected = footprint_before;
681 footprint_expected += (pagetable_after - pagetable_before);
682 T_LOG("deallocating owned memory while holding memory entry "
683 "does not change phys_footprint");
684 T_EXPECT_EQ(footprint_after, footprint_expected,
685 "deallocated %lld dirty bytes: "
686 "footprint %lld -> %lld expected %lld delta %lld",
687 dirty_size, footprint_before, footprint_after,
688 footprint_expected, footprint_after - footprint_expected);
689
690 /* changing ownership of memory entry (from self to self)... */
691 get_ledger_info(&footprint_before, &pagetable_before);
692 kr = mach_memory_entry_ownership(me_port,
693 #if VM_LEDGER_TAG_UNCHANGED
694 MACH_PORT_NULL, /* owner remains unchanged */
695 VM_LEDGER_TAG_UNCHANGED, /* ledger-tag remains unchanged */
696 #else /* VM_LEDGER_TAG_UNCHANGED */
697 mach_task_self(),
698 VM_LEDGER_TAG_DEFAULT,
699 #endif /* VM_LEDGER_TAG_UNCHANGED */
700 0);
701 T_QUIET;
702 T_EXPECT_EQ(kr, KERN_SUCCESS,
703 "mach_memory_entry_ownership() error 0x%x (%s)",
704 kr, mach_error_string(kr));
705 /* ... should not change footprint */
706 get_ledger_info(&footprint_after, &pagetable_after);
707 footprint_expected = footprint_before;
708 footprint_expected += (pagetable_after - pagetable_before);
709 T_LOG("changing ownership from self to self "
710 "does not change phys_footprint");
711 T_EXPECT_EQ(footprint_after, footprint_expected,
712 "changed ownership of %lld dirty bytes: "
713 "footprint %lld -> %lld expected %lld delta %lld",
714 dirty_size, footprint_before, footprint_after,
715 footprint_expected, footprint_after - footprint_expected);
716
717 /* releasing the memory entry... */
718 kr = mach_port_deallocate(mach_task_self(), me_port);
719 T_QUIET;
720 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
721 kr, mach_error_string(kr));
722 /* ... should decrease footprint */
723 get_ledger_info(&footprint_after, &pagetable_after);
724 footprint_expected = footprint_before - dirty_size;
725 footprint_expected += (pagetable_after - pagetable_before);
726 T_LOG("releasing memory entry decreases phys_footprint");
727 T_EXPECT_EQ(footprint_after, footprint_expected,
728 "made volatile %lld dirty bytes: "
729 "footprint %lld -> %lld expected %lld delta %lld",
730 dirty_size, footprint_before, footprint_after,
731 footprint_expected, footprint_after - footprint_expected);
732 }
733
734 #if VM_LEDGER_TAG_UNCHANGED
735 T_DECL(phys_footprint_multiple_purgeable_ownership,
736 "phys_footprint for owned purgeable memory (MAP_MEM_VM_SHARE)",
737 T_META_NAMESPACE(TEST_VM_NAMESPACE),
738 T_META_LTEPHASE(LTE_POSTINIT))
739 {
740 uint64_t footprint_before, pagetable_before;
741 uint64_t footprint_after, pagetable_after;
742 uint64_t footprint_expected;
743 kern_return_t kr;
744 mach_vm_address_t pre_vm_addr, vm_addr;
745 mach_vm_size_t vm_size, dirty_size, me_size;
746 int state;
747 mach_port_t me_port;
748
749 /* pre-warm to account for page table expansion */
750 pre_vm_addr = pre_warm(MEM_SIZE);
751
752 /* allocating purgeable virtual memory... */
753 get_ledger_info(&footprint_before, &pagetable_before);
754 vm_addr = 0;
755 vm_size = MEM_SIZE;
756 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
757 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
758 T_QUIET;
759 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
760 kr, mach_error_string(kr));
761 T_QUIET;
762 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
763 /* ... should not change footprint */
764 get_ledger_info(&footprint_after, &pagetable_after);
765 footprint_expected = footprint_before;
766 footprint_expected += (pagetable_after - pagetable_before);
767 T_LOG("purgeable virtual allocation does not change phys_footprint");
768 T_EXPECT_EQ(footprint_after, footprint_expected,
769 "purgeable virtual allocation of %lld bytes: "
770 "footprint %lld -> %lld expected %lld delta %lld",
771 vm_size, footprint_before, footprint_after,
772 footprint_expected, footprint_after - footprint_expected);
773
774 /* touching memory... */
775 get_ledger_info(&footprint_before, &pagetable_before);
776 dirty_size = vm_size / 2;
777 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
778 /* ... should increase footprint */
779 get_ledger_info(&footprint_after, &pagetable_after);
780 footprint_expected = footprint_before + dirty_size;
781 footprint_expected += (pagetable_after - pagetable_before);
782 T_LOG("modifying anonymous memory increases phys_footprint");
783 T_EXPECT_EQ(footprint_after, footprint_expected,
784 "touched %lld bytes: "
785 "footprint %lld -> %lld expected %lld delta %lld",
786 dirty_size, footprint_before, footprint_after,
787 footprint_expected, footprint_after - footprint_expected);
788
789 /* making it volatile... */
790 get_ledger_info(&footprint_before, &pagetable_before);
791 state = VM_PURGABLE_VOLATILE;
792 T_QUIET;
793 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
794 vm_addr,
795 VM_PURGABLE_SET_STATE,
796 &state),
797 KERN_SUCCESS,
798 "vm_purgable_control(VOLATILE)");
799 T_QUIET;
800 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
801 "memory was non-volatile");
802 /* ... should decrease footprint */
803 get_ledger_info(&footprint_after, &pagetable_after);
804 footprint_expected = footprint_before - dirty_size;
805 footprint_expected += (pagetable_after - pagetable_before);
806 T_LOG("making volatile decreases phys_footprint");
807 T_EXPECT_EQ(footprint_after, footprint_expected,
808 "made volatile %lld dirty bytes: "
809 "footprint %lld -> %lld expected %lld delta %lld",
810 dirty_size, footprint_before, footprint_after,
811 footprint_expected, footprint_after - footprint_expected);
812
813 /* making it non-volatile... */
814 get_ledger_info(&footprint_before, &pagetable_before);
815 state = VM_PURGABLE_NONVOLATILE;
816 T_QUIET;
817 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
818 vm_addr,
819 VM_PURGABLE_SET_STATE,
820 &state),
821 KERN_SUCCESS,
822 "vm_purgable_control(NONVOLATILE)");
823 T_QUIET;
824 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
825 "memory was volatile");
826 /* ... should increase footprint */
827 get_ledger_info(&footprint_after, &pagetable_after);
828 footprint_expected = footprint_before + dirty_size;
829 footprint_expected += (pagetable_after - pagetable_before);
830 T_LOG("making non-volatile increases phys_footprint");
831 T_EXPECT_EQ(footprint_after, footprint_expected,
832 "made non-volatile %lld dirty bytes: "
833 "footprint %lld -> %lld expected %lld delta %lld",
834 dirty_size, footprint_before, footprint_after,
835 footprint_expected, footprint_after - footprint_expected);
836
837 /* making a memory entry... */
838 get_ledger_info(&footprint_before, &pagetable_before);
839 me_size = vm_size;
840 me_port = MACH_PORT_NULL;
841 kr = mach_make_memory_entry_64(mach_task_self(),
842 &me_size,
843 vm_addr,
844 MAP_MEM_VM_SHARE | VM_PROT_READ | VM_PROT_WRITE,
845 &me_port,
846 MACH_PORT_NULL);
847 T_QUIET;
848 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
849 kr, mach_error_string(kr));
850 T_QUIET;
851 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
852 /* ... should not change footprint */
853 get_ledger_info(&footprint_after, &pagetable_after);
854 footprint_expected = footprint_before;
855 footprint_expected += (pagetable_after - pagetable_before);
856 T_LOG("making a memory entry does not change phys_footprint");
857 T_EXPECT_EQ(footprint_after, footprint_expected,
858 "making a memory entry of %lld bytes: "
859 "footprint %lld -> %lld expected %lld delta %lld",
860 vm_size, footprint_before, footprint_after,
861 footprint_expected, footprint_after - footprint_expected);
862
863 /* deallocating memory while holding memory entry... */
864 get_ledger_info(&footprint_before, &pagetable_before);
865 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
866 T_QUIET;
867 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
868 kr, mach_error_string(kr));
869 /* ... should not change footprint */
870 get_ledger_info(&footprint_after, &pagetable_after);
871 footprint_expected = footprint_before;
872 footprint_expected += (pagetable_after - pagetable_before);
873 T_LOG("deallocating owned memory while holding memory entry "
874 "does not change phys_footprint");
875 T_EXPECT_EQ(footprint_after, footprint_expected,
876 "deallocated %lld dirty bytes: "
877 "footprint %lld -> %lld expected %lld delta %lld",
878 dirty_size, footprint_before, footprint_after,
879 footprint_expected, footprint_after - footprint_expected);
880
881 /* changing ownership of memory entry (from self to self)... */
882 get_ledger_info(&footprint_before, &pagetable_before);
883 kr = mach_memory_entry_ownership(me_port,
884 MACH_PORT_NULL, /* owner remains unchanged */
885 VM_LEDGER_TAG_UNCHANGED, /* ledger-tag remains unchanged */
886 0);
887 T_QUIET;
888 T_EXPECT_EQ(kr, KERN_SUCCESS,
889 "mach_memory_entry_ownership() error 0x%x (%s)",
890 kr, mach_error_string(kr));
891 /* ... should not change footprint */
892 get_ledger_info(&footprint_after, &pagetable_after);
893 footprint_expected = footprint_before;
894 footprint_expected += (pagetable_after - pagetable_before);
895 T_LOG("changing ownership from self to self "
896 "does not change phys_footprint");
897 T_EXPECT_EQ(footprint_after, footprint_expected,
898 "changed ownership of %lld dirty bytes: "
899 "footprint %lld -> %lld expected %lld delta %lld",
900 dirty_size, footprint_before, footprint_after,
901 footprint_expected, footprint_after - footprint_expected);
902
903 /* releasing the memory entry... */
904 kr = mach_port_deallocate(mach_task_self(), me_port);
905 T_QUIET;
906 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
907 kr, mach_error_string(kr));
908 /* ... should decrease footprint */
909 get_ledger_info(&footprint_after, &pagetable_after);
910 footprint_expected = footprint_before - dirty_size;
911 footprint_expected += (pagetable_after - pagetable_before);
912 T_LOG("releasing memory entry decreases phys_footprint");
913 T_EXPECT_EQ(footprint_after, footprint_expected,
914 "made volatile %lld dirty bytes: "
915 "footprint %lld -> %lld expected %lld delta %lld",
916 dirty_size, footprint_before, footprint_after,
917 footprint_expected, footprint_after - footprint_expected);
918 }
919 #endif /* VM_LEDGER_TAG_UNCHANGED */
920
921 #ifdef MAP_MEM_LEDGER_TAGGED
922 T_DECL(phys_footprint_ledger_purgeable_owned,
923 "phys_footprint for ledger-tagged purgeable memory ownership",
924 T_META_LTEPHASE(LTE_POSTINIT))
925 {
926 uint64_t footprint_before, pagetable_before;
927 uint64_t footprint_after, pagetable_after;
928 uint64_t footprint_expected;
929 kern_return_t kr;
930 mach_vm_address_t pre_vm_addr, vm_addr;
931 mach_vm_size_t vm_size, dirty_size, me_size;
932 int state;
933 mach_port_t me_port;
934
935 /* pre-warm to account for page table expansion */
936 pre_vm_addr = pre_warm(MEM_SIZE);
937
938 /* making a memory entry... */
939 get_ledger_info(&footprint_before, &pagetable_before);
940 vm_size = MEM_SIZE;
941 me_size = vm_size;
942 me_port = MACH_PORT_NULL;
943 kr = mach_make_memory_entry_64(mach_task_self(),
944 &me_size,
945 0,
946 (MAP_MEM_NAMED_CREATE |
947 MAP_MEM_LEDGER_TAGGED |
948 MAP_MEM_PURGABLE |
949 VM_PROT_READ | VM_PROT_WRITE),
950 &me_port,
951 MACH_PORT_NULL);
952 T_QUIET;
953 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
954 kr, mach_error_string(kr));
955 T_QUIET;
956 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
957 /* ... should not change footprint */
958 get_ledger_info(&footprint_after, &pagetable_after);
959 footprint_expected = footprint_before;
960 footprint_expected += (pagetable_after - pagetable_before);
961 T_LOG("making a memory entry does not change phys_footprint");
962 T_EXPECT_EQ(footprint_after, footprint_expected,
963 "making a memory entry of %lld bytes: "
964 "footprint %lld -> %lld expected %lld delta %lld",
965 vm_size, footprint_before, footprint_after,
966 footprint_expected, footprint_after - footprint_expected);
967
968 /* mapping ledger-tagged virtual memory... */
969 get_ledger_info(&footprint_before, &pagetable_before);
970 vm_addr = 0;
971 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
972 0, /* mask */
973 VM_FLAGS_ANYWHERE,
974 me_port,
975 0, /* offset */
976 FALSE, /* copy */
977 VM_PROT_READ | VM_PROT_WRITE,
978 VM_PROT_READ | VM_PROT_WRITE,
979 VM_INHERIT_DEFAULT);
980 T_QUIET;
981 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
982 kr, mach_error_string(kr));
983 T_QUIET;
984 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
985 /* ... should not change footprint */
986 get_ledger_info(&footprint_after, &pagetable_after);
987 footprint_expected = footprint_before;
988 footprint_expected += (pagetable_after - pagetable_before);
989 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
990 T_EXPECT_EQ(footprint_after, footprint_expected,
991 "ledger-tagged mapping of %lld bytes: "
992 "footprint %lld -> %lld expected %lld delta %lld",
993 vm_size, footprint_before, footprint_after,
994 footprint_expected, footprint_after - footprint_expected);
995
996 /* touching memory... */
997 get_ledger_info(&footprint_before, &pagetable_before);
998 dirty_size = vm_size / 2;
999 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1000 /* ... should increase footprint */
1001 get_ledger_info(&footprint_after, &pagetable_after);
1002 footprint_expected = footprint_before + dirty_size;
1003 footprint_expected += (pagetable_after - pagetable_before);
1004 T_LOG("modifying ledger-tagged memory increases phys_footprint");
1005 T_EXPECT_EQ(footprint_after, footprint_expected,
1006 "touched %lld bytes: "
1007 "footprint %lld -> %lld expected %lld delta %lld",
1008 dirty_size, footprint_before, footprint_after,
1009 footprint_expected, footprint_after - footprint_expected);
1010
1011 /* making it volatile... */
1012 get_ledger_info(&footprint_before, &pagetable_before);
1013 state = VM_PURGABLE_VOLATILE;
1014 T_QUIET;
1015 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
1016 vm_addr,
1017 VM_PURGABLE_SET_STATE,
1018 &state),
1019 KERN_SUCCESS,
1020 "vm_purgable_control(VOLATILE)");
1021 T_QUIET;
1022 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
1023 "memory was non-volatile");
1024 /* ... should decrease footprint */
1025 get_ledger_info(&footprint_after, &pagetable_after);
1026 footprint_expected = footprint_before - dirty_size;
1027 footprint_expected += (pagetable_after - pagetable_before);
1028 T_LOG("making volatile decreases phys_footprint");
1029 T_EXPECT_EQ(footprint_after, footprint_expected,
1030 "made volatile %lld dirty bytes: "
1031 "footprint %lld -> %lld expected %lld delta %lld",
1032 dirty_size, footprint_before, footprint_after,
1033 footprint_expected, footprint_after - footprint_expected);
1034
1035 /* making it non-volatile... */
1036 get_ledger_info(&footprint_before, &pagetable_before);
1037 state = VM_PURGABLE_NONVOLATILE;
1038 T_QUIET;
1039 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
1040 vm_addr,
1041 VM_PURGABLE_SET_STATE,
1042 &state),
1043 KERN_SUCCESS,
1044 "vm_purgable_control(NONVOLATILE)");
1045 T_QUIET;
1046 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
1047 "memory was volatile");
1048 /* ... should increase footprint */
1049 get_ledger_info(&footprint_after, &pagetable_after);
1050 footprint_expected = footprint_before + dirty_size;
1051 footprint_expected += (pagetable_after - pagetable_before);
1052 T_LOG("making non-volatile increases phys_footprint");
1053 T_EXPECT_EQ(footprint_after, footprint_expected,
1054 "made non-volatile %lld dirty bytes: "
1055 "footprint %lld -> %lld expected %lld delta %lld",
1056 dirty_size, footprint_before, footprint_after,
1057 footprint_expected, footprint_after - footprint_expected);
1058
1059 /* deallocating memory while holding memory entry... */
1060 get_ledger_info(&footprint_before, &pagetable_before);
1061 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1062 T_QUIET;
1063 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1064 kr, mach_error_string(kr));
1065 /* ... should not change footprint */
1066 get_ledger_info(&footprint_after, &pagetable_after);
1067 footprint_expected = footprint_before;
1068 footprint_expected += (pagetable_after - pagetable_before);
1069 T_LOG("deallocating owned memory while holding memory entry "
1070 "does not change phys_footprint");
1071 T_EXPECT_EQ(footprint_after, footprint_expected,
1072 "deallocated %lld dirty bytes: "
1073 "footprint %lld -> %lld expected %lld delta %lld",
1074 dirty_size, footprint_before, footprint_after,
1075 footprint_expected, footprint_after - footprint_expected);
1076
1077 /* releasing the memory entry... */
1078 kr = mach_port_deallocate(mach_task_self(), me_port);
1079 T_QUIET;
1080 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1081 kr, mach_error_string(kr));
1082 /* ... should decrease footprint */
1083 get_ledger_info(&footprint_after, &pagetable_after);
1084 footprint_expected = footprint_before - dirty_size;
1085 footprint_expected += (pagetable_after - pagetable_before);
1086 T_LOG("releasing memory entry decreases phys_footprint");
1087 T_EXPECT_EQ(footprint_after, footprint_expected,
1088 "made volatile %lld dirty bytes: "
1089 "footprint %lld -> %lld expected %lld delta %lld",
1090 dirty_size, footprint_before, footprint_after,
1091 footprint_expected, footprint_after - footprint_expected);
1092 }
1093
1094 T_DECL(phys_footprint_ledger_owned,
1095 "phys_footprint for ledger-tagged memory ownership",
1096 T_META_LTEPHASE(LTE_POSTINIT))
1097 {
1098 uint64_t footprint_before, pagetable_before;
1099 uint64_t footprint_after, pagetable_after;
1100 uint64_t footprint_expected;
1101 kern_return_t kr;
1102 mach_vm_address_t pre_vm_addr, vm_addr;
1103 mach_vm_size_t vm_size, dirty_size, me_size;
1104 mach_port_t me_port;
1105
1106 /* pre-warm to account for page table expansion */
1107 pre_vm_addr = pre_warm(MEM_SIZE);
1108
1109 /* making a memory entry... */
1110 get_ledger_info(&footprint_before, &pagetable_before);
1111 vm_size = MEM_SIZE;
1112 me_size = vm_size;
1113 me_port = MACH_PORT_NULL;
1114 kr = mach_make_memory_entry_64(mach_task_self(),
1115 &me_size,
1116 0,
1117 (MAP_MEM_NAMED_CREATE |
1118 MAP_MEM_LEDGER_TAGGED |
1119 VM_PROT_READ | VM_PROT_WRITE),
1120 &me_port,
1121 MACH_PORT_NULL);
1122 T_QUIET;
1123 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1124 kr, mach_error_string(kr));
1125 T_QUIET;
1126 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1127 /* ... should not change footprint */
1128 get_ledger_info(&footprint_after, &pagetable_after);
1129 footprint_expected = footprint_before;
1130 footprint_expected += (pagetable_after - pagetable_before);
1131 T_LOG("making a memory entry does not change phys_footprint");
1132 T_EXPECT_EQ(footprint_after, footprint_expected,
1133 "making a memory entry of %lld bytes: "
1134 "footprint %lld -> %lld expected %lld delta %lld",
1135 vm_size, footprint_before, footprint_after,
1136 footprint_expected, footprint_after - footprint_expected);
1137
1138 /* mapping ledger-tagged virtual memory... */
1139 get_ledger_info(&footprint_before, &pagetable_before);
1140 vm_addr = 0;
1141 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1142 0, /* mask */
1143 VM_FLAGS_ANYWHERE,
1144 me_port,
1145 0, /* offset */
1146 FALSE, /* copy */
1147 VM_PROT_READ | VM_PROT_WRITE,
1148 VM_PROT_READ | VM_PROT_WRITE,
1149 VM_INHERIT_DEFAULT);
1150 T_QUIET;
1151 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1152 kr, mach_error_string(kr));
1153 T_QUIET;
1154 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1155 /* ... should not change footprint */
1156 get_ledger_info(&footprint_after, &pagetable_after);
1157 footprint_expected = footprint_before;
1158 footprint_expected += (pagetable_after - pagetable_before);
1159 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1160 T_EXPECT_EQ(footprint_after, footprint_expected,
1161 "ledger-tagged mapping of %lld bytes: "
1162 "footprint %lld -> %lld expected %lld delta %lld",
1163 vm_size, footprint_before, footprint_after,
1164 footprint_expected, footprint_after - footprint_expected);
1165
1166 /* touching memory... */
1167 get_ledger_info(&footprint_before, &pagetable_before);
1168 dirty_size = vm_size / 2;
1169 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1170 /* ... should increase footprint */
1171 get_ledger_info(&footprint_after, &pagetable_after);
1172 footprint_expected = footprint_before + dirty_size;
1173 footprint_expected += (pagetable_after - pagetable_before);
1174 T_LOG("modifying ledger-tagged memory increases phys_footprint");
1175 T_EXPECT_EQ(footprint_after, footprint_expected,
1176 "touched %lld bytes: "
1177 "footprint %lld -> %lld expected %lld delta %lld",
1178 dirty_size, footprint_before, footprint_after,
1179 footprint_expected, footprint_after - footprint_expected);
1180
1181 /* deallocating memory while holding memory entry... */
1182 get_ledger_info(&footprint_before, &pagetable_before);
1183 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1184 T_QUIET;
1185 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1186 kr, mach_error_string(kr));
1187 /* ... should not change footprint */
1188 get_ledger_info(&footprint_after, &pagetable_after);
1189 footprint_expected = footprint_before;
1190 footprint_expected += (pagetable_after - pagetable_before);
1191 T_LOG("deallocating owned memory while holding memory entry "
1192 "does not change phys_footprint");
1193 T_EXPECT_EQ(footprint_after, footprint_expected,
1194 "deallocated %lld dirty bytes: "
1195 "footprint %lld -> %lld expected %lld delta %lld",
1196 dirty_size, footprint_before, footprint_after,
1197 footprint_expected, footprint_after - footprint_expected);
1198
1199 /* releasing the memory entry... */
1200 kr = mach_port_deallocate(mach_task_self(), me_port);
1201 T_QUIET;
1202 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1203 kr, mach_error_string(kr));
1204 /* ... should decrease footprint */
1205 get_ledger_info(&footprint_after, &pagetable_after);
1206 footprint_expected = footprint_before - dirty_size;
1207 footprint_expected += (pagetable_after - pagetable_before);
1208 T_LOG("releasing memory entry decreases phys_footprint");
1209 T_EXPECT_EQ(footprint_after, footprint_expected,
1210 "made volatile %lld dirty bytes: "
1211 "footprint %lld -> %lld expected %lld delta %lld",
1212 dirty_size, footprint_before, footprint_after,
1213 footprint_expected, footprint_after - footprint_expected);
1214 }
1215
1216 T_DECL(phys_footprint_no_footprint_for_debug,
1217 "phys_footprint for no_footprint_for_debug",
1218 T_META_LTEPHASE(LTE_POSTINIT))
1219 {
1220 uint64_t footprint_before, pagetable_before;
1221 uint64_t footprint_after, pagetable_after;
1222 uint64_t footprint_expected;
1223 kern_return_t kr;
1224 mach_vm_address_t pre_vm_addr, vm_addr;
1225 mach_vm_size_t vm_size, dirty_size, me_size;
1226 mach_port_t me_port;
1227 int new_value, ret;
1228
1229 /* pre-warm to account for page table expansion */
1230 pre_vm_addr = pre_warm(MEM_SIZE);
1231
1232 /* making a memory entry... */
1233 get_ledger_info(&footprint_before, &pagetable_before);
1234 vm_size = MEM_SIZE;
1235 me_size = vm_size;
1236 me_port = MACH_PORT_NULL;
1237 kr = mach_make_memory_entry_64(mach_task_self(),
1238 &me_size,
1239 0,
1240 (MAP_MEM_NAMED_CREATE |
1241 MAP_MEM_LEDGER_TAGGED |
1242 VM_PROT_READ | VM_PROT_WRITE),
1243 &me_port,
1244 MACH_PORT_NULL);
1245 T_QUIET;
1246 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1247 kr, mach_error_string(kr));
1248 T_QUIET;
1249 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1250 /* ... should not change footprint */
1251 get_ledger_info(&footprint_after, &pagetable_after);
1252 footprint_expected = footprint_before;
1253 footprint_expected += (pagetable_after - pagetable_before);
1254 T_LOG("making a memory entry does not change phys_footprint");
1255 T_EXPECT_EQ(footprint_after, footprint_expected,
1256 "making a memory entry of %lld bytes: "
1257 "footprint %lld -> %lld expected %lld delta %lld",
1258 vm_size, footprint_before, footprint_after,
1259 footprint_expected, footprint_after - footprint_expected);
1260
1261 /* trying to hide debug memory from footprint while not allowed */
1262 kr = mach_memory_entry_ownership(me_port,
1263 mach_task_self(),
1264 VM_LEDGER_TAG_DEFAULT,
1265 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1266 T_QUIET;
1267 T_EXPECT_EQ(kr, KERN_NO_ACCESS,
1268 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) fails without sysctl");
1269
1270 /* let's get permission to hide debug memory */
1271 new_value = 1;
1272 ret = sysctlbyname("vm.task_no_footprint_for_debug", NULL, NULL, &new_value, sizeof(new_value));
1273 if (ret == -1 && errno == ENOENT) {
1274 T_SKIP("sysctlbyname(vm.task_no_footprint_for_debug) ENOENT");
1275 }
1276 T_QUIET;
1277 T_ASSERT_POSIX_SUCCESS(ret, "sysctlbyname(vm.task_no_footprint_for_debug)");
1278
1279 /* trying to hide debug memory from footprint while allowed */
1280 task_id_token_t token;
1281 kr = task_create_identity_token(mach_task_self(), &token);
1282 T_QUIET;
1283 T_ASSERT_EQ(kr, KERN_SUCCESS,
1284 "task_create_identity_token");
1285
1286 kr = mach_memory_entry_ownership(me_port,
1287 token, /* this time also try calling it with id token */
1288 VM_LEDGER_TAG_DEFAULT,
1289 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1290 T_QUIET;
1291 T_EXPECT_EQ(kr, KERN_SUCCESS,
1292 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) succeeds after sysctl");
1293 mach_port_deallocate(mach_task_self(), token);
1294
1295 /* mapping ledger-tagged virtual memory... */
1296 get_ledger_info(&footprint_before, &pagetable_before);
1297 vm_addr = 0;
1298 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1299 0, /* mask */
1300 VM_FLAGS_ANYWHERE,
1301 me_port,
1302 0, /* offset */
1303 FALSE, /* copy */
1304 VM_PROT_READ | VM_PROT_WRITE,
1305 VM_PROT_READ | VM_PROT_WRITE,
1306 VM_INHERIT_DEFAULT);
1307 T_QUIET;
1308 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1309 kr, mach_error_string(kr));
1310 T_QUIET;
1311 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1312 /* ... should not change footprint */
1313 get_ledger_info(&footprint_after, &pagetable_after);
1314 footprint_expected = footprint_before;
1315 footprint_expected += (pagetable_after - pagetable_before);
1316 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1317 T_EXPECT_EQ(footprint_after, footprint_expected,
1318 "ledger-tagged mapping of %lld bytes: "
1319 "footprint %lld -> %lld expected %lld delta %lld",
1320 vm_size, footprint_before, footprint_after,
1321 footprint_expected, footprint_after - footprint_expected);
1322
1323 /* touching memory... */
1324 get_ledger_info(&footprint_before, &pagetable_before);
1325 dirty_size = vm_size / 2;
1326 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1327 /* ... should not increase footprint */
1328 get_ledger_info(&footprint_after, &pagetable_after);
1329 footprint_expected = footprint_before;
1330 footprint_expected += (pagetable_after - pagetable_before);
1331 T_LOG("modifying no_footprint_for_debug memory does not increase phys_footprint");
1332 T_EXPECT_EQ(footprint_after, footprint_expected,
1333 "touched %lld bytes: "
1334 "footprint %lld -> %lld expected %lld delta %lld",
1335 dirty_size, footprint_before, footprint_after,
1336 footprint_expected, footprint_after - footprint_expected);
1337
1338 /* deallocating memory while holding memory entry... */
1339 get_ledger_info(&footprint_before, &pagetable_before);
1340 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1341 T_QUIET;
1342 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1343 kr, mach_error_string(kr));
1344 /* ... should not change footprint */
1345 get_ledger_info(&footprint_after, &pagetable_after);
1346 footprint_expected = footprint_before;
1347 footprint_expected += (pagetable_after - pagetable_before);
1348 T_LOG("deallocating owned memory while holding memory entry "
1349 "does not change phys_footprint");
1350 T_EXPECT_EQ(footprint_after, footprint_expected,
1351 "deallocated %lld dirty bytes: "
1352 "footprint %lld -> %lld expected %lld delta %lld",
1353 dirty_size, footprint_before, footprint_after,
1354 footprint_expected, footprint_after - footprint_expected);
1355
1356 /* releasing the memory entry... */
1357 kr = mach_port_deallocate(mach_task_self(), me_port);
1358 T_QUIET;
1359 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1360 kr, mach_error_string(kr));
1361 /* ... should not change footprint */
1362 get_ledger_info(&footprint_after, &pagetable_after);
1363 footprint_expected = footprint_before;
1364 footprint_expected += (pagetable_after - pagetable_before);
1365 T_LOG("releasing memory entry does not change phys_footprint");
1366 T_EXPECT_EQ(footprint_after, footprint_expected,
1367 "made volatile %lld dirty bytes: "
1368 "footprint %lld -> %lld expected %lld delta %lld",
1369 dirty_size, footprint_before, footprint_after,
1370 footprint_expected, footprint_after - footprint_expected);
1371 }
1372 #endif /* MAP_MEM_LEDGER_TAGGED */
1373
1374 /* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
1375 #include <CoreFoundation/CoreFoundation.h>
1376 #include <IOSurface/IOSurface.h>
1377 #include <IOSurface/IOSurfacePrivate.h>
1378 static size_t
bytes_per_element(uint32_t format)1379 bytes_per_element(uint32_t format)
1380 {
1381 size_t bpe = 0;
1382 switch (format) {
1383 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1384 bpe = 4;
1385 break;
1386 default:
1387 bpe = 0;
1388 break;
1389 }
1390 return bpe;
1391 }
1392 static size_t
bytes_per_pixel(uint32_t format)1393 bytes_per_pixel(uint32_t format)
1394 {
1395 size_t bpe = 0;
1396 switch (format) {
1397 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1398 bpe = 4;
1399 break;
1400 default:
1401 bpe = 0;
1402 break;
1403 }
1404 return bpe;
1405 }
1406 static inline size_t
roundSizeToMultiple(size_t size,size_t mult)1407 roundSizeToMultiple(size_t size, size_t mult)
1408 {
1409 return ((size + mult - 1) / mult) * mult;
1410 }
1411 static inline void
setIntValue(CFMutableDictionaryRef dict,const CFStringRef key,int value)1412 setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
1413 {
1414 CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
1415 CFDictionarySetValue(dict, key, number);
1416 CFRelease(number);
1417 }
1418 static inline void
setBoolValue(CFMutableDictionaryRef dict,const CFStringRef key,bool value)1419 setBoolValue(CFMutableDictionaryRef dict, const CFStringRef key, bool value)
1420 {
1421 CFDictionarySetValue(dict, key, value ? kCFBooleanTrue : kCFBooleanFalse);
1422 }
1423 typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
1424 static IOReturn
SurfaceApplyPlaneBlock(IOSurfaceRef surface,SurfacePlaneBlock block)1425 SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
1426 {
1427 if (surface == nil || block == nil) {
1428 return kIOReturnBadArgument;
1429 }
1430
1431 IOReturn result = kIOReturnSuccess;
1432 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1433
1434 if (planeCount == 0) {
1435 result = IOSurfaceLock(surface, 0, NULL);
1436 if (result != kIOReturnSuccess) {
1437 return result;
1438 }
1439
1440 void* base = IOSurfaceGetBaseAddress(surface);
1441 size_t rb = IOSurfaceGetBytesPerRow(surface);
1442 size_t w = IOSurfaceGetWidth(surface);
1443 size_t h = IOSurfaceGetHeight(surface);
1444
1445 if (base && rb && w && h) {
1446 block(base, 0, w, h, rb);
1447 }
1448
1449 IOSurfaceUnlock(surface, 0, NULL);
1450 } else if (planeCount == 2) {
1451 for (size_t i = 0; i < planeCount; i++) {
1452 result = IOSurfaceLock(surface, 0, NULL);
1453 if (result != kIOReturnSuccess) {
1454 return result;
1455 }
1456
1457 void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
1458 size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
1459 size_t w = IOSurfaceGetWidthOfPlane(surface, i);
1460 size_t h = IOSurfaceGetHeightOfPlane(surface, i);
1461
1462 if (base && rb && w && h) {
1463 block(base, i, w, h, rb);
1464 }
1465
1466 IOSurfaceUnlock(surface, 0, NULL);
1467 }
1468 }
1469 return result;
1470 }
1471 static void
ClearSurface(IOSurfaceRef surface)1472 ClearSurface(IOSurfaceRef surface)
1473 {
1474 const int zero = 0;
1475 (void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
1476 {
1477 if (i == 0) {
1478 memset(p, zero, rb * h);
1479 } else {
1480 memset(p, 128, rb * h);
1481 }
1482 });
1483 }
1484 static size_t
SurfaceGetMemorySize(IOSurfaceRef surface)1485 SurfaceGetMemorySize(IOSurfaceRef surface)
1486 {
1487 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1488
1489 if (planeCount == 0) {
1490 size_t rb = IOSurfaceGetBytesPerRow(surface);
1491 size_t h = IOSurfaceGetHeight(surface);
1492 return rb * h;
1493 } else if (planeCount == 2) {
1494 size_t rb0 = IOSurfaceGetBytesPerRowOfPlane(surface, 0);
1495 size_t h0 = IOSurfaceGetHeightOfPlane(surface, 0);
1496 size_t rb1 = IOSurfaceGetBytesPerRowOfPlane(surface, 1);
1497 size_t h1 = IOSurfaceGetHeightOfPlane(surface, 1);
1498 return rb0 * h0 + rb1 * h1;
1499 }
1500 return 0;
1501 }
1502 static IOSurfaceRef
CreateSurface(uint32_t pixelsWide,uint32_t pixelsHigh,uint32_t rowBytesAlignment,uint32_t fmt,bool purgeable,bool clear)1503 CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
1504 {
1505 IOSurfaceRef surface = nil;
1506
1507 if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
1508 return nil;
1509 }
1510
1511 size_t bpp = bytes_per_pixel(fmt);
1512 size_t bpe = bytes_per_element(fmt);
1513 if (bpp == 0 || bpe == 0) {
1514 return nil;
1515 }
1516
1517 size_t rowbytes = pixelsWide * bpp;
1518 if (rowBytesAlignment == 0) {
1519 rowBytesAlignment = 16;
1520 }
1521 rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);
1522
1523 CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1524 setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
1525 setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
1526 setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
1527 setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
1528 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1529 setBoolValue(props, kIOSurfaceNonPurgeable, !purgeable);
1530 #else /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1531 (void)purgeable;
1532 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1533 {
1534 if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
1535 setIntValue(props, kIOSurfaceElementWidth, 2);
1536 setIntValue(props, kIOSurfaceElementHeight, 1);
1537 }
1538 setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
1539 }
1540
1541 surface = IOSurfaceCreate(props);
1542
1543 if (clear) {
1544 ClearSurface(surface);
1545 }
1546
1547 CFRelease(props);
1548 return surface;
1549 }
1550 T_DECL(phys_footprint_purgeable_iokit,
1551 "phys_footprint for purgeable IOKit memory",
1552 T_META_LTEPHASE(LTE_POSTINIT))
1553 {
1554 uint64_t footprint_before, pagetable_before;
1555 uint64_t footprint_after, pagetable_after;
1556 uint64_t footprint_expected, footprint_delta_slop;
1557 int64_t footprint_delta;
1558 IOSurfaceRef surface;
1559 uint32_t old_state;
1560 uint64_t surface_size;
1561
1562 T_SETUPBEGIN;
1563 footprint_delta_slop = 8 * vm_kernel_page_size;
1564 ledger_init();
1565 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1566 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1567 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1568 CFRelease(surface);
1569 T_SETUPEND;
1570
1571 surface_size = 1024 * 1024 * 4;
1572
1573 /* create IOsurface: footprint grows */
1574 get_ledger_info(&footprint_before, &pagetable_before);
1575 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1576 get_ledger_info(&footprint_after, &pagetable_after);
1577 if (legacy_footprint) {
1578 footprint_expected = footprint_before;
1579 footprint_expected += (pagetable_after - pagetable_before);
1580 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1581 T_LOG("LEGACY FOOTPRINT: creating purgeable IOSurface: no footprint impact");
1582 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1583 "create purgeable IOSurface %lld bytes: "
1584 "footprint %lld -> %lld expected %lld delta %lld",
1585 surface_size, footprint_before, footprint_after,
1586 footprint_expected, footprint_delta);
1587 } else {
1588 footprint_expected = footprint_before + surface_size;
1589 footprint_expected += (pagetable_after - pagetable_before);
1590 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1591 T_LOG("creating purgeable IOSurface increases phys_footprint");
1592 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1593 "create purgeable IOSurface %lld bytes: "
1594 "footprint %lld -> %lld expected %lld delta %lld",
1595 surface_size, footprint_before, footprint_after,
1596 footprint_expected, footprint_delta);
1597 }
1598
1599 /* make IOSurface volatile: footprint shrinks */
1600 get_ledger_info(&footprint_before, &pagetable_before);
1601 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1602 get_ledger_info(&footprint_after, &pagetable_after);
1603 if (legacy_footprint) {
1604 footprint_expected = footprint_before;
1605 footprint_expected += (pagetable_after - pagetable_before);
1606 T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
1607 T_EXPECT_EQ(footprint_after, footprint_expected,
1608 "volatile IOSurface %lld bytes: "
1609 "footprint %lld -> %lld expected %lld delta %lld",
1610 surface_size, footprint_before, footprint_after,
1611 footprint_expected, footprint_after - footprint_expected);
1612 } else {
1613 footprint_expected = footprint_before - surface_size;
1614 footprint_expected += (pagetable_after - pagetable_before);
1615 T_LOG("making IOSurface volatile decreases phys_footprint");
1616 T_EXPECT_EQ(footprint_after, footprint_expected,
1617 "made volatile %lld bytes: "
1618 "footprint %lld -> %lld expected %lld delta %lld",
1619 surface_size, footprint_before, footprint_after,
1620 footprint_expected, footprint_after - footprint_expected);
1621 }
1622
1623 /* make IOSurface non-volatile: footprint grows */
1624 get_ledger_info(&footprint_before, &pagetable_before);
1625 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1626 get_ledger_info(&footprint_after, &pagetable_after);
1627 if (legacy_footprint) {
1628 footprint_expected = footprint_before;
1629 footprint_expected += (pagetable_after - pagetable_before);
1630 T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
1631 T_EXPECT_EQ(footprint_after, footprint_expected,
1632 "non-volatile IOSurface %lld bytes: "
1633 "footprint %lld -> %lld expected %lld delta %lld",
1634 surface_size, footprint_before, footprint_after,
1635 footprint_expected, footprint_after - footprint_expected);
1636 } else {
1637 footprint_expected = footprint_before + surface_size;
1638 footprint_expected += (pagetable_after - pagetable_before);
1639 T_LOG("making IOSurface non-volatile increases phys_footprint");
1640 T_EXPECT_EQ(footprint_after, footprint_expected,
1641 "made non-volatile %lld bytes: "
1642 "footprint %lld -> %lld expected %lld delta %lld",
1643 surface_size, footprint_before, footprint_after,
1644 footprint_expected, footprint_after - footprint_expected);
1645 }
1646
1647 /* accessing IOSurface re-mapping: no footprint impact */
1648
1649 /* deallocating IOSurface re-mapping: no footprint impact */
1650
1651 /* release IOSurface: footprint shrinks */
1652 get_ledger_info(&footprint_before, &pagetable_before);
1653 CFRelease(surface);
1654 get_ledger_info(&footprint_after, &pagetable_after);
1655 if (legacy_footprint) {
1656 footprint_expected = footprint_before;
1657 footprint_expected += (pagetable_after - pagetable_before);
1658 T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
1659 T_EXPECT_EQ(footprint_after, footprint_expected,
1660 "releasing IOSurface %lld bytes: "
1661 "footprint %lld -> %lld expected %lld delta %lld",
1662 surface_size, footprint_before, footprint_after,
1663 footprint_expected, footprint_after - footprint_expected);
1664 } else {
1665 footprint_expected = footprint_before - surface_size;
1666 footprint_expected += (pagetable_after - pagetable_before);
1667 T_LOG("releasing IOSurface decreases phys_footprint");
1668 T_EXPECT_EQ(footprint_after, footprint_expected,
1669 "released IOSurface %lld bytes: "
1670 "footprint %lld -> %lld expected %lld delta %lld",
1671 surface_size, footprint_before, footprint_after,
1672 footprint_expected, footprint_after - footprint_expected);
1673 }
1674 }
1675
1676 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1677 T_DECL(phys_footprint_nonpurgeable_iokit,
1678 "phys_footprint for non-purgeable IOKit memory",
1679 T_META_LTEPHASE(LTE_POSTINIT))
1680 {
1681 uint64_t footprint_before, pagetable_before;
1682 uint64_t footprint_after, pagetable_after;
1683 uint64_t footprint_expected, footprint_delta_slop;
1684 int64_t footprint_delta;
1685 IOSurfaceRef surface;
1686 uint64_t surface_size;
1687 void *map_base;
1688 size_t map_size;
1689 mach_vm_address_t remap_addr;
1690 kern_return_t kr;
1691 vm_prot_t cur_prot, max_prot;
1692 uint32_t old_state;
1693
1694
1695 T_SETUPBEGIN;
1696 ledger_init();
1697 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1698 CFRelease(surface);
1699 footprint_delta_slop = 8 * vm_kernel_page_size;
1700 T_SETUPEND;
1701
1702 surface_size = 1024 * 1024 * 4;
1703
1704 /* create IOsurface: footprint grows */
1705 get_ledger_info(&footprint_before, &pagetable_before);
1706 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1707 get_ledger_info(&footprint_after, &pagetable_after);
1708 footprint_expected = footprint_before + surface_size;
1709 footprint_expected += (pagetable_after - pagetable_before);
1710 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1711 T_LOG("creating non-purgeable IOSurface increases phys_footprint");
1712 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1713 "create non-purgeable IOSurface %lld bytes: "
1714 "footprint %lld -> %lld expected %lld delta %lld",
1715 surface_size, footprint_before, footprint_after,
1716 footprint_expected, footprint_delta);
1717
1718 /* make IOSurface volatile: fail and no footprint impact */
1719 get_ledger_info(&footprint_before, &pagetable_before);
1720 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1721 get_ledger_info(&footprint_after, &pagetable_after);
1722 footprint_expected = footprint_before;
1723 footprint_expected += (pagetable_after - pagetable_before);
1724 T_LOG("making non-purgeable IOSurface volatile: no footprint impact");
1725 T_EXPECT_EQ(footprint_after, footprint_expected,
1726 "made volatile %lld non-purgeable bytes: "
1727 "footprint %lld -> %lld expected %lld delta %lld",
1728 surface_size, footprint_before, footprint_after,
1729 footprint_expected, footprint_after - footprint_expected);
1730
1731 /* re-mapping IOSurface: no footprint impact */
1732 get_ledger_info(&footprint_before, &pagetable_before);
1733 map_base = IOSurfaceGetBaseAddress(surface);
1734 map_size = SurfaceGetMemorySize(surface);
1735 // T_EXPECT_EQ(map_size, surface_size, "map_size %lld surface_size %lld",
1736 // map_size, surface_size);
1737 remap_addr = 0;
1738 kr = mach_vm_remap(mach_task_self(),
1739 &remap_addr,
1740 (mach_vm_size_t)surface_size,
1741 0,
1742 VM_FLAGS_ANYWHERE,
1743 mach_task_self(),
1744 (mach_vm_address_t)map_base,
1745 FALSE, /* copy */
1746 &cur_prot,
1747 &max_prot,
1748 VM_INHERIT_DEFAULT);
1749 T_QUIET;
1750 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_remap() error 0x%x (%s)",
1751 kr, mach_error_string(kr));
1752 get_ledger_info(&footprint_after, &pagetable_after);
1753 footprint_expected = footprint_before;
1754 footprint_expected += (pagetable_after - pagetable_before);
1755 T_LOG("re-mapping IOSurface does not impact phys_footprint");
1756 T_EXPECT_EQ(footprint_after, footprint_expected,
1757 "remapping IOSurface %lld bytes: "
1758 "footprint %lld -> %lld expected %lld delta %lld",
1759 surface_size, footprint_before, footprint_after,
1760 footprint_expected, footprint_after - footprint_expected);
1761
1762 /* accessing IOSurface re-mapping: footprint grows */
1763 get_ledger_info(&footprint_before, &pagetable_before);
1764 memset((char *)(uintptr_t)remap_addr, 'p', (size_t)surface_size);
1765 get_ledger_info(&footprint_after, &pagetable_after);
1766 footprint_expected = footprint_before + surface_size;
1767 footprint_expected += (pagetable_after - pagetable_before);
1768 T_LOG("accessing re-mapped IOSurface grows phys_footprint");
1769 T_EXPECT_EQ(footprint_after, footprint_expected,
1770 "accessing remapped IOSurface %lld bytes: "
1771 "footprint %lld -> %lld expected %lld delta %lld",
1772 surface_size, footprint_before, footprint_after,
1773 footprint_expected, footprint_after - footprint_expected);
1774
1775 /* deallocating IOSurface re-mapping: footprint shrinks */
1776 get_ledger_info(&footprint_before, &pagetable_before);
1777 kr = mach_vm_deallocate(mach_task_self(),
1778 remap_addr,
1779 (mach_vm_size_t)surface_size);
1780 T_QUIET;
1781 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1782 kr, mach_error_string(kr));
1783 get_ledger_info(&footprint_after, &pagetable_after);
1784 footprint_expected = footprint_before - surface_size;
1785 footprint_expected += (pagetable_after - pagetable_before);
1786 T_LOG("deallocating re-mapping of IOSurface shrinks phys_footprint");
1787 T_EXPECT_EQ(footprint_after, footprint_expected,
1788 "deallocating remapped IOSurface %lld bytes: "
1789 "footprint %lld -> %lld expected %lld delta %lld",
1790 surface_size, footprint_before, footprint_after,
1791 footprint_expected, footprint_after - footprint_expected);
1792
1793 /* release IOSurface: footprint shrinks */
1794 get_ledger_info(&footprint_before, &pagetable_before);
1795 CFRelease(surface);
1796 get_ledger_info(&footprint_after, &pagetable_after);
1797 footprint_expected = footprint_before - surface_size;
1798 footprint_expected += (pagetable_after - pagetable_before);
1799 T_LOG("releasing IOSurface decreases phys_footprint");
1800 T_EXPECT_EQ(footprint_after, footprint_expected,
1801 "released IOSurface %lld bytes: "
1802 "footprint %lld -> %lld expected %lld delta %lld",
1803 surface_size, footprint_before, footprint_after,
1804 footprint_expected, footprint_after - footprint_expected);
1805 }
1806 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1807
1808 #if 0
1809 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
1810 T_DECL(legacy_footprint_entitled_set_and_query,
1811 "legacy footprint entitled is sticky with memlimit set properties and can be queried",
1812 T_META_BOOTARGS_SET("legacy_footprint_entitlement_mode=3"))
1813 {
1814 int ret;
1815 int32_t max_task_pmem = 0, legacy_footprint_bonus_mb = 0;
1816 size_t size_max_task_pmem = sizeof(max_task_pmem);
1817 size_t size_legacy_footprint_bonus_mb = sizeof(legacy_footprint_bonus_mb);
1818 memorystatus_memlimit_properties2_t mmprops;
1819
1820 /* Get the default limit */
1821 ret = sysctlbyname("kern.max_task_pmem", &max_task_pmem, &size_max_task_pmem, NULL, 0);
1822 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "call sysctlbyname to get max task physical memory.");
1823
1824 ret = sysctlbyname("kern.legacy_footprint_bonus_mb", &legacy_footprint_bonus_mb,
1825 &size_legacy_footprint_bonus_mb, NULL, 0);
1826 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "call sysctlbyname to get the bonus value.");
1827
1828 mmprops.v1.memlimit_active = -1;
1829 mmprops.v1.memlimit_inactive = -1;
1830 ret = memorystatus_control(MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES, getpid(), 0, &mmprops.v1, sizeof(mmprops.v1));
1831 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1832
1833 /* Check our memlimt */
1834 ret = memorystatus_control(MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES, getpid(), 0, &mmprops, sizeof(mmprops));
1835 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1836
1837 T_QUIET; T_ASSERT_EQ(mmprops.v1.memlimit_active, max_task_pmem + legacy_footprint_bonus_mb, "active limit");
1838 T_QUIET; T_ASSERT_EQ(mmprops.v1.memlimit_inactive, max_task_pmem + legacy_footprint_bonus_mb, "inactive limit");
1839
1840 /* Verify MEMORYSTATUS_CMD_CONVERT_MEMLIMIT_MB */
1841 ret = memorystatus_control(MEMORYSTATUS_CMD_CONVERT_MEMLIMIT_MB, getpid(), (uint32_t) -1, NULL, 0);
1842 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1843 T_QUIET; T_ASSERT_EQ(ret, max_task_pmem + legacy_footprint_bonus_mb, "got bonus adjustment");
1844 }
1845 #endif /* LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__) */
1846 #endif
1847