1 #include <darwintest.h>
2 #include <darwintest_utils.h>
3
4 #include <mach/mach_error.h>
5 #include <mach/mach_init.h>
6 #include <mach/memory_entry.h>
7 #include <mach/mach_port.h>
8 #include <mach/mach_vm.h>
9 #include <mach/memory_entry.h>
10 #include <mach/task.h>
11 #include <mach/task_info.h>
12 #include <mach/vm_map.h>
13
14 #include <sys/kern_memorystatus.h>
15 #include <sys/mman.h>
16 #include <sys/types.h>
17 #include <sys/sysctl.h>
18
19 #include <TargetConditionals.h>
20
21 #include <Kernel/kern/ledger.h>
22 extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);
23
24 T_GLOBAL_META(T_META_RUN_CONCURRENTLY(true));
25 static boolean_t legacy_footprint = FALSE;
26
27 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
28 #define TEST_VM_NAMESPACE "xnu.vm_legacy"
29 #else /* ENTITLED && __arm64__ */
30 #define TEST_VM_NAMESPACE "xnu.vm"
31 #endif /* ENTITLED && __arm64__ */
32
33 T_GLOBAL_META(
34 T_META_NAMESPACE(TEST_VM_NAMESPACE),
35 T_META_RADAR_COMPONENT_NAME("xnu"),
36 T_META_RADAR_COMPONENT_VERSION("VM"),
37 T_META_RUN_CONCURRENTLY(true));
38
39 #define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */
40
41 static int64_t ledger_count = -1;
42 static int footprint_index = -1;
43 static int pagetable_index = -1;
44 static struct ledger_entry_info *lei = NULL;
45
46 static void
ledger_init(void)47 ledger_init(void)
48 {
49 static int ledger_inited = 0;
50 struct ledger_info li;
51 struct ledger_template_info *templateInfo;
52 int64_t templateCnt;
53 int i;
54
55 if (ledger_inited) {
56 return;
57 }
58 ledger_inited = 1;
59
60 T_SETUPBEGIN;
61
62 legacy_footprint = FALSE;
63 #if LEGACY_FOOTPRINT_ENTITLED
64 int legacy_footprint_entitlement_mode;
65 size_t oldlen;
66 int ret;
67
68 T_QUIET;
69 T_WITH_ERRNO;
70 oldlen = sizeof(legacy_footprint_entitlement_mode);
71 ret = sysctlbyname("kern.legacy_footprint_entitlement_mode",
72 &legacy_footprint_entitlement_mode,
73 &oldlen,
74 NULL,
75 0);
76 if (ret == 0 && legacy_footprint_entitlement_mode == 2) {
77 legacy_footprint = TRUE;
78 }
79 #endif /* LEGACY_FOOTPRINT_ENTITLED */
80
81 T_QUIET;
82 T_WITH_ERRNO;
83 T_ASSERT_EQ(ledger(LEDGER_INFO,
84 (caddr_t)(uintptr_t)getpid(),
85 (caddr_t)&li,
86 NULL),
87 0,
88 "ledger(LEDGER_INFO)");
89
90 templateCnt = li.li_entries;
91 templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
92 T_QUIET;
93 T_WITH_ERRNO;
94 T_ASSERT_NE(templateInfo, NULL, "malloc()");
95
96 ledger_count = li.li_entries;
97 footprint_index = -1;
98 pagetable_index = -1;
99 T_QUIET;
100 T_WITH_ERRNO;
101 T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
102 (caddr_t)templateInfo,
103 (caddr_t)&templateCnt,
104 NULL),
105 0,
106 "ledger(LEDGER_TEMPLATE_INFO)");
107 for (i = 0; i < templateCnt; i++) {
108 if (!strncmp(templateInfo[i].lti_name,
109 "phys_footprint",
110 strlen("phys_footprint"))) {
111 footprint_index = i;
112 } else if (!strncmp(templateInfo[i].lti_name,
113 "page_table",
114 strlen("page_table"))) {
115 pagetable_index = i;
116 }
117 }
118 free(templateInfo);
119
120 lei = (struct ledger_entry_info *)
121 malloc((size_t)ledger_count * sizeof(*lei));
122 T_QUIET;
123 T_WITH_ERRNO;
124 T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");
125
126 T_QUIET;
127 T_ASSERT_NE(footprint_index, -1, "no footprint_index");
128 T_QUIET;
129 T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");
130
131 T_SETUPEND;
132 }
133
134 static void
get_ledger_info(uint64_t * phys_footprint,uint64_t * page_table)135 get_ledger_info(
136 uint64_t *phys_footprint,
137 uint64_t *page_table)
138 {
139 int64_t count;
140
141 count = ledger_count;
142 T_QUIET;
143 T_WITH_ERRNO;
144 T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
145 (caddr_t)(uintptr_t)getpid(),
146 (caddr_t)lei,
147 (caddr_t)&count),
148 0,
149 "ledger(LEDGER_ENTRY_INFO)");
150 T_QUIET;
151 T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
152 T_QUIET;
153 T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
154 if (phys_footprint) {
155 *phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
156 }
157 if (page_table) {
158 *page_table = (uint64_t)(lei[pagetable_index].lei_balance);
159 }
160 }
161
162 static mach_vm_address_t
pre_warm_with_tag(mach_vm_size_t vm_size,int tag)163 pre_warm_with_tag(
164 mach_vm_size_t vm_size,
165 int tag)
166 {
167 kern_return_t kr;
168 mach_vm_address_t vm_addr;
169 unsigned char BigBufOnStack[100 * 1024];
170 uint64_t footprint, page_table;
171
172 /* make sure ledgers are ready to be queried */
173 ledger_init();
174
175 T_SETUPBEGIN;
176
177 /*
178 * Touch a few pages ahead on the stack, to make
179 * sure we don't see a footprint increase due to
180 * an extra stack page later.
181 */
182 memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
183 T_QUIET;
184 T_EXPECT_EQ(BigBufOnStack[0], 0xb,
185 "BigBufOnStack[0] == 0x%x",
186 BigBufOnStack[0]);
187 T_QUIET;
188 T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
189 "BigBufOnStack[%lu] == 0x%x",
190 sizeof(BigBufOnStack),
191 BigBufOnStack[sizeof(BigBufOnStack) - 1]);
192
193 /*
194 * Pre-allocate, touch and then release the same amount
195 * of memory we'll be allocating later during the test,
196 * to account for any memory overhead (page tables, global
197 * variables, ...). Supplied VM tag should ensure memory
198 * is allocated in the appropriate VM range.
199 */
200 vm_addr = 0;
201 kr = mach_vm_allocate(mach_task_self(),
202 &vm_addr,
203 vm_size,
204 VM_FLAGS_ANYWHERE | VM_MAKE_TAG(tag));
205 T_QUIET;
206 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
207 vm_size, kr, mach_error_string(kr));
208 memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
209 kr = mach_vm_deallocate(mach_task_self(),
210 vm_addr,
211 vm_size);
212 T_QUIET;
213 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
214 kr, mach_error_string(kr));
215
216 /*
217 * Exercise the ledger code to make sure it's ready to run
218 * without any extra memory overhead later.
219 */
220 get_ledger_info(&footprint, &page_table);
221
222 T_SETUPEND;
223
224 /*
225 * Return the start of the virtual range we pre-warmed, so that the
226 * test can check that it's using the same range.
227 */
228 return vm_addr;
229 }
230
231 static mach_vm_address_t
pre_warm(mach_vm_size_t vm_size)232 pre_warm(
233 mach_vm_size_t vm_size)
234 {
235 /*
236 * Default to not tagging pre_warm memory.
237 */
238 return pre_warm_with_tag(vm_size, 0);
239 }
240
241 T_DECL(phys_footprint_anonymous,
242 "phys_footprint for anonymous memory",
243 T_META_LTEPHASE(LTE_POSTINIT))
244 {
245 uint64_t footprint_before, pagetable_before;
246 uint64_t footprint_after, pagetable_after;
247 uint64_t footprint_expected;
248 kern_return_t kr;
249 mach_vm_address_t pre_vm_addr, vm_addr;
250 mach_vm_size_t vm_size, dirty_size;
251
252 /* pre-warm to account for page table expansion */
253 pre_vm_addr = pre_warm(MEM_SIZE);
254
255 /* allocating virtual memory... */
256 get_ledger_info(&footprint_before, &pagetable_before);
257 vm_addr = 0;
258 vm_size = MEM_SIZE;
259 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
260 VM_FLAGS_ANYWHERE);
261 T_QUIET;
262 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
263 kr, mach_error_string(kr));
264 T_QUIET;
265 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
266 /* ... should not change footprint */
267 get_ledger_info(&footprint_after, &pagetable_after);
268 footprint_expected = footprint_before;
269 footprint_expected += (pagetable_after - pagetable_before);
270 T_LOG("virtual allocation does not change phys_footprint");
271 T_EXPECT_EQ(footprint_after, footprint_expected,
272 "virtual allocation of %lld bytes: "
273 "footprint %lld -> %lld expected %lld delta %lld",
274 vm_size, footprint_before, footprint_after,
275 footprint_expected, footprint_after - footprint_expected);
276
277 /* touching memory... */
278 get_ledger_info(&footprint_before, &pagetable_before);
279 dirty_size = vm_size / 2;
280 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
281 /* ... should increase footprint */
282 get_ledger_info(&footprint_after, &pagetable_after);
283 footprint_expected = footprint_before + dirty_size;
284 footprint_expected += (pagetable_after - pagetable_before);
285 T_LOG("modifying anonymous memory increases phys_footprint");
286 T_EXPECT_EQ(footprint_after, footprint_expected,
287 "touched %lld bytes: "
288 "footprint %lld -> %lld expected %lld delta %lld",
289 dirty_size, footprint_before, footprint_after,
290 footprint_expected, footprint_after - footprint_expected);
291
292 /* deallocating memory... */
293 get_ledger_info(&footprint_before, &pagetable_before);
294 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
295 T_QUIET;
296 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
297 kr, mach_error_string(kr));
298 /* ... should decrease footprint */
299 get_ledger_info(&footprint_after, &pagetable_after);
300 footprint_expected = footprint_before - dirty_size;
301 footprint_expected += (pagetable_after - pagetable_before);
302 T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
303 T_EXPECT_EQ(footprint_after, footprint_expected,
304 "deallocated %lld dirty bytes: "
305 "footprint %lld -> %lld expected %lld delta %lld",
306 dirty_size, footprint_before, footprint_after,
307 footprint_expected, footprint_after - footprint_expected);
308 }
309
310 #define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
311 #define TEMP_FILE_SIZE (1 * 1024 * 1024)
312
313 T_DECL(phys_footprint_file,
314 "phys_footprint for mapped file",
315 T_META_LTEPHASE(LTE_POSTINIT))
316 {
317 uint64_t footprint_before, pagetable_before;
318 uint64_t footprint_after, pagetable_after;
319 uint64_t footprint_expected;
320 mach_vm_address_t pre_vm_addr;
321 int fd;
322 char *map_addr;
323 size_t map_size, dirty_size;
324 ssize_t nbytes;
325 char tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
326 char *buf;
327 size_t buf_size;
328
329 T_SETUPBEGIN;
330 buf_size = TEMP_FILE_SIZE;
331 T_QUIET;
332 T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
333 "allocate %zu-byte buffer", buf_size);
334 memset(buf, 'f', buf_size);
335 T_WITH_ERRNO;
336 T_QUIET;
337 T_ASSERT_NOTNULL(mktemp(tmp_file_name),
338 "create temporary file name");
339 T_WITH_ERRNO;
340 T_QUIET;
341 T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
342 0,
343 "create temp file");
344 T_WITH_ERRNO;
345 T_QUIET;
346 T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
347 (ssize_t)buf_size,
348 "write %zu bytes", buf_size);
349 free(buf);
350 T_SETUPEND;
351
352 /* pre-warm to account for page table expansion */
353 pre_vm_addr = pre_warm_with_tag(TEMP_FILE_SIZE, VM_MEMORY_MALLOC);
354
355 /* mapping a file does not impact footprint... */
356 get_ledger_info(&footprint_before, &pagetable_before);
357 map_size = TEMP_FILE_SIZE;
358 T_WITH_ERRNO;
359 T_QUIET;
360 T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
361 PROT_READ | PROT_WRITE,
362 MAP_FILE | MAP_SHARED, fd, 0),
363 "mmap()");
364 T_QUIET;
365 T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
366 "pre-warm mishap");
367 /* ... should not change footprint */
368 get_ledger_info(&footprint_after, &pagetable_after);
369 footprint_expected = footprint_before;
370 footprint_expected += (pagetable_after - pagetable_before);
371 T_LOG("mapping file does not change phys_footprint");
372 T_EXPECT_EQ(footprint_after, footprint_expected,
373 "mapping file with %zu bytes: "
374 "footprint %lld -> %lld expected %lld delta %lld",
375 map_size, footprint_before, footprint_after,
376 footprint_expected, footprint_after - footprint_expected);
377
378 /* touching file-backed memory... */
379 get_ledger_info(&footprint_before, &pagetable_before);
380 dirty_size = map_size / 2;
381 memset(map_addr, 'F', dirty_size);
382 /* ... should not impact footprint */
383 get_ledger_info(&footprint_after, &pagetable_after);
384 footprint_expected = footprint_before;
385 footprint_expected += (pagetable_after - pagetable_before);
386 T_LOG("modifying file-backed memory does not impact phys_footprint");
387 T_EXPECT_EQ(footprint_after, footprint_expected,
388 "touched %zu bytes: "
389 "footprint %lld -> %lld expected %lld delta %lld",
390 dirty_size, footprint_before, footprint_after,
391 footprint_expected, footprint_after - footprint_expected);
392
393 /* deallocating file-backed memory... */
394 get_ledger_info(&footprint_before, &pagetable_before);
395 T_WITH_ERRNO;
396 T_QUIET;
397 T_ASSERT_EQ(munmap(map_addr, map_size),
398 0,
399 "unmap file");
400 /* ... should not impact footprint */
401 get_ledger_info(&footprint_after, &pagetable_after);
402 footprint_expected = footprint_before;
403 footprint_expected += (pagetable_after - pagetable_before);
404 T_LOG("unmapping file-backed memory does not impact phys_footprint");
405 T_EXPECT_EQ(footprint_after, footprint_expected,
406 "unmapped %zu dirty bytes: "
407 "footprint %lld -> %lld expected %lld delta %lld",
408 dirty_size, footprint_before, footprint_after,
409 footprint_expected, footprint_after - footprint_expected);
410 }
411
412 T_DECL(phys_footprint_purgeable,
413 "phys_footprint for purgeable memory",
414 T_META_LTEPHASE(LTE_POSTINIT))
415 {
416 uint64_t footprint_before, pagetable_before;
417 uint64_t footprint_after, pagetable_after;
418 uint64_t footprint_expected;
419 kern_return_t kr;
420 mach_vm_address_t pre_vm_addr, vm_addr;
421 mach_vm_size_t vm_size, dirty_size;
422 int state;
423
424 /* pre-warm to account for page table expansion */
425 pre_vm_addr = pre_warm(MEM_SIZE);
426
427 /* allocating purgeable virtual memory... */
428 get_ledger_info(&footprint_before, &pagetable_before);
429 vm_addr = 0;
430 vm_size = MEM_SIZE;
431 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
432 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
433 T_QUIET;
434 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
435 kr, mach_error_string(kr));
436 T_QUIET;
437 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
438 /* ... should not change footprint */
439 get_ledger_info(&footprint_after, &pagetable_after);
440 footprint_expected = footprint_before;
441 footprint_expected += (pagetable_after - pagetable_before);
442 T_LOG("purgeable virtual allocation does not change phys_footprint");
443 T_EXPECT_EQ(footprint_after, footprint_expected,
444 "purgeable virtual allocation of %lld bytes: "
445 "footprint %lld -> %lld expected %lld delta %lld",
446 vm_size, footprint_before, footprint_after,
447 footprint_expected, footprint_after - footprint_expected);
448
449 /* touching memory... */
450 get_ledger_info(&footprint_before, &pagetable_before);
451 dirty_size = vm_size / 2;
452 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
453 /* ... should increase footprint */
454 get_ledger_info(&footprint_after, &pagetable_after);
455 footprint_expected = footprint_before + dirty_size;
456 footprint_expected += (pagetable_after - pagetable_before);
457 T_LOG("modifying anonymous memory increases phys_footprint");
458 T_EXPECT_EQ(footprint_after, footprint_expected,
459 "touched %lld bytes: "
460 "footprint %lld -> %lld expected %lld delta %lld",
461 dirty_size, footprint_before, footprint_after,
462 footprint_expected, footprint_after - footprint_expected);
463
464 /* making it volatile... */
465 get_ledger_info(&footprint_before, &pagetable_before);
466 state = VM_PURGABLE_VOLATILE;
467 T_QUIET;
468 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
469 vm_addr,
470 VM_PURGABLE_SET_STATE,
471 &state),
472 KERN_SUCCESS,
473 "vm_purgable_control(VOLATILE)");
474 T_QUIET;
475 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
476 "memory was non-volatile");
477 /* ... should decrease footprint */
478 get_ledger_info(&footprint_after, &pagetable_after);
479 footprint_expected = footprint_before - dirty_size;
480 footprint_expected += (pagetable_after - pagetable_before);
481 T_LOG("making volatile decreases phys_footprint");
482 T_EXPECT_EQ(footprint_after, footprint_expected,
483 "made volatile %lld dirty bytes: "
484 "footprint %lld -> %lld expected %lld delta %lld",
485 dirty_size, footprint_before, footprint_after,
486 footprint_expected, footprint_after - footprint_expected);
487
488 /* making it non-volatile... */
489 get_ledger_info(&footprint_before, &pagetable_before);
490 state = VM_PURGABLE_NONVOLATILE;
491 T_QUIET;
492 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
493 vm_addr,
494 VM_PURGABLE_SET_STATE,
495 &state),
496 KERN_SUCCESS,
497 "vm_purgable_control(NONVOLATILE)");
498 T_QUIET;
499 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
500 "memory was volatile");
501 /* ... should increase footprint */
502 get_ledger_info(&footprint_after, &pagetable_after);
503 footprint_expected = footprint_before + dirty_size;
504 footprint_expected += (pagetable_after - pagetable_before);
505 T_LOG("making non-volatile increases phys_footprint");
506 T_EXPECT_EQ(footprint_after, footprint_expected,
507 "made non-volatile %lld dirty bytes: "
508 "footprint %lld -> %lld expected %lld delta %lld",
509 dirty_size, footprint_before, footprint_after,
510 footprint_expected, footprint_after - footprint_expected);
511
512 /* deallocating memory... */
513 get_ledger_info(&footprint_before, &pagetable_before);
514 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
515 T_QUIET;
516 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
517 kr, mach_error_string(kr));
518 /* ... should decrease footprint */
519 get_ledger_info(&footprint_after, &pagetable_after);
520 footprint_expected = footprint_before - dirty_size;
521 footprint_expected += (pagetable_after - pagetable_before);
522 T_LOG("deallocating memory decreases phys_footprint");
523 T_EXPECT_EQ(footprint_after, footprint_expected,
524 "deallocated %lld dirty bytes: "
525 "footprint %lld -> %lld expected %lld delta %lld",
526 dirty_size, footprint_before, footprint_after,
527 footprint_expected, footprint_after - footprint_expected);
528 }
529
530 T_DECL(phys_footprint_purgeable_ownership,
531 "phys_footprint for owned purgeable memory",
532 T_META_LTEPHASE(LTE_POSTINIT))
533 {
534 uint64_t footprint_before, pagetable_before;
535 uint64_t footprint_after, pagetable_after;
536 uint64_t footprint_expected;
537 kern_return_t kr;
538 mach_vm_address_t pre_vm_addr, vm_addr;
539 mach_vm_size_t vm_size, dirty_size, me_size;
540 int state;
541 mach_port_t me_port;
542
543 /* pre-warm to account for page table expansion */
544 pre_vm_addr = pre_warm(MEM_SIZE);
545
546 /* allocating purgeable virtual memory... */
547 get_ledger_info(&footprint_before, &pagetable_before);
548 vm_addr = 0;
549 vm_size = MEM_SIZE;
550 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
551 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
552 T_QUIET;
553 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
554 kr, mach_error_string(kr));
555 T_QUIET;
556 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
557 /* ... should not change footprint */
558 get_ledger_info(&footprint_after, &pagetable_after);
559 footprint_expected = footprint_before;
560 footprint_expected += (pagetable_after - pagetable_before);
561 T_LOG("purgeable virtual allocation does not change phys_footprint");
562 T_EXPECT_EQ(footprint_after, footprint_expected,
563 "purgeable virtual allocation of %lld bytes: "
564 "footprint %lld -> %lld expected %lld delta %lld",
565 vm_size, footprint_before, footprint_after,
566 footprint_expected, footprint_after - footprint_expected);
567
568 /* touching memory... */
569 get_ledger_info(&footprint_before, &pagetable_before);
570 dirty_size = vm_size / 2;
571 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
572 /* ... should increase footprint */
573 get_ledger_info(&footprint_after, &pagetable_after);
574 footprint_expected = footprint_before + dirty_size;
575 footprint_expected += (pagetable_after - pagetable_before);
576 T_LOG("modifying anonymous memory increases phys_footprint");
577 T_EXPECT_EQ(footprint_after, footprint_expected,
578 "touched %lld bytes: "
579 "footprint %lld -> %lld expected %lld delta %lld",
580 dirty_size, footprint_before, footprint_after,
581 footprint_expected, footprint_after - footprint_expected);
582
583 /* making it volatile... */
584 get_ledger_info(&footprint_before, &pagetable_before);
585 state = VM_PURGABLE_VOLATILE;
586 T_QUIET;
587 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
588 vm_addr,
589 VM_PURGABLE_SET_STATE,
590 &state),
591 KERN_SUCCESS,
592 "vm_purgable_control(VOLATILE)");
593 T_QUIET;
594 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
595 "memory was non-volatile");
596 /* ... should decrease footprint */
597 get_ledger_info(&footprint_after, &pagetable_after);
598 footprint_expected = footprint_before - dirty_size;
599 footprint_expected += (pagetable_after - pagetable_before);
600 T_LOG("making volatile decreases phys_footprint");
601 T_EXPECT_EQ(footprint_after, footprint_expected,
602 "made volatile %lld dirty bytes: "
603 "footprint %lld -> %lld expected %lld delta %lld",
604 dirty_size, footprint_before, footprint_after,
605 footprint_expected, footprint_after - footprint_expected);
606
607 /* making it non-volatile... */
608 get_ledger_info(&footprint_before, &pagetable_before);
609 state = VM_PURGABLE_NONVOLATILE;
610 T_QUIET;
611 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
612 vm_addr,
613 VM_PURGABLE_SET_STATE,
614 &state),
615 KERN_SUCCESS,
616 "vm_purgable_control(NONVOLATILE)");
617 T_QUIET;
618 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
619 "memory was volatile");
620 /* ... should increase footprint */
621 get_ledger_info(&footprint_after, &pagetable_after);
622 footprint_expected = footprint_before + dirty_size;
623 footprint_expected += (pagetable_after - pagetable_before);
624 T_LOG("making non-volatile increases phys_footprint");
625 T_EXPECT_EQ(footprint_after, footprint_expected,
626 "made non-volatile %lld dirty bytes: "
627 "footprint %lld -> %lld expected %lld delta %lld",
628 dirty_size, footprint_before, footprint_after,
629 footprint_expected, footprint_after - footprint_expected);
630
631 /* making a memory entry... */
632 get_ledger_info(&footprint_before, &pagetable_before);
633 me_size = vm_size;
634 me_port = MACH_PORT_NULL;
635 kr = mach_make_memory_entry_64(mach_task_self(),
636 &me_size,
637 vm_addr,
638 VM_PROT_READ | VM_PROT_WRITE,
639 &me_port,
640 MACH_PORT_NULL);
641 T_QUIET;
642 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
643 kr, mach_error_string(kr));
644 T_QUIET;
645 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
646 /* ... should not change footprint */
647 get_ledger_info(&footprint_after, &pagetable_after);
648 footprint_expected = footprint_before;
649 footprint_expected += (pagetable_after - pagetable_before);
650 T_LOG("making a memory entry does not change phys_footprint");
651 T_EXPECT_EQ(footprint_after, footprint_expected,
652 "making a memory entry of %lld bytes: "
653 "footprint %lld -> %lld expected %lld delta %lld",
654 vm_size, footprint_before, footprint_after,
655 footprint_expected, footprint_after - footprint_expected);
656
657 /* deallocating memory while holding memory entry... */
658 get_ledger_info(&footprint_before, &pagetable_before);
659 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
660 T_QUIET;
661 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
662 kr, mach_error_string(kr));
663 /* ... should not change footprint */
664 get_ledger_info(&footprint_after, &pagetable_after);
665 footprint_expected = footprint_before;
666 footprint_expected += (pagetable_after - pagetable_before);
667 T_LOG("deallocating owned memory while holding memory entry "
668 "does not change phys_footprint");
669 T_EXPECT_EQ(footprint_after, footprint_expected,
670 "deallocated %lld dirty bytes: "
671 "footprint %lld -> %lld expected %lld delta %lld",
672 dirty_size, footprint_before, footprint_after,
673 footprint_expected, footprint_after - footprint_expected);
674
675 get_ledger_info(&footprint_after, &pagetable_after);
676 footprint_expected = footprint_before;
677 footprint_expected += (pagetable_after - pagetable_before);
678 T_LOG("deallocating owned memory while holding memory entry "
679 "does not change phys_footprint");
680 T_EXPECT_EQ(footprint_after, footprint_expected,
681 "deallocated %lld dirty bytes: "
682 "footprint %lld -> %lld expected %lld delta %lld",
683 dirty_size, footprint_before, footprint_after,
684 footprint_expected, footprint_after - footprint_expected);
685
686 /* changing ownership of memory entry (from self to self)... */
687 get_ledger_info(&footprint_before, &pagetable_before);
688 kr = mach_memory_entry_ownership(me_port,
689 #if VM_LEDGER_TAG_UNCHANGED
690 MACH_PORT_NULL, /* owner remains unchanged */
691 VM_LEDGER_TAG_UNCHANGED, /* ledger-tag remains unchanged */
692 #else /* VM_LEDGER_TAG_UNCHANGED */
693 mach_task_self(),
694 VM_LEDGER_TAG_DEFAULT,
695 #endif /* VM_LEDGER_TAG_UNCHANGED */
696 0);
697 T_QUIET;
698 T_EXPECT_EQ(kr, KERN_SUCCESS,
699 "mach_memory_entry_ownership() error 0x%x (%s)",
700 kr, mach_error_string(kr));
701 /* ... should not change footprint */
702 get_ledger_info(&footprint_after, &pagetable_after);
703 footprint_expected = footprint_before;
704 footprint_expected += (pagetable_after - pagetable_before);
705 T_LOG("changing ownership from self to self "
706 "does not change phys_footprint");
707 T_EXPECT_EQ(footprint_after, footprint_expected,
708 "changed ownership of %lld dirty bytes: "
709 "footprint %lld -> %lld expected %lld delta %lld",
710 dirty_size, footprint_before, footprint_after,
711 footprint_expected, footprint_after - footprint_expected);
712
713 /* releasing the memory entry... */
714 kr = mach_port_deallocate(mach_task_self(), me_port);
715 T_QUIET;
716 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
717 kr, mach_error_string(kr));
718 /* ... should decrease footprint */
719 get_ledger_info(&footprint_after, &pagetable_after);
720 footprint_expected = footprint_before - dirty_size;
721 footprint_expected += (pagetable_after - pagetable_before);
722 T_LOG("releasing memory entry decreases phys_footprint");
723 T_EXPECT_EQ(footprint_after, footprint_expected,
724 "made volatile %lld dirty bytes: "
725 "footprint %lld -> %lld expected %lld delta %lld",
726 dirty_size, footprint_before, footprint_after,
727 footprint_expected, footprint_after - footprint_expected);
728 }
729
730 #if VM_LEDGER_TAG_UNCHANGED
731 T_DECL(phys_footprint_multiple_purgeable_ownership,
732 "phys_footprint for owned purgeable memory (MAP_MEM_VM_SHARE)",
733 T_META_NAMESPACE(TEST_VM_NAMESPACE),
734 T_META_LTEPHASE(LTE_POSTINIT))
735 {
736 uint64_t footprint_before, pagetable_before;
737 uint64_t footprint_after, pagetable_after;
738 uint64_t footprint_expected;
739 kern_return_t kr;
740 mach_vm_address_t pre_vm_addr, vm_addr;
741 mach_vm_size_t vm_size, dirty_size, me_size;
742 int state;
743 mach_port_t me_port;
744
745 /* pre-warm to account for page table expansion */
746 pre_vm_addr = pre_warm(MEM_SIZE);
747
748 /* allocating purgeable virtual memory... */
749 get_ledger_info(&footprint_before, &pagetable_before);
750 vm_addr = 0;
751 vm_size = MEM_SIZE;
752 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
753 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
754 T_QUIET;
755 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
756 kr, mach_error_string(kr));
757 T_QUIET;
758 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
759 /* ... should not change footprint */
760 get_ledger_info(&footprint_after, &pagetable_after);
761 footprint_expected = footprint_before;
762 footprint_expected += (pagetable_after - pagetable_before);
763 T_LOG("purgeable virtual allocation does not change phys_footprint");
764 T_EXPECT_EQ(footprint_after, footprint_expected,
765 "purgeable virtual allocation of %lld bytes: "
766 "footprint %lld -> %lld expected %lld delta %lld",
767 vm_size, footprint_before, footprint_after,
768 footprint_expected, footprint_after - footprint_expected);
769
770 /* touching memory... */
771 get_ledger_info(&footprint_before, &pagetable_before);
772 dirty_size = vm_size / 2;
773 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
774 /* ... should increase footprint */
775 get_ledger_info(&footprint_after, &pagetable_after);
776 footprint_expected = footprint_before + dirty_size;
777 footprint_expected += (pagetable_after - pagetable_before);
778 T_LOG("modifying anonymous memory increases phys_footprint");
779 T_EXPECT_EQ(footprint_after, footprint_expected,
780 "touched %lld bytes: "
781 "footprint %lld -> %lld expected %lld delta %lld",
782 dirty_size, footprint_before, footprint_after,
783 footprint_expected, footprint_after - footprint_expected);
784
785 /* making it volatile... */
786 get_ledger_info(&footprint_before, &pagetable_before);
787 state = VM_PURGABLE_VOLATILE;
788 T_QUIET;
789 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
790 vm_addr,
791 VM_PURGABLE_SET_STATE,
792 &state),
793 KERN_SUCCESS,
794 "vm_purgable_control(VOLATILE)");
795 T_QUIET;
796 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
797 "memory was non-volatile");
798 /* ... should decrease footprint */
799 get_ledger_info(&footprint_after, &pagetable_after);
800 footprint_expected = footprint_before - dirty_size;
801 footprint_expected += (pagetable_after - pagetable_before);
802 T_LOG("making volatile decreases phys_footprint");
803 T_EXPECT_EQ(footprint_after, footprint_expected,
804 "made volatile %lld dirty bytes: "
805 "footprint %lld -> %lld expected %lld delta %lld",
806 dirty_size, footprint_before, footprint_after,
807 footprint_expected, footprint_after - footprint_expected);
808
809 /* making it non-volatile... */
810 get_ledger_info(&footprint_before, &pagetable_before);
811 state = VM_PURGABLE_NONVOLATILE;
812 T_QUIET;
813 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
814 vm_addr,
815 VM_PURGABLE_SET_STATE,
816 &state),
817 KERN_SUCCESS,
818 "vm_purgable_control(NONVOLATILE)");
819 T_QUIET;
820 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
821 "memory was volatile");
822 /* ... should increase footprint */
823 get_ledger_info(&footprint_after, &pagetable_after);
824 footprint_expected = footprint_before + dirty_size;
825 footprint_expected += (pagetable_after - pagetable_before);
826 T_LOG("making non-volatile increases phys_footprint");
827 T_EXPECT_EQ(footprint_after, footprint_expected,
828 "made non-volatile %lld dirty bytes: "
829 "footprint %lld -> %lld expected %lld delta %lld",
830 dirty_size, footprint_before, footprint_after,
831 footprint_expected, footprint_after - footprint_expected);
832
833 /* making a memory entry... */
834 get_ledger_info(&footprint_before, &pagetable_before);
835 me_size = vm_size;
836 me_port = MACH_PORT_NULL;
837 kr = mach_make_memory_entry_64(mach_task_self(),
838 &me_size,
839 vm_addr,
840 MAP_MEM_VM_SHARE | VM_PROT_READ | VM_PROT_WRITE,
841 &me_port,
842 MACH_PORT_NULL);
843 T_QUIET;
844 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
845 kr, mach_error_string(kr));
846 T_QUIET;
847 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
848 /* ... should not change footprint */
849 get_ledger_info(&footprint_after, &pagetable_after);
850 footprint_expected = footprint_before;
851 footprint_expected += (pagetable_after - pagetable_before);
852 T_LOG("making a memory entry does not change phys_footprint");
853 T_EXPECT_EQ(footprint_after, footprint_expected,
854 "making a memory entry of %lld bytes: "
855 "footprint %lld -> %lld expected %lld delta %lld",
856 vm_size, footprint_before, footprint_after,
857 footprint_expected, footprint_after - footprint_expected);
858
859 /* deallocating memory while holding memory entry... */
860 get_ledger_info(&footprint_before, &pagetable_before);
861 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
862 T_QUIET;
863 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
864 kr, mach_error_string(kr));
865 /* ... should not change footprint */
866 get_ledger_info(&footprint_after, &pagetable_after);
867 footprint_expected = footprint_before;
868 footprint_expected += (pagetable_after - pagetable_before);
869 T_LOG("deallocating owned memory while holding memory entry "
870 "does not change phys_footprint");
871 T_EXPECT_EQ(footprint_after, footprint_expected,
872 "deallocated %lld dirty bytes: "
873 "footprint %lld -> %lld expected %lld delta %lld",
874 dirty_size, footprint_before, footprint_after,
875 footprint_expected, footprint_after - footprint_expected);
876
877 /* changing ownership of memory entry (from self to self)... */
878 get_ledger_info(&footprint_before, &pagetable_before);
879 kr = mach_memory_entry_ownership(me_port,
880 MACH_PORT_NULL, /* owner remains unchanged */
881 VM_LEDGER_TAG_UNCHANGED, /* ledger-tag remains unchanged */
882 0);
883 T_QUIET;
884 T_EXPECT_EQ(kr, KERN_SUCCESS,
885 "mach_memory_entry_ownership() error 0x%x (%s)",
886 kr, mach_error_string(kr));
887 /* ... should not change footprint */
888 get_ledger_info(&footprint_after, &pagetable_after);
889 footprint_expected = footprint_before;
890 footprint_expected += (pagetable_after - pagetable_before);
891 T_LOG("changing ownership from self to self "
892 "does not change phys_footprint");
893 T_EXPECT_EQ(footprint_after, footprint_expected,
894 "changed ownership of %lld dirty bytes: "
895 "footprint %lld -> %lld expected %lld delta %lld",
896 dirty_size, footprint_before, footprint_after,
897 footprint_expected, footprint_after - footprint_expected);
898
899 /* releasing the memory entry... */
900 kr = mach_port_deallocate(mach_task_self(), me_port);
901 T_QUIET;
902 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
903 kr, mach_error_string(kr));
904 /* ... should decrease footprint */
905 get_ledger_info(&footprint_after, &pagetable_after);
906 footprint_expected = footprint_before - dirty_size;
907 footprint_expected += (pagetable_after - pagetable_before);
908 T_LOG("releasing memory entry decreases phys_footprint");
909 T_EXPECT_EQ(footprint_after, footprint_expected,
910 "made volatile %lld dirty bytes: "
911 "footprint %lld -> %lld expected %lld delta %lld",
912 dirty_size, footprint_before, footprint_after,
913 footprint_expected, footprint_after - footprint_expected);
914 }
915 #endif /* VM_LEDGER_TAG_UNCHANGED */
916
917 #ifdef MAP_MEM_LEDGER_TAGGED
918 T_DECL(phys_footprint_ledger_purgeable_owned,
919 "phys_footprint for ledger-tagged purgeable memory ownership",
920 T_META_LTEPHASE(LTE_POSTINIT))
921 {
922 uint64_t footprint_before, pagetable_before;
923 uint64_t footprint_after, pagetable_after;
924 uint64_t footprint_expected;
925 kern_return_t kr;
926 mach_vm_address_t pre_vm_addr, vm_addr;
927 mach_vm_size_t vm_size, dirty_size, me_size;
928 int state;
929 mach_port_t me_port;
930
931 /* pre-warm to account for page table expansion */
932 pre_vm_addr = pre_warm(MEM_SIZE);
933
934 /* making a memory entry... */
935 get_ledger_info(&footprint_before, &pagetable_before);
936 vm_size = MEM_SIZE;
937 me_size = vm_size;
938 me_port = MACH_PORT_NULL;
939 kr = mach_make_memory_entry_64(mach_task_self(),
940 &me_size,
941 0,
942 (MAP_MEM_NAMED_CREATE |
943 MAP_MEM_LEDGER_TAGGED |
944 MAP_MEM_PURGABLE |
945 VM_PROT_READ | VM_PROT_WRITE),
946 &me_port,
947 MACH_PORT_NULL);
948 T_QUIET;
949 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
950 kr, mach_error_string(kr));
951 T_QUIET;
952 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
953 /* ... should not change footprint */
954 get_ledger_info(&footprint_after, &pagetable_after);
955 footprint_expected = footprint_before;
956 footprint_expected += (pagetable_after - pagetable_before);
957 T_LOG("making a memory entry does not change phys_footprint");
958 T_EXPECT_EQ(footprint_after, footprint_expected,
959 "making a memory entry of %lld bytes: "
960 "footprint %lld -> %lld expected %lld delta %lld",
961 vm_size, footprint_before, footprint_after,
962 footprint_expected, footprint_after - footprint_expected);
963
964 /* mapping ledger-tagged virtual memory... */
965 get_ledger_info(&footprint_before, &pagetable_before);
966 vm_addr = 0;
967 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
968 0, /* mask */
969 VM_FLAGS_ANYWHERE,
970 me_port,
971 0, /* offset */
972 FALSE, /* copy */
973 VM_PROT_READ | VM_PROT_WRITE,
974 VM_PROT_READ | VM_PROT_WRITE,
975 VM_INHERIT_DEFAULT);
976 T_QUIET;
977 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
978 kr, mach_error_string(kr));
979 T_QUIET;
980 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
981 /* ... should not change footprint */
982 get_ledger_info(&footprint_after, &pagetable_after);
983 footprint_expected = footprint_before;
984 footprint_expected += (pagetable_after - pagetable_before);
985 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
986 T_EXPECT_EQ(footprint_after, footprint_expected,
987 "ledger-tagged mapping of %lld bytes: "
988 "footprint %lld -> %lld expected %lld delta %lld",
989 vm_size, footprint_before, footprint_after,
990 footprint_expected, footprint_after - footprint_expected);
991
992 /* touching memory... */
993 get_ledger_info(&footprint_before, &pagetable_before);
994 dirty_size = vm_size / 2;
995 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
996 /* ... should increase footprint */
997 get_ledger_info(&footprint_after, &pagetable_after);
998 footprint_expected = footprint_before + dirty_size;
999 footprint_expected += (pagetable_after - pagetable_before);
1000 T_LOG("modifying ledger-tagged memory increases phys_footprint");
1001 T_EXPECT_EQ(footprint_after, footprint_expected,
1002 "touched %lld bytes: "
1003 "footprint %lld -> %lld expected %lld delta %lld",
1004 dirty_size, footprint_before, footprint_after,
1005 footprint_expected, footprint_after - footprint_expected);
1006
1007 /* making it volatile... */
1008 get_ledger_info(&footprint_before, &pagetable_before);
1009 state = VM_PURGABLE_VOLATILE;
1010 T_QUIET;
1011 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
1012 vm_addr,
1013 VM_PURGABLE_SET_STATE,
1014 &state),
1015 KERN_SUCCESS,
1016 "vm_purgable_control(VOLATILE)");
1017 T_QUIET;
1018 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
1019 "memory was non-volatile");
1020 /* ... should decrease footprint */
1021 get_ledger_info(&footprint_after, &pagetable_after);
1022 footprint_expected = footprint_before - dirty_size;
1023 footprint_expected += (pagetable_after - pagetable_before);
1024 T_LOG("making volatile decreases phys_footprint");
1025 T_EXPECT_EQ(footprint_after, footprint_expected,
1026 "made volatile %lld dirty bytes: "
1027 "footprint %lld -> %lld expected %lld delta %lld",
1028 dirty_size, footprint_before, footprint_after,
1029 footprint_expected, footprint_after - footprint_expected);
1030
1031 /* making it non-volatile... */
1032 get_ledger_info(&footprint_before, &pagetable_before);
1033 state = VM_PURGABLE_NONVOLATILE;
1034 T_QUIET;
1035 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
1036 vm_addr,
1037 VM_PURGABLE_SET_STATE,
1038 &state),
1039 KERN_SUCCESS,
1040 "vm_purgable_control(NONVOLATILE)");
1041 T_QUIET;
1042 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
1043 "memory was volatile");
1044 /* ... should increase footprint */
1045 get_ledger_info(&footprint_after, &pagetable_after);
1046 footprint_expected = footprint_before + dirty_size;
1047 footprint_expected += (pagetable_after - pagetable_before);
1048 T_LOG("making non-volatile increases phys_footprint");
1049 T_EXPECT_EQ(footprint_after, footprint_expected,
1050 "made non-volatile %lld dirty bytes: "
1051 "footprint %lld -> %lld expected %lld delta %lld",
1052 dirty_size, footprint_before, footprint_after,
1053 footprint_expected, footprint_after - footprint_expected);
1054
1055 /* deallocating memory while holding memory entry... */
1056 get_ledger_info(&footprint_before, &pagetable_before);
1057 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1058 T_QUIET;
1059 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1060 kr, mach_error_string(kr));
1061 /* ... should not change footprint */
1062 get_ledger_info(&footprint_after, &pagetable_after);
1063 footprint_expected = footprint_before;
1064 footprint_expected += (pagetable_after - pagetable_before);
1065 T_LOG("deallocating owned memory while holding memory entry "
1066 "does not change phys_footprint");
1067 T_EXPECT_EQ(footprint_after, footprint_expected,
1068 "deallocated %lld dirty bytes: "
1069 "footprint %lld -> %lld expected %lld delta %lld",
1070 dirty_size, footprint_before, footprint_after,
1071 footprint_expected, footprint_after - footprint_expected);
1072
1073 /* releasing the memory entry... */
1074 kr = mach_port_deallocate(mach_task_self(), me_port);
1075 T_QUIET;
1076 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1077 kr, mach_error_string(kr));
1078 /* ... should decrease footprint */
1079 get_ledger_info(&footprint_after, &pagetable_after);
1080 footprint_expected = footprint_before - dirty_size;
1081 footprint_expected += (pagetable_after - pagetable_before);
1082 T_LOG("releasing memory entry decreases phys_footprint");
1083 T_EXPECT_EQ(footprint_after, footprint_expected,
1084 "made volatile %lld dirty bytes: "
1085 "footprint %lld -> %lld expected %lld delta %lld",
1086 dirty_size, footprint_before, footprint_after,
1087 footprint_expected, footprint_after - footprint_expected);
1088 }
1089
1090 T_DECL(phys_footprint_ledger_owned,
1091 "phys_footprint for ledger-tagged memory ownership",
1092 T_META_LTEPHASE(LTE_POSTINIT))
1093 {
1094 uint64_t footprint_before, pagetable_before;
1095 uint64_t footprint_after, pagetable_after;
1096 uint64_t footprint_expected;
1097 kern_return_t kr;
1098 mach_vm_address_t pre_vm_addr, vm_addr;
1099 mach_vm_size_t vm_size, dirty_size, me_size;
1100 mach_port_t me_port;
1101
1102 /* pre-warm to account for page table expansion */
1103 pre_vm_addr = pre_warm(MEM_SIZE);
1104
1105 /* making a memory entry... */
1106 get_ledger_info(&footprint_before, &pagetable_before);
1107 vm_size = MEM_SIZE;
1108 me_size = vm_size;
1109 me_port = MACH_PORT_NULL;
1110 kr = mach_make_memory_entry_64(mach_task_self(),
1111 &me_size,
1112 0,
1113 (MAP_MEM_NAMED_CREATE |
1114 MAP_MEM_LEDGER_TAGGED |
1115 VM_PROT_READ | VM_PROT_WRITE),
1116 &me_port,
1117 MACH_PORT_NULL);
1118 T_QUIET;
1119 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1120 kr, mach_error_string(kr));
1121 T_QUIET;
1122 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1123 /* ... should not change footprint */
1124 get_ledger_info(&footprint_after, &pagetable_after);
1125 footprint_expected = footprint_before;
1126 footprint_expected += (pagetable_after - pagetable_before);
1127 T_LOG("making a memory entry does not change phys_footprint");
1128 T_EXPECT_EQ(footprint_after, footprint_expected,
1129 "making a memory entry of %lld bytes: "
1130 "footprint %lld -> %lld expected %lld delta %lld",
1131 vm_size, footprint_before, footprint_after,
1132 footprint_expected, footprint_after - footprint_expected);
1133
1134 /* mapping ledger-tagged virtual memory... */
1135 get_ledger_info(&footprint_before, &pagetable_before);
1136 vm_addr = 0;
1137 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1138 0, /* mask */
1139 VM_FLAGS_ANYWHERE,
1140 me_port,
1141 0, /* offset */
1142 FALSE, /* copy */
1143 VM_PROT_READ | VM_PROT_WRITE,
1144 VM_PROT_READ | VM_PROT_WRITE,
1145 VM_INHERIT_DEFAULT);
1146 T_QUIET;
1147 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1148 kr, mach_error_string(kr));
1149 T_QUIET;
1150 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1151 /* ... should not change footprint */
1152 get_ledger_info(&footprint_after, &pagetable_after);
1153 footprint_expected = footprint_before;
1154 footprint_expected += (pagetable_after - pagetable_before);
1155 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1156 T_EXPECT_EQ(footprint_after, footprint_expected,
1157 "ledger-tagged mapping of %lld bytes: "
1158 "footprint %lld -> %lld expected %lld delta %lld",
1159 vm_size, footprint_before, footprint_after,
1160 footprint_expected, footprint_after - footprint_expected);
1161
1162 /* touching memory... */
1163 get_ledger_info(&footprint_before, &pagetable_before);
1164 dirty_size = vm_size / 2;
1165 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1166 /* ... should increase footprint */
1167 get_ledger_info(&footprint_after, &pagetable_after);
1168 footprint_expected = footprint_before + dirty_size;
1169 footprint_expected += (pagetable_after - pagetable_before);
1170 T_LOG("modifying ledger-tagged memory increases phys_footprint");
1171 T_EXPECT_EQ(footprint_after, footprint_expected,
1172 "touched %lld bytes: "
1173 "footprint %lld -> %lld expected %lld delta %lld",
1174 dirty_size, footprint_before, footprint_after,
1175 footprint_expected, footprint_after - footprint_expected);
1176
1177 /* deallocating memory while holding memory entry... */
1178 get_ledger_info(&footprint_before, &pagetable_before);
1179 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1180 T_QUIET;
1181 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1182 kr, mach_error_string(kr));
1183 /* ... should not change footprint */
1184 get_ledger_info(&footprint_after, &pagetable_after);
1185 footprint_expected = footprint_before;
1186 footprint_expected += (pagetable_after - pagetable_before);
1187 T_LOG("deallocating owned memory while holding memory entry "
1188 "does not change phys_footprint");
1189 T_EXPECT_EQ(footprint_after, footprint_expected,
1190 "deallocated %lld dirty bytes: "
1191 "footprint %lld -> %lld expected %lld delta %lld",
1192 dirty_size, footprint_before, footprint_after,
1193 footprint_expected, footprint_after - footprint_expected);
1194
1195 /* releasing the memory entry... */
1196 kr = mach_port_deallocate(mach_task_self(), me_port);
1197 T_QUIET;
1198 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1199 kr, mach_error_string(kr));
1200 /* ... should decrease footprint */
1201 get_ledger_info(&footprint_after, &pagetable_after);
1202 footprint_expected = footprint_before - dirty_size;
1203 footprint_expected += (pagetable_after - pagetable_before);
1204 T_LOG("releasing memory entry decreases phys_footprint");
1205 T_EXPECT_EQ(footprint_after, footprint_expected,
1206 "made volatile %lld dirty bytes: "
1207 "footprint %lld -> %lld expected %lld delta %lld",
1208 dirty_size, footprint_before, footprint_after,
1209 footprint_expected, footprint_after - footprint_expected);
1210 }
1211
1212 T_DECL(phys_footprint_no_footprint_for_debug,
1213 "phys_footprint for no_footprint_for_debug",
1214 T_META_LTEPHASE(LTE_POSTINIT))
1215 {
1216 uint64_t footprint_before, pagetable_before;
1217 uint64_t footprint_after, pagetable_after;
1218 uint64_t footprint_expected;
1219 kern_return_t kr;
1220 mach_vm_address_t pre_vm_addr, vm_addr;
1221 mach_vm_size_t vm_size, dirty_size, me_size;
1222 mach_port_t me_port;
1223 int new_value, ret;
1224
1225 /* pre-warm to account for page table expansion */
1226 pre_vm_addr = pre_warm(MEM_SIZE);
1227
1228 /* making a memory entry... */
1229 get_ledger_info(&footprint_before, &pagetable_before);
1230 vm_size = MEM_SIZE;
1231 me_size = vm_size;
1232 me_port = MACH_PORT_NULL;
1233 kr = mach_make_memory_entry_64(mach_task_self(),
1234 &me_size,
1235 0,
1236 (MAP_MEM_NAMED_CREATE |
1237 MAP_MEM_LEDGER_TAGGED |
1238 VM_PROT_READ | VM_PROT_WRITE),
1239 &me_port,
1240 MACH_PORT_NULL);
1241 T_QUIET;
1242 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1243 kr, mach_error_string(kr));
1244 T_QUIET;
1245 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1246 /* ... should not change footprint */
1247 get_ledger_info(&footprint_after, &pagetable_after);
1248 footprint_expected = footprint_before;
1249 footprint_expected += (pagetable_after - pagetable_before);
1250 T_LOG("making a memory entry does not change phys_footprint");
1251 T_EXPECT_EQ(footprint_after, footprint_expected,
1252 "making a memory entry of %lld bytes: "
1253 "footprint %lld -> %lld expected %lld delta %lld",
1254 vm_size, footprint_before, footprint_after,
1255 footprint_expected, footprint_after - footprint_expected);
1256
1257 /* trying to hide debug memory from footprint while not allowed */
1258 kr = mach_memory_entry_ownership(me_port,
1259 mach_task_self(),
1260 VM_LEDGER_TAG_DEFAULT,
1261 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1262 T_QUIET;
1263 T_EXPECT_EQ(kr, KERN_NO_ACCESS,
1264 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) fails without sysctl");
1265
1266 /* let's get permission to hide debug memory */
1267 new_value = 1;
1268 ret = sysctlbyname("vm.task_no_footprint_for_debug", NULL, NULL, &new_value, sizeof(new_value));
1269 if (ret == -1 && errno == ENOENT) {
1270 T_SKIP("sysctlbyname(vm.task_no_footprint_for_debug) ENOENT");
1271 }
1272 T_QUIET;
1273 T_ASSERT_POSIX_SUCCESS(ret, "sysctlbyname(vm.task_no_footprint_for_debug)");
1274
1275 /* trying to hide debug memory from footprint while allowed */
1276 task_id_token_t token;
1277 kr = task_create_identity_token(mach_task_self(), &token);
1278 T_QUIET;
1279 T_ASSERT_EQ(kr, KERN_SUCCESS,
1280 "task_create_identity_token");
1281
1282 kr = mach_memory_entry_ownership(me_port,
1283 token, /* this time also try calling it with id token */
1284 VM_LEDGER_TAG_DEFAULT,
1285 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1286 T_QUIET;
1287 T_EXPECT_EQ(kr, KERN_SUCCESS,
1288 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) succeeds after sysctl");
1289 mach_port_deallocate(mach_task_self(), token);
1290
1291 /* mapping ledger-tagged virtual memory... */
1292 get_ledger_info(&footprint_before, &pagetable_before);
1293 vm_addr = 0;
1294 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1295 0, /* mask */
1296 VM_FLAGS_ANYWHERE,
1297 me_port,
1298 0, /* offset */
1299 FALSE, /* copy */
1300 VM_PROT_READ | VM_PROT_WRITE,
1301 VM_PROT_READ | VM_PROT_WRITE,
1302 VM_INHERIT_DEFAULT);
1303 T_QUIET;
1304 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1305 kr, mach_error_string(kr));
1306 T_QUIET;
1307 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1308 /* ... should not change footprint */
1309 get_ledger_info(&footprint_after, &pagetable_after);
1310 footprint_expected = footprint_before;
1311 footprint_expected += (pagetable_after - pagetable_before);
1312 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1313 T_EXPECT_EQ(footprint_after, footprint_expected,
1314 "ledger-tagged mapping of %lld bytes: "
1315 "footprint %lld -> %lld expected %lld delta %lld",
1316 vm_size, footprint_before, footprint_after,
1317 footprint_expected, footprint_after - footprint_expected);
1318
1319 /* touching memory... */
1320 get_ledger_info(&footprint_before, &pagetable_before);
1321 dirty_size = vm_size / 2;
1322 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1323 /* ... should not increase footprint */
1324 get_ledger_info(&footprint_after, &pagetable_after);
1325 footprint_expected = footprint_before;
1326 footprint_expected += (pagetable_after - pagetable_before);
1327 T_LOG("modifying no_footprint_for_debug memory does not increase phys_footprint");
1328 T_EXPECT_EQ(footprint_after, footprint_expected,
1329 "touched %lld bytes: "
1330 "footprint %lld -> %lld expected %lld delta %lld",
1331 dirty_size, footprint_before, footprint_after,
1332 footprint_expected, footprint_after - footprint_expected);
1333
1334 /* deallocating memory while holding memory entry... */
1335 get_ledger_info(&footprint_before, &pagetable_before);
1336 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1337 T_QUIET;
1338 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1339 kr, mach_error_string(kr));
1340 /* ... should not change footprint */
1341 get_ledger_info(&footprint_after, &pagetable_after);
1342 footprint_expected = footprint_before;
1343 footprint_expected += (pagetable_after - pagetable_before);
1344 T_LOG("deallocating owned memory while holding memory entry "
1345 "does not change phys_footprint");
1346 T_EXPECT_EQ(footprint_after, footprint_expected,
1347 "deallocated %lld dirty bytes: "
1348 "footprint %lld -> %lld expected %lld delta %lld",
1349 dirty_size, footprint_before, footprint_after,
1350 footprint_expected, footprint_after - footprint_expected);
1351
1352 /* releasing the memory entry... */
1353 kr = mach_port_deallocate(mach_task_self(), me_port);
1354 T_QUIET;
1355 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1356 kr, mach_error_string(kr));
1357 /* ... should not change footprint */
1358 get_ledger_info(&footprint_after, &pagetable_after);
1359 footprint_expected = footprint_before;
1360 footprint_expected += (pagetable_after - pagetable_before);
1361 T_LOG("releasing memory entry does not change phys_footprint");
1362 T_EXPECT_EQ(footprint_after, footprint_expected,
1363 "made volatile %lld dirty bytes: "
1364 "footprint %lld -> %lld expected %lld delta %lld",
1365 dirty_size, footprint_before, footprint_after,
1366 footprint_expected, footprint_after - footprint_expected);
1367 }
1368 #endif /* MAP_MEM_LEDGER_TAGGED */
1369
1370 /* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
1371 #include <CoreFoundation/CoreFoundation.h>
1372 #include <IOSurface/IOSurface.h>
1373 #include <IOSurface/IOSurfacePrivate.h>
1374 static size_t
bytes_per_element(uint32_t format)1375 bytes_per_element(uint32_t format)
1376 {
1377 size_t bpe = 0;
1378 switch (format) {
1379 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1380 bpe = 4;
1381 break;
1382 default:
1383 bpe = 0;
1384 break;
1385 }
1386 return bpe;
1387 }
1388 static size_t
bytes_per_pixel(uint32_t format)1389 bytes_per_pixel(uint32_t format)
1390 {
1391 size_t bpe = 0;
1392 switch (format) {
1393 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1394 bpe = 4;
1395 break;
1396 default:
1397 bpe = 0;
1398 break;
1399 }
1400 return bpe;
1401 }
1402 static inline size_t
roundSizeToMultiple(size_t size,size_t mult)1403 roundSizeToMultiple(size_t size, size_t mult)
1404 {
1405 return ((size + mult - 1) / mult) * mult;
1406 }
1407 static inline void
setIntValue(CFMutableDictionaryRef dict,const CFStringRef key,int value)1408 setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
1409 {
1410 CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
1411 CFDictionarySetValue(dict, key, number);
1412 CFRelease(number);
1413 }
1414 static inline void
setBoolValue(CFMutableDictionaryRef dict,const CFStringRef key,bool value)1415 setBoolValue(CFMutableDictionaryRef dict, const CFStringRef key, bool value)
1416 {
1417 CFDictionarySetValue(dict, key, value ? kCFBooleanTrue : kCFBooleanFalse);
1418 }
1419 typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
1420 static IOReturn
SurfaceApplyPlaneBlock(IOSurfaceRef surface,SurfacePlaneBlock block)1421 SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
1422 {
1423 if (surface == nil || block == nil) {
1424 return kIOReturnBadArgument;
1425 }
1426
1427 IOReturn result = kIOReturnSuccess;
1428 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1429
1430 if (planeCount == 0) {
1431 result = IOSurfaceLock(surface, 0, NULL);
1432 if (result != kIOReturnSuccess) {
1433 return result;
1434 }
1435
1436 void* base = IOSurfaceGetBaseAddress(surface);
1437 size_t rb = IOSurfaceGetBytesPerRow(surface);
1438 size_t w = IOSurfaceGetWidth(surface);
1439 size_t h = IOSurfaceGetHeight(surface);
1440
1441 if (base && rb && w && h) {
1442 block(base, 0, w, h, rb);
1443 }
1444
1445 IOSurfaceUnlock(surface, 0, NULL);
1446 } else if (planeCount == 2) {
1447 for (size_t i = 0; i < planeCount; i++) {
1448 result = IOSurfaceLock(surface, 0, NULL);
1449 if (result != kIOReturnSuccess) {
1450 return result;
1451 }
1452
1453 void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
1454 size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
1455 size_t w = IOSurfaceGetWidthOfPlane(surface, i);
1456 size_t h = IOSurfaceGetHeightOfPlane(surface, i);
1457
1458 if (base && rb && w && h) {
1459 block(base, i, w, h, rb);
1460 }
1461
1462 IOSurfaceUnlock(surface, 0, NULL);
1463 }
1464 }
1465 return result;
1466 }
1467 static void
ClearSurface(IOSurfaceRef surface)1468 ClearSurface(IOSurfaceRef surface)
1469 {
1470 const int zero = 0;
1471 (void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
1472 {
1473 if (i == 0) {
1474 memset(p, zero, rb * h);
1475 } else {
1476 memset(p, 128, rb * h);
1477 }
1478 });
1479 }
1480 static size_t
SurfaceGetMemorySize(IOSurfaceRef surface)1481 SurfaceGetMemorySize(IOSurfaceRef surface)
1482 {
1483 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1484
1485 if (planeCount == 0) {
1486 size_t rb = IOSurfaceGetBytesPerRow(surface);
1487 size_t h = IOSurfaceGetHeight(surface);
1488 return rb * h;
1489 } else if (planeCount == 2) {
1490 size_t rb0 = IOSurfaceGetBytesPerRowOfPlane(surface, 0);
1491 size_t h0 = IOSurfaceGetHeightOfPlane(surface, 0);
1492 size_t rb1 = IOSurfaceGetBytesPerRowOfPlane(surface, 1);
1493 size_t h1 = IOSurfaceGetHeightOfPlane(surface, 1);
1494 return rb0 * h0 + rb1 * h1;
1495 }
1496 return 0;
1497 }
1498 static IOSurfaceRef
CreateSurface(uint32_t pixelsWide,uint32_t pixelsHigh,uint32_t rowBytesAlignment,uint32_t fmt,bool purgeable,bool clear)1499 CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
1500 {
1501 IOSurfaceRef surface = nil;
1502
1503 if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
1504 return nil;
1505 }
1506
1507 size_t bpp = bytes_per_pixel(fmt);
1508 size_t bpe = bytes_per_element(fmt);
1509 if (bpp == 0 || bpe == 0) {
1510 return nil;
1511 }
1512
1513 size_t rowbytes = pixelsWide * bpp;
1514 if (rowBytesAlignment == 0) {
1515 rowBytesAlignment = 16;
1516 }
1517 rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);
1518
1519 CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1520 setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
1521 setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
1522 setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
1523 setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
1524 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1525 setBoolValue(props, kIOSurfaceNonPurgeable, !purgeable);
1526 #else /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1527 (void)purgeable;
1528 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1529 {
1530 if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
1531 setIntValue(props, kIOSurfaceElementWidth, 2);
1532 setIntValue(props, kIOSurfaceElementHeight, 1);
1533 }
1534 setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
1535 }
1536
1537 surface = IOSurfaceCreate(props);
1538
1539 if (clear) {
1540 ClearSurface(surface);
1541 }
1542
1543 CFRelease(props);
1544 return surface;
1545 }
1546 T_DECL(phys_footprint_purgeable_iokit,
1547 "phys_footprint for purgeable IOKit memory",
1548 T_META_LTEPHASE(LTE_POSTINIT))
1549 {
1550 uint64_t footprint_before, pagetable_before;
1551 uint64_t footprint_after, pagetable_after;
1552 uint64_t footprint_expected, footprint_delta_slop;
1553 int64_t footprint_delta;
1554 IOSurfaceRef surface;
1555 uint32_t old_state;
1556 uint64_t surface_size;
1557
1558 T_SETUPBEGIN;
1559 footprint_delta_slop = 8 * vm_kernel_page_size;
1560 ledger_init();
1561 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1562 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1563 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1564 CFRelease(surface);
1565 T_SETUPEND;
1566
1567 surface_size = 1024 * 1024 * 4;
1568
1569 /* create IOsurface: footprint grows */
1570 get_ledger_info(&footprint_before, &pagetable_before);
1571 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1572 get_ledger_info(&footprint_after, &pagetable_after);
1573 if (legacy_footprint) {
1574 footprint_expected = footprint_before;
1575 footprint_expected += (pagetable_after - pagetable_before);
1576 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1577 T_LOG("LEGACY FOOTPRINT: creating purgeable IOSurface: no footprint impact");
1578 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1579 "create purgeable IOSurface %lld bytes: "
1580 "footprint %lld -> %lld expected %lld delta %lld",
1581 surface_size, footprint_before, footprint_after,
1582 footprint_expected, footprint_delta);
1583 } else {
1584 footprint_expected = footprint_before + surface_size;
1585 footprint_expected += (pagetable_after - pagetable_before);
1586 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1587 T_LOG("creating purgeable IOSurface increases phys_footprint");
1588 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1589 "create purgeable IOSurface %lld bytes: "
1590 "footprint %lld -> %lld expected %lld delta %lld",
1591 surface_size, footprint_before, footprint_after,
1592 footprint_expected, footprint_delta);
1593 }
1594
1595 /* make IOSurface volatile: footprint shrinks */
1596 get_ledger_info(&footprint_before, &pagetable_before);
1597 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1598 get_ledger_info(&footprint_after, &pagetable_after);
1599 if (legacy_footprint) {
1600 footprint_expected = footprint_before;
1601 footprint_expected += (pagetable_after - pagetable_before);
1602 T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
1603 T_EXPECT_EQ(footprint_after, footprint_expected,
1604 "volatile IOSurface %lld bytes: "
1605 "footprint %lld -> %lld expected %lld delta %lld",
1606 surface_size, footprint_before, footprint_after,
1607 footprint_expected, footprint_after - footprint_expected);
1608 } else {
1609 footprint_expected = footprint_before - surface_size;
1610 footprint_expected += (pagetable_after - pagetable_before);
1611 T_LOG("making IOSurface volatile decreases phys_footprint");
1612 T_EXPECT_EQ(footprint_after, footprint_expected,
1613 "made volatile %lld bytes: "
1614 "footprint %lld -> %lld expected %lld delta %lld",
1615 surface_size, footprint_before, footprint_after,
1616 footprint_expected, footprint_after - footprint_expected);
1617 }
1618
1619 /* make IOSurface non-volatile: footprint grows */
1620 get_ledger_info(&footprint_before, &pagetable_before);
1621 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1622 get_ledger_info(&footprint_after, &pagetable_after);
1623 if (legacy_footprint) {
1624 footprint_expected = footprint_before;
1625 footprint_expected += (pagetable_after - pagetable_before);
1626 T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
1627 T_EXPECT_EQ(footprint_after, footprint_expected,
1628 "non-volatile IOSurface %lld bytes: "
1629 "footprint %lld -> %lld expected %lld delta %lld",
1630 surface_size, footprint_before, footprint_after,
1631 footprint_expected, footprint_after - footprint_expected);
1632 } else {
1633 footprint_expected = footprint_before + surface_size;
1634 footprint_expected += (pagetable_after - pagetable_before);
1635 T_LOG("making IOSurface non-volatile increases phys_footprint");
1636 T_EXPECT_EQ(footprint_after, footprint_expected,
1637 "made non-volatile %lld bytes: "
1638 "footprint %lld -> %lld expected %lld delta %lld",
1639 surface_size, footprint_before, footprint_after,
1640 footprint_expected, footprint_after - footprint_expected);
1641 }
1642
1643 /* accessing IOSurface re-mapping: no footprint impact */
1644
1645 /* deallocating IOSurface re-mapping: no footprint impact */
1646
1647 /* release IOSurface: footprint shrinks */
1648 get_ledger_info(&footprint_before, &pagetable_before);
1649 CFRelease(surface);
1650 get_ledger_info(&footprint_after, &pagetable_after);
1651 if (legacy_footprint) {
1652 footprint_expected = footprint_before;
1653 footprint_expected += (pagetable_after - pagetable_before);
1654 T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
1655 T_EXPECT_EQ(footprint_after, footprint_expected,
1656 "releasing IOSurface %lld bytes: "
1657 "footprint %lld -> %lld expected %lld delta %lld",
1658 surface_size, footprint_before, footprint_after,
1659 footprint_expected, footprint_after - footprint_expected);
1660 } else {
1661 footprint_expected = footprint_before - surface_size;
1662 footprint_expected += (pagetable_after - pagetable_before);
1663 T_LOG("releasing IOSurface decreases phys_footprint");
1664 T_EXPECT_EQ(footprint_after, footprint_expected,
1665 "released IOSurface %lld bytes: "
1666 "footprint %lld -> %lld expected %lld delta %lld",
1667 surface_size, footprint_before, footprint_after,
1668 footprint_expected, footprint_after - footprint_expected);
1669 }
1670 }
1671
1672 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1673 T_DECL(phys_footprint_nonpurgeable_iokit,
1674 "phys_footprint for non-purgeable IOKit memory",
1675 T_META_LTEPHASE(LTE_POSTINIT))
1676 {
1677 uint64_t footprint_before, pagetable_before;
1678 uint64_t footprint_after, pagetable_after;
1679 uint64_t footprint_expected, footprint_delta_slop;
1680 int64_t footprint_delta;
1681 IOSurfaceRef surface;
1682 uint64_t surface_size;
1683 void *map_base;
1684 size_t map_size;
1685 mach_vm_address_t remap_addr;
1686 kern_return_t kr;
1687 vm_prot_t cur_prot, max_prot;
1688 uint32_t old_state;
1689
1690
1691 T_SETUPBEGIN;
1692 ledger_init();
1693 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1694 CFRelease(surface);
1695 footprint_delta_slop = 8 * vm_kernel_page_size;
1696 T_SETUPEND;
1697
1698 surface_size = 1024 * 1024 * 4;
1699
1700 /* create IOsurface: footprint grows */
1701 get_ledger_info(&footprint_before, &pagetable_before);
1702 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1703 get_ledger_info(&footprint_after, &pagetable_after);
1704 footprint_expected = footprint_before + surface_size;
1705 footprint_expected += (pagetable_after - pagetable_before);
1706 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1707 T_LOG("creating non-purgeable IOSurface increases phys_footprint");
1708 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1709 "create non-purgeable IOSurface %lld bytes: "
1710 "footprint %lld -> %lld expected %lld delta %lld",
1711 surface_size, footprint_before, footprint_after,
1712 footprint_expected, footprint_delta);
1713
1714 /* make IOSurface volatile: fail and no footprint impact */
1715 get_ledger_info(&footprint_before, &pagetable_before);
1716 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1717 get_ledger_info(&footprint_after, &pagetable_after);
1718 footprint_expected = footprint_before;
1719 footprint_expected += (pagetable_after - pagetable_before);
1720 T_LOG("making non-purgeable IOSurface volatile: no footprint impact");
1721 T_EXPECT_EQ(footprint_after, footprint_expected,
1722 "made volatile %lld non-purgeable bytes: "
1723 "footprint %lld -> %lld expected %lld delta %lld",
1724 surface_size, footprint_before, footprint_after,
1725 footprint_expected, footprint_after - footprint_expected);
1726
1727 /* re-mapping IOSurface: no footprint impact */
1728 get_ledger_info(&footprint_before, &pagetable_before);
1729 map_base = IOSurfaceGetBaseAddress(surface);
1730 map_size = SurfaceGetMemorySize(surface);
1731 // T_EXPECT_EQ(map_size, surface_size, "map_size %lld surface_size %lld",
1732 // map_size, surface_size);
1733 remap_addr = 0;
1734 kr = mach_vm_remap(mach_task_self(),
1735 &remap_addr,
1736 (mach_vm_size_t)surface_size,
1737 0,
1738 VM_FLAGS_ANYWHERE,
1739 mach_task_self(),
1740 (mach_vm_address_t)map_base,
1741 FALSE, /* copy */
1742 &cur_prot,
1743 &max_prot,
1744 VM_INHERIT_DEFAULT);
1745 T_QUIET;
1746 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_remap() error 0x%x (%s)",
1747 kr, mach_error_string(kr));
1748 get_ledger_info(&footprint_after, &pagetable_after);
1749 footprint_expected = footprint_before;
1750 footprint_expected += (pagetable_after - pagetable_before);
1751 T_LOG("re-mapping IOSurface does not impact phys_footprint");
1752 T_EXPECT_EQ(footprint_after, footprint_expected,
1753 "remapping IOSurface %lld bytes: "
1754 "footprint %lld -> %lld expected %lld delta %lld",
1755 surface_size, footprint_before, footprint_after,
1756 footprint_expected, footprint_after - footprint_expected);
1757
1758 /* accessing IOSurface re-mapping: footprint grows */
1759 get_ledger_info(&footprint_before, &pagetable_before);
1760 memset((char *)(uintptr_t)remap_addr, 'p', (size_t)surface_size);
1761 get_ledger_info(&footprint_after, &pagetable_after);
1762 footprint_expected = footprint_before + surface_size;
1763 footprint_expected += (pagetable_after - pagetable_before);
1764 T_LOG("accessing re-mapped IOSurface grows phys_footprint");
1765 T_EXPECT_EQ(footprint_after, footprint_expected,
1766 "accessing remapped IOSurface %lld bytes: "
1767 "footprint %lld -> %lld expected %lld delta %lld",
1768 surface_size, footprint_before, footprint_after,
1769 footprint_expected, footprint_after - footprint_expected);
1770
1771 /* deallocating IOSurface re-mapping: footprint shrinks */
1772 get_ledger_info(&footprint_before, &pagetable_before);
1773 kr = mach_vm_deallocate(mach_task_self(),
1774 remap_addr,
1775 (mach_vm_size_t)surface_size);
1776 T_QUIET;
1777 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1778 kr, mach_error_string(kr));
1779 get_ledger_info(&footprint_after, &pagetable_after);
1780 footprint_expected = footprint_before - surface_size;
1781 footprint_expected += (pagetable_after - pagetable_before);
1782 T_LOG("deallocating re-mapping of IOSurface shrinks phys_footprint");
1783 T_EXPECT_EQ(footprint_after, footprint_expected,
1784 "deallocating remapped IOSurface %lld bytes: "
1785 "footprint %lld -> %lld expected %lld delta %lld",
1786 surface_size, footprint_before, footprint_after,
1787 footprint_expected, footprint_after - footprint_expected);
1788
1789 /* release IOSurface: footprint shrinks */
1790 get_ledger_info(&footprint_before, &pagetable_before);
1791 CFRelease(surface);
1792 get_ledger_info(&footprint_after, &pagetable_after);
1793 footprint_expected = footprint_before - surface_size;
1794 footprint_expected += (pagetable_after - pagetable_before);
1795 T_LOG("releasing IOSurface decreases phys_footprint");
1796 T_EXPECT_EQ(footprint_after, footprint_expected,
1797 "released IOSurface %lld bytes: "
1798 "footprint %lld -> %lld expected %lld delta %lld",
1799 surface_size, footprint_before, footprint_after,
1800 footprint_expected, footprint_after - footprint_expected);
1801 }
1802 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1803
1804 #if 0
1805 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
1806 T_DECL(legacy_footprint_entitled_set_and_query,
1807 "legacy footprint entitled is sticky with memlimit set properties and can be queried",
1808 T_META_BOOTARGS_SET("legacy_footprint_entitlement_mode=3"))
1809 {
1810 int ret;
1811 int32_t max_task_pmem = 0, legacy_footprint_bonus_mb = 0;
1812 size_t size_max_task_pmem = sizeof(max_task_pmem);
1813 size_t size_legacy_footprint_bonus_mb = sizeof(legacy_footprint_bonus_mb);
1814 memorystatus_memlimit_properties2_t mmprops;
1815
1816 /* Get the default limit */
1817 ret = sysctlbyname("kern.max_task_pmem", &max_task_pmem, &size_max_task_pmem, NULL, 0);
1818 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "call sysctlbyname to get max task physical memory.");
1819
1820 ret = sysctlbyname("kern.legacy_footprint_bonus_mb", &legacy_footprint_bonus_mb,
1821 &size_legacy_footprint_bonus_mb, NULL, 0);
1822 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "call sysctlbyname to get the bonus value.");
1823
1824 mmprops.v1.memlimit_active = -1;
1825 mmprops.v1.memlimit_inactive = -1;
1826 ret = memorystatus_control(MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES, getpid(), 0, &mmprops.v1, sizeof(mmprops.v1));
1827 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1828
1829 /* Check our memlimt */
1830 ret = memorystatus_control(MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES, getpid(), 0, &mmprops, sizeof(mmprops));
1831 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1832
1833 T_QUIET; T_ASSERT_EQ(mmprops.v1.memlimit_active, max_task_pmem + legacy_footprint_bonus_mb, "active limit");
1834 T_QUIET; T_ASSERT_EQ(mmprops.v1.memlimit_inactive, max_task_pmem + legacy_footprint_bonus_mb, "inactive limit");
1835
1836 /* Verify MEMORYSTATUS_CMD_CONVERT_MEMLIMIT_MB */
1837 ret = memorystatus_control(MEMORYSTATUS_CMD_CONVERT_MEMLIMIT_MB, getpid(), (uint32_t) -1, NULL, 0);
1838 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "memorystatus_control");
1839 T_QUIET; T_ASSERT_EQ(ret, max_task_pmem + legacy_footprint_bonus_mb, "got bonus adjustment");
1840 }
1841 #endif /* LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__) */
1842 #endif
1843