1 #include <darwintest.h>
2 #include <darwintest_utils.h>
3
4 #include <mach/mach_error.h>
5 #include <mach/mach_init.h>
6 #include <mach/memory_entry.h>
7 #include <mach/mach_port.h>
8 #include <mach/mach_vm.h>
9 #include <mach/task.h>
10 #include <mach/task_info.h>
11 #include <mach/vm_map.h>
12
13 #include <sys/mman.h>
14 #include <sys/types.h>
15 #include <sys/sysctl.h>
16
17 #include <TargetConditionals.h>
18
19 #include <Kernel/kern/ledger.h>
20 extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);
21
22 boolean_t legacy_footprint;
23
24 #if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
25 #define TEST_VM_NAMESPACE "xnu.vm_legacy"
26 #else /* ENTITLED && __arm64__ */
27 #define TEST_VM_NAMESPACE "xnu.vm"
28 #endif /* ENTITLED && __arm64__ */
29
30 T_GLOBAL_META(
31 T_META_NAMESPACE(TEST_VM_NAMESPACE),
32 T_META_RADAR_COMPONENT_NAME("xnu"),
33 T_META_RADAR_COMPONENT_VERSION("VM"),
34 T_META_RUN_CONCURRENTLY(true));
35
36 #define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */
37
38 static int64_t ledger_count = -1;
39 static int footprint_index = -1;
40 static int pagetable_index = -1;
41 static struct ledger_entry_info *lei = NULL;
42
43 static void
ledger_init(void)44 ledger_init(void)
45 {
46 static int ledger_inited = 0;
47 struct ledger_info li;
48 struct ledger_template_info *templateInfo;
49 int64_t templateCnt;
50 int i;
51 int legacy_footprint_entitlement_mode;
52 size_t oldlen;
53
54 if (ledger_inited) {
55 return;
56 }
57 ledger_inited = 1;
58
59 T_SETUPBEGIN;
60
61 legacy_footprint = FALSE;
62 #if LEGACY_FOOTPRINT_ENTITLED
63 int ret;
64
65 T_QUIET;
66 T_WITH_ERRNO;
67 oldlen = sizeof(legacy_footprint_entitlement_mode);
68 ret = sysctlbyname("kern.legacy_footprint_entitlement_mode",
69 &legacy_footprint_entitlement_mode,
70 &oldlen,
71 NULL,
72 0);
73 if (ret == 0 && legacy_footprint_entitlement_mode == 2) {
74 legacy_footprint = TRUE;
75 }
76 #endif /* LEGACY_FOOTPRINT_ENTITLED */
77
78 T_QUIET;
79 T_WITH_ERRNO;
80 T_ASSERT_EQ(ledger(LEDGER_INFO,
81 (caddr_t)(uintptr_t)getpid(),
82 (caddr_t)&li,
83 NULL),
84 0,
85 "ledger(LEDGER_INFO)");
86
87 templateCnt = li.li_entries;
88 templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
89 T_QUIET;
90 T_WITH_ERRNO;
91 T_ASSERT_NE(templateInfo, NULL, "malloc()");
92
93 ledger_count = li.li_entries;
94 footprint_index = -1;
95 pagetable_index = -1;
96 T_QUIET;
97 T_WITH_ERRNO;
98 T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
99 (caddr_t)templateInfo,
100 (caddr_t)&templateCnt,
101 NULL),
102 0,
103 "ledger(LEDGER_TEMPLATE_INFO)");
104 for (i = 0; i < templateCnt; i++) {
105 if (!strncmp(templateInfo[i].lti_name,
106 "phys_footprint",
107 strlen("phys_footprint"))) {
108 footprint_index = i;
109 } else if (!strncmp(templateInfo[i].lti_name,
110 "page_table",
111 strlen("page_table"))) {
112 pagetable_index = i;
113 }
114 }
115 free(templateInfo);
116
117 lei = (struct ledger_entry_info *)
118 malloc((size_t)ledger_count * sizeof(*lei));
119 T_QUIET;
120 T_WITH_ERRNO;
121 T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");
122
123 T_QUIET;
124 T_ASSERT_NE(footprint_index, -1, "no footprint_index");
125 T_QUIET;
126 T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");
127
128 T_SETUPEND;
129 }
130
131 static void
get_ledger_info(uint64_t * phys_footprint,uint64_t * page_table)132 get_ledger_info(
133 uint64_t *phys_footprint,
134 uint64_t *page_table)
135 {
136 int64_t count;
137
138 count = ledger_count;
139 T_QUIET;
140 T_WITH_ERRNO;
141 T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
142 (caddr_t)(uintptr_t)getpid(),
143 (caddr_t)lei,
144 (caddr_t)&count),
145 0,
146 "ledger(LEDGER_ENTRY_INFO)");
147 T_QUIET;
148 T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
149 T_QUIET;
150 T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
151 if (phys_footprint) {
152 *phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
153 }
154 if (page_table) {
155 *page_table = (uint64_t)(lei[pagetable_index].lei_balance);
156 }
157 }
158
159 static mach_vm_address_t
pre_warm(mach_vm_size_t vm_size)160 pre_warm(
161 mach_vm_size_t vm_size)
162 {
163 kern_return_t kr;
164 mach_vm_address_t vm_addr;
165 unsigned char BigBufOnStack[100 * 1024];
166 uint64_t footprint, page_table;
167
168 /* make sure ledgers are ready to be queried */
169 ledger_init();
170
171 T_SETUPBEGIN;
172
173 /*
174 * Touch a few pages ahead on the stack, to make
175 * sure we don't see a footprint increase due to
176 * an extra stack page later.
177 */
178 memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
179 T_QUIET;
180 T_EXPECT_EQ(BigBufOnStack[0], 0xb,
181 "BigBufOnStack[0] == 0x%x",
182 BigBufOnStack[0]);
183 T_QUIET;
184 T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
185 "BigBufOnStack[%lu] == 0x%x",
186 sizeof(BigBufOnStack),
187 BigBufOnStack[sizeof(BigBufOnStack) - 1]);
188
189 /*
190 * Pre-allocate, touch and then release the same amount
191 * of memory we'll be allocating later during the test,
192 * to account for any memory overhead (page tables, global
193 * variables, ...).
194 */
195 vm_addr = 0;
196 kr = mach_vm_allocate(mach_task_self(),
197 &vm_addr,
198 vm_size,
199 VM_FLAGS_ANYWHERE);
200 T_QUIET;
201 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
202 vm_size, kr, mach_error_string(kr));
203 memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
204 kr = mach_vm_deallocate(mach_task_self(),
205 vm_addr,
206 vm_size);
207 T_QUIET;
208 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
209 kr, mach_error_string(kr));
210
211 /*
212 * Exercise the ledger code to make sure it's ready to run
213 * without any extra memory overhead later.
214 */
215 get_ledger_info(&footprint, &page_table);
216
217 T_SETUPEND;
218
219 /*
220 * Return the start of the virtual range we pre-warmed, so that the
221 * test can check that it's using the same range.
222 */
223 return vm_addr;
224 }
225
226 T_DECL(phys_footprint_anonymous,
227 "phys_footprint for anonymous memory",
228 T_META_LTEPHASE(LTE_POSTINIT))
229 {
230 uint64_t footprint_before, pagetable_before;
231 uint64_t footprint_after, pagetable_after;
232 uint64_t footprint_expected;
233 kern_return_t kr;
234 mach_vm_address_t pre_vm_addr, vm_addr;
235 mach_vm_size_t vm_size, dirty_size;
236
237 /* pre-warm to account for page table expansion */
238 pre_vm_addr = pre_warm(MEM_SIZE);
239
240 /* allocating virtual memory... */
241 get_ledger_info(&footprint_before, &pagetable_before);
242 vm_addr = 0;
243 vm_size = MEM_SIZE;
244 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
245 VM_FLAGS_ANYWHERE);
246 T_QUIET;
247 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
248 kr, mach_error_string(kr));
249 T_QUIET;
250 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
251 /* ... should not change footprint */
252 get_ledger_info(&footprint_after, &pagetable_after);
253 footprint_expected = footprint_before;
254 footprint_expected += (pagetable_after - pagetable_before);
255 T_LOG("virtual allocation does not change phys_footprint");
256 T_EXPECT_EQ(footprint_after, footprint_expected,
257 "virtual allocation of %lld bytes: "
258 "footprint %lld -> %lld expected %lld delta %lld",
259 vm_size, footprint_before, footprint_after,
260 footprint_expected, footprint_after - footprint_expected);
261
262 /* touching memory... */
263 get_ledger_info(&footprint_before, &pagetable_before);
264 dirty_size = vm_size / 2;
265 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
266 /* ... should increase footprint */
267 get_ledger_info(&footprint_after, &pagetable_after);
268 footprint_expected = footprint_before + dirty_size;
269 footprint_expected += (pagetable_after - pagetable_before);
270 T_LOG("modifying anonymous memory increases phys_footprint");
271 T_EXPECT_EQ(footprint_after, footprint_expected,
272 "touched %lld bytes: "
273 "footprint %lld -> %lld expected %lld delta %lld",
274 dirty_size, footprint_before, footprint_after,
275 footprint_expected, footprint_after - footprint_expected);
276
277 /* deallocating memory... */
278 get_ledger_info(&footprint_before, &pagetable_before);
279 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
280 T_QUIET;
281 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
282 kr, mach_error_string(kr));
283 /* ... should decrease footprint */
284 get_ledger_info(&footprint_after, &pagetable_after);
285 footprint_expected = footprint_before - dirty_size;
286 footprint_expected += (pagetable_after - pagetable_before);
287 T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
288 T_EXPECT_EQ(footprint_after, footprint_expected,
289 "deallocated %lld dirty bytes: "
290 "footprint %lld -> %lld expected %lld delta %lld",
291 dirty_size, footprint_before, footprint_after,
292 footprint_expected, footprint_after - footprint_expected);
293 }
294
295 #define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
296 #define TEMP_FILE_SIZE (1 * 1024 * 1024)
297
298 T_DECL(phys_footprint_file,
299 "phys_footprint for mapped file",
300 T_META_LTEPHASE(LTE_POSTINIT))
301 {
302 uint64_t footprint_before, pagetable_before;
303 uint64_t footprint_after, pagetable_after;
304 uint64_t footprint_expected;
305 mach_vm_address_t pre_vm_addr;
306 int fd;
307 char *map_addr;
308 size_t map_size, dirty_size;
309 ssize_t nbytes;
310 char tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
311 char *buf;
312 size_t buf_size;
313
314 T_SETUPBEGIN;
315 buf_size = TEMP_FILE_SIZE;
316 T_QUIET;
317 T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
318 "allocate %zu-byte buffer", buf_size);
319 memset(buf, 'f', buf_size);
320 T_WITH_ERRNO;
321 T_QUIET;
322 T_ASSERT_NOTNULL(mktemp(tmp_file_name),
323 "create temporary file name");
324 T_WITH_ERRNO;
325 T_QUIET;
326 T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
327 0,
328 "create temp file");
329 T_WITH_ERRNO;
330 T_QUIET;
331 T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
332 (ssize_t)buf_size,
333 "write %zu bytes", buf_size);
334 free(buf);
335 T_SETUPEND;
336
337 /* pre-warm to account for page table expansion */
338 pre_vm_addr = pre_warm(TEMP_FILE_SIZE);
339
340 /* mapping a file does not impact footprint... */
341 get_ledger_info(&footprint_before, &pagetable_before);
342 map_size = TEMP_FILE_SIZE;
343 T_WITH_ERRNO;
344 T_QUIET;
345 T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
346 PROT_READ | PROT_WRITE,
347 MAP_FILE | MAP_SHARED, fd, 0),
348 "mmap()");
349 T_QUIET;
350 T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
351 "pre-warm mishap");
352 /* ... should not change footprint */
353 get_ledger_info(&footprint_after, &pagetable_after);
354 footprint_expected = footprint_before;
355 footprint_expected += (pagetable_after - pagetable_before);
356 T_LOG("mapping file does not change phys_footprint");
357 T_EXPECT_EQ(footprint_after, footprint_expected,
358 "mapping file with %zu bytes: "
359 "footprint %lld -> %lld expected %lld delta %lld",
360 map_size, footprint_before, footprint_after,
361 footprint_expected, footprint_after - footprint_expected);
362
363 /* touching file-backed memory... */
364 get_ledger_info(&footprint_before, &pagetable_before);
365 dirty_size = map_size / 2;
366 memset(map_addr, 'F', dirty_size);
367 /* ... should not impact footprint */
368 get_ledger_info(&footprint_after, &pagetable_after);
369 footprint_expected = footprint_before;
370 footprint_expected += (pagetable_after - pagetable_before);
371 T_LOG("modifying file-backed memory does not impact phys_footprint");
372 T_EXPECT_EQ(footprint_after, footprint_expected,
373 "touched %zu bytes: "
374 "footprint %lld -> %lld expected %lld delta %lld",
375 dirty_size, footprint_before, footprint_after,
376 footprint_expected, footprint_after - footprint_expected);
377
378 /* deallocating file-backed memory... */
379 get_ledger_info(&footprint_before, &pagetable_before);
380 T_WITH_ERRNO;
381 T_QUIET;
382 T_ASSERT_EQ(munmap(map_addr, map_size),
383 0,
384 "unmap file");
385 /* ... should not impact footprint */
386 get_ledger_info(&footprint_after, &pagetable_after);
387 footprint_expected = footprint_before;
388 footprint_expected += (pagetable_after - pagetable_before);
389 T_LOG("unmapping file-backed memory does not impact phys_footprint");
390 T_EXPECT_EQ(footprint_after, footprint_expected,
391 "unmapped %zu dirty bytes: "
392 "footprint %lld -> %lld expected %lld delta %lld",
393 dirty_size, footprint_before, footprint_after,
394 footprint_expected, footprint_after - footprint_expected);
395 }
396
397 T_DECL(phys_footprint_purgeable,
398 "phys_footprint for purgeable memory",
399 T_META_LTEPHASE(LTE_POSTINIT))
400 {
401 uint64_t footprint_before, pagetable_before;
402 uint64_t footprint_after, pagetable_after;
403 uint64_t footprint_expected;
404 kern_return_t kr;
405 mach_vm_address_t pre_vm_addr, vm_addr;
406 mach_vm_size_t vm_size, dirty_size;
407 int state;
408
409 /* pre-warm to account for page table expansion */
410 pre_vm_addr = pre_warm(MEM_SIZE);
411
412 /* allocating purgeable virtual memory... */
413 get_ledger_info(&footprint_before, &pagetable_before);
414 vm_addr = 0;
415 vm_size = MEM_SIZE;
416 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
417 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
418 T_QUIET;
419 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
420 kr, mach_error_string(kr));
421 T_QUIET;
422 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
423 /* ... should not change footprint */
424 get_ledger_info(&footprint_after, &pagetable_after);
425 footprint_expected = footprint_before;
426 footprint_expected += (pagetable_after - pagetable_before);
427 T_LOG("purgeable virtual allocation does not change phys_footprint");
428 T_EXPECT_EQ(footprint_after, footprint_expected,
429 "purgeable virtual allocation of %lld bytes: "
430 "footprint %lld -> %lld expected %lld delta %lld",
431 vm_size, footprint_before, footprint_after,
432 footprint_expected, footprint_after - footprint_expected);
433
434 /* touching memory... */
435 get_ledger_info(&footprint_before, &pagetable_before);
436 dirty_size = vm_size / 2;
437 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
438 /* ... should increase footprint */
439 get_ledger_info(&footprint_after, &pagetable_after);
440 footprint_expected = footprint_before + dirty_size;
441 footprint_expected += (pagetable_after - pagetable_before);
442 T_LOG("modifying anonymous memory increases phys_footprint");
443 T_EXPECT_EQ(footprint_after, footprint_expected,
444 "touched %lld bytes: "
445 "footprint %lld -> %lld expected %lld delta %lld",
446 dirty_size, footprint_before, footprint_after,
447 footprint_expected, footprint_after - footprint_expected);
448
449 /* making it volatile... */
450 get_ledger_info(&footprint_before, &pagetable_before);
451 state = VM_PURGABLE_VOLATILE;
452 T_QUIET;
453 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
454 vm_addr,
455 VM_PURGABLE_SET_STATE,
456 &state),
457 KERN_SUCCESS,
458 "vm_purgable_control(VOLATILE)");
459 T_QUIET;
460 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
461 "memory was non-volatile");
462 /* ... should decrease footprint */
463 get_ledger_info(&footprint_after, &pagetable_after);
464 footprint_expected = footprint_before - dirty_size;
465 footprint_expected += (pagetable_after - pagetable_before);
466 T_LOG("making volatile decreases phys_footprint");
467 T_EXPECT_EQ(footprint_after, footprint_expected,
468 "made volatile %lld dirty bytes: "
469 "footprint %lld -> %lld expected %lld delta %lld",
470 dirty_size, footprint_before, footprint_after,
471 footprint_expected, footprint_after - footprint_expected);
472
473 /* making it non-volatile... */
474 get_ledger_info(&footprint_before, &pagetable_before);
475 state = VM_PURGABLE_NONVOLATILE;
476 T_QUIET;
477 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
478 vm_addr,
479 VM_PURGABLE_SET_STATE,
480 &state),
481 KERN_SUCCESS,
482 "vm_purgable_control(NONVOLATILE)");
483 T_QUIET;
484 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
485 "memory was volatile");
486 /* ... should increase footprint */
487 get_ledger_info(&footprint_after, &pagetable_after);
488 footprint_expected = footprint_before + dirty_size;
489 footprint_expected += (pagetable_after - pagetable_before);
490 T_LOG("making non-volatile increases phys_footprint");
491 T_EXPECT_EQ(footprint_after, footprint_expected,
492 "made non-volatile %lld dirty bytes: "
493 "footprint %lld -> %lld expected %lld delta %lld",
494 dirty_size, footprint_before, footprint_after,
495 footprint_expected, footprint_after - footprint_expected);
496
497 /* deallocating memory... */
498 get_ledger_info(&footprint_before, &pagetable_before);
499 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
500 T_QUIET;
501 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
502 kr, mach_error_string(kr));
503 /* ... should decrease footprint */
504 get_ledger_info(&footprint_after, &pagetable_after);
505 footprint_expected = footprint_before - dirty_size;
506 footprint_expected += (pagetable_after - pagetable_before);
507 T_LOG("deallocating memory decreases phys_footprint");
508 T_EXPECT_EQ(footprint_after, footprint_expected,
509 "deallocated %lld dirty bytes: "
510 "footprint %lld -> %lld expected %lld delta %lld",
511 dirty_size, footprint_before, footprint_after,
512 footprint_expected, footprint_after - footprint_expected);
513 }
514
515 T_DECL(phys_footprint_purgeable_ownership,
516 "phys_footprint for owned purgeable memory",
517 T_META_LTEPHASE(LTE_POSTINIT))
518 {
519 uint64_t footprint_before, pagetable_before;
520 uint64_t footprint_after, pagetable_after;
521 uint64_t footprint_expected;
522 kern_return_t kr;
523 mach_vm_address_t pre_vm_addr, vm_addr;
524 mach_vm_size_t vm_size, dirty_size, me_size;
525 int state;
526 mach_port_t me_port;
527
528 /* pre-warm to account for page table expansion */
529 pre_vm_addr = pre_warm(MEM_SIZE);
530
531 /* allocating purgeable virtual memory... */
532 get_ledger_info(&footprint_before, &pagetable_before);
533 vm_addr = 0;
534 vm_size = MEM_SIZE;
535 kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
536 VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
537 T_QUIET;
538 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
539 kr, mach_error_string(kr));
540 T_QUIET;
541 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
542 /* ... should not change footprint */
543 get_ledger_info(&footprint_after, &pagetable_after);
544 footprint_expected = footprint_before;
545 footprint_expected += (pagetable_after - pagetable_before);
546 T_LOG("purgeable virtual allocation does not change phys_footprint");
547 T_EXPECT_EQ(footprint_after, footprint_expected,
548 "purgeable virtual allocation of %lld bytes: "
549 "footprint %lld -> %lld expected %lld delta %lld",
550 vm_size, footprint_before, footprint_after,
551 footprint_expected, footprint_after - footprint_expected);
552
553 /* touching memory... */
554 get_ledger_info(&footprint_before, &pagetable_before);
555 dirty_size = vm_size / 2;
556 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
557 /* ... should increase footprint */
558 get_ledger_info(&footprint_after, &pagetable_after);
559 footprint_expected = footprint_before + dirty_size;
560 footprint_expected += (pagetable_after - pagetable_before);
561 T_LOG("modifying anonymous memory increases phys_footprint");
562 T_EXPECT_EQ(footprint_after, footprint_expected,
563 "touched %lld bytes: "
564 "footprint %lld -> %lld expected %lld delta %lld",
565 dirty_size, footprint_before, footprint_after,
566 footprint_expected, footprint_after - footprint_expected);
567
568 /* making it volatile... */
569 get_ledger_info(&footprint_before, &pagetable_before);
570 state = VM_PURGABLE_VOLATILE;
571 T_QUIET;
572 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
573 vm_addr,
574 VM_PURGABLE_SET_STATE,
575 &state),
576 KERN_SUCCESS,
577 "vm_purgable_control(VOLATILE)");
578 T_QUIET;
579 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
580 "memory was non-volatile");
581 /* ... should decrease footprint */
582 get_ledger_info(&footprint_after, &pagetable_after);
583 footprint_expected = footprint_before - dirty_size;
584 footprint_expected += (pagetable_after - pagetable_before);
585 T_LOG("making volatile decreases phys_footprint");
586 T_EXPECT_EQ(footprint_after, footprint_expected,
587 "made volatile %lld dirty bytes: "
588 "footprint %lld -> %lld expected %lld delta %lld",
589 dirty_size, footprint_before, footprint_after,
590 footprint_expected, footprint_after - footprint_expected);
591
592 /* making it non-volatile... */
593 get_ledger_info(&footprint_before, &pagetable_before);
594 state = VM_PURGABLE_NONVOLATILE;
595 T_QUIET;
596 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
597 vm_addr,
598 VM_PURGABLE_SET_STATE,
599 &state),
600 KERN_SUCCESS,
601 "vm_purgable_control(NONVOLATILE)");
602 T_QUIET;
603 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
604 "memory was volatile");
605 /* ... should increase footprint */
606 get_ledger_info(&footprint_after, &pagetable_after);
607 footprint_expected = footprint_before + dirty_size;
608 footprint_expected += (pagetable_after - pagetable_before);
609 T_LOG("making non-volatile increases phys_footprint");
610 T_EXPECT_EQ(footprint_after, footprint_expected,
611 "made non-volatile %lld dirty bytes: "
612 "footprint %lld -> %lld expected %lld delta %lld",
613 dirty_size, footprint_before, footprint_after,
614 footprint_expected, footprint_after - footprint_expected);
615
616 /* making a memory entry... */
617 get_ledger_info(&footprint_before, &pagetable_before);
618 me_size = vm_size;
619 me_port = MACH_PORT_NULL;
620 kr = mach_make_memory_entry_64(mach_task_self(),
621 &me_size,
622 vm_addr,
623 VM_PROT_READ | VM_PROT_WRITE,
624 &me_port,
625 MACH_PORT_NULL);
626 T_QUIET;
627 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
628 kr, mach_error_string(kr));
629 T_QUIET;
630 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
631 /* ... should not change footprint */
632 get_ledger_info(&footprint_after, &pagetable_after);
633 footprint_expected = footprint_before;
634 footprint_expected += (pagetable_after - pagetable_before);
635 T_LOG("making a memory entry does not change phys_footprint");
636 T_EXPECT_EQ(footprint_after, footprint_expected,
637 "making a memory entry of %lld bytes: "
638 "footprint %lld -> %lld expected %lld delta %lld",
639 vm_size, footprint_before, footprint_after,
640 footprint_expected, footprint_after - footprint_expected);
641
642 /* deallocating memory while holding memory entry... */
643 get_ledger_info(&footprint_before, &pagetable_before);
644 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
645 T_QUIET;
646 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
647 kr, mach_error_string(kr));
648 /* ... should not change footprint */
649 get_ledger_info(&footprint_after, &pagetable_after);
650 footprint_expected = footprint_before;
651 footprint_expected += (pagetable_after - pagetable_before);
652 T_LOG("deallocating owned memory while holding memory entry "
653 "does not change phys_footprint");
654 T_EXPECT_EQ(footprint_after, footprint_expected,
655 "deallocated %lld dirty bytes: "
656 "footprint %lld -> %lld expected %lld delta %lld",
657 dirty_size, footprint_before, footprint_after,
658 footprint_expected, footprint_after - footprint_expected);
659
660 /* releasing the memory entry... */
661 kr = mach_port_deallocate(mach_task_self(), me_port);
662 T_QUIET;
663 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
664 kr, mach_error_string(kr));
665 /* ... should decrease footprint */
666 get_ledger_info(&footprint_after, &pagetable_after);
667 footprint_expected = footprint_before - dirty_size;
668 footprint_expected += (pagetable_after - pagetable_before);
669 T_LOG("releasing memory entry decreases phys_footprint");
670 T_EXPECT_EQ(footprint_after, footprint_expected,
671 "made volatile %lld dirty bytes: "
672 "footprint %lld -> %lld expected %lld delta %lld",
673 dirty_size, footprint_before, footprint_after,
674 footprint_expected, footprint_after - footprint_expected);
675 }
676
677 #ifdef MAP_MEM_LEDGER_TAGGED
678 T_DECL(phys_footprint_ledger_purgeable_owned,
679 "phys_footprint for ledger-tagged purgeable memory ownership",
680 T_META_LTEPHASE(LTE_POSTINIT))
681 {
682 uint64_t footprint_before, pagetable_before;
683 uint64_t footprint_after, pagetable_after;
684 uint64_t footprint_expected;
685 kern_return_t kr;
686 mach_vm_address_t pre_vm_addr, vm_addr;
687 mach_vm_size_t vm_size, dirty_size, me_size;
688 int state;
689 mach_port_t me_port;
690
691 /* pre-warm to account for page table expansion */
692 pre_vm_addr = pre_warm(MEM_SIZE);
693
694 /* making a memory entry... */
695 get_ledger_info(&footprint_before, &pagetable_before);
696 vm_size = MEM_SIZE;
697 me_size = vm_size;
698 me_port = MACH_PORT_NULL;
699 kr = mach_make_memory_entry_64(mach_task_self(),
700 &me_size,
701 0,
702 (MAP_MEM_NAMED_CREATE |
703 MAP_MEM_LEDGER_TAGGED |
704 MAP_MEM_PURGABLE |
705 VM_PROT_READ | VM_PROT_WRITE),
706 &me_port,
707 MACH_PORT_NULL);
708 T_QUIET;
709 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
710 kr, mach_error_string(kr));
711 T_QUIET;
712 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
713 /* ... should not change footprint */
714 get_ledger_info(&footprint_after, &pagetable_after);
715 footprint_expected = footprint_before;
716 footprint_expected += (pagetable_after - pagetable_before);
717 T_LOG("making a memory entry does not change phys_footprint");
718 T_EXPECT_EQ(footprint_after, footprint_expected,
719 "making a memory entry of %lld bytes: "
720 "footprint %lld -> %lld expected %lld delta %lld",
721 vm_size, footprint_before, footprint_after,
722 footprint_expected, footprint_after - footprint_expected);
723
724 /* mapping ledger-tagged virtual memory... */
725 get_ledger_info(&footprint_before, &pagetable_before);
726 vm_addr = 0;
727 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
728 0, /* mask */
729 VM_FLAGS_ANYWHERE,
730 me_port,
731 0, /* offset */
732 FALSE, /* copy */
733 VM_PROT_READ | VM_PROT_WRITE,
734 VM_PROT_READ | VM_PROT_WRITE,
735 VM_INHERIT_DEFAULT);
736 T_QUIET;
737 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
738 kr, mach_error_string(kr));
739 T_QUIET;
740 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
741 /* ... should not change footprint */
742 get_ledger_info(&footprint_after, &pagetable_after);
743 footprint_expected = footprint_before;
744 footprint_expected += (pagetable_after - pagetable_before);
745 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
746 T_EXPECT_EQ(footprint_after, footprint_expected,
747 "ledger-tagged mapping of %lld bytes: "
748 "footprint %lld -> %lld expected %lld delta %lld",
749 vm_size, footprint_before, footprint_after,
750 footprint_expected, footprint_after - footprint_expected);
751
752 /* touching memory... */
753 get_ledger_info(&footprint_before, &pagetable_before);
754 dirty_size = vm_size / 2;
755 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
756 /* ... should increase footprint */
757 get_ledger_info(&footprint_after, &pagetable_after);
758 footprint_expected = footprint_before + dirty_size;
759 footprint_expected += (pagetable_after - pagetable_before);
760 T_LOG("modifying ledger-tagged memory increases phys_footprint");
761 T_EXPECT_EQ(footprint_after, footprint_expected,
762 "touched %lld bytes: "
763 "footprint %lld -> %lld expected %lld delta %lld",
764 dirty_size, footprint_before, footprint_after,
765 footprint_expected, footprint_after - footprint_expected);
766
767 /* making it volatile... */
768 get_ledger_info(&footprint_before, &pagetable_before);
769 state = VM_PURGABLE_VOLATILE;
770 T_QUIET;
771 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
772 vm_addr,
773 VM_PURGABLE_SET_STATE,
774 &state),
775 KERN_SUCCESS,
776 "vm_purgable_control(VOLATILE)");
777 T_QUIET;
778 T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
779 "memory was non-volatile");
780 /* ... should decrease footprint */
781 get_ledger_info(&footprint_after, &pagetable_after);
782 footprint_expected = footprint_before - dirty_size;
783 footprint_expected += (pagetable_after - pagetable_before);
784 T_LOG("making volatile decreases phys_footprint");
785 T_EXPECT_EQ(footprint_after, footprint_expected,
786 "made volatile %lld dirty bytes: "
787 "footprint %lld -> %lld expected %lld delta %lld",
788 dirty_size, footprint_before, footprint_after,
789 footprint_expected, footprint_after - footprint_expected);
790
791 /* making it non-volatile... */
792 get_ledger_info(&footprint_before, &pagetable_before);
793 state = VM_PURGABLE_NONVOLATILE;
794 T_QUIET;
795 T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
796 vm_addr,
797 VM_PURGABLE_SET_STATE,
798 &state),
799 KERN_SUCCESS,
800 "vm_purgable_control(NONVOLATILE)");
801 T_QUIET;
802 T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
803 "memory was volatile");
804 /* ... should increase footprint */
805 get_ledger_info(&footprint_after, &pagetable_after);
806 footprint_expected = footprint_before + dirty_size;
807 footprint_expected += (pagetable_after - pagetable_before);
808 T_LOG("making non-volatile increases phys_footprint");
809 T_EXPECT_EQ(footprint_after, footprint_expected,
810 "made non-volatile %lld dirty bytes: "
811 "footprint %lld -> %lld expected %lld delta %lld",
812 dirty_size, footprint_before, footprint_after,
813 footprint_expected, footprint_after - footprint_expected);
814
815 /* deallocating memory while holding memory entry... */
816 get_ledger_info(&footprint_before, &pagetable_before);
817 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
818 T_QUIET;
819 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
820 kr, mach_error_string(kr));
821 /* ... should not change footprint */
822 get_ledger_info(&footprint_after, &pagetable_after);
823 footprint_expected = footprint_before;
824 footprint_expected += (pagetable_after - pagetable_before);
825 T_LOG("deallocating owned memory while holding memory entry "
826 "does not change phys_footprint");
827 T_EXPECT_EQ(footprint_after, footprint_expected,
828 "deallocated %lld dirty bytes: "
829 "footprint %lld -> %lld expected %lld delta %lld",
830 dirty_size, footprint_before, footprint_after,
831 footprint_expected, footprint_after - footprint_expected);
832
833 /* releasing the memory entry... */
834 kr = mach_port_deallocate(mach_task_self(), me_port);
835 T_QUIET;
836 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
837 kr, mach_error_string(kr));
838 /* ... should decrease footprint */
839 get_ledger_info(&footprint_after, &pagetable_after);
840 footprint_expected = footprint_before - dirty_size;
841 footprint_expected += (pagetable_after - pagetable_before);
842 T_LOG("releasing memory entry decreases phys_footprint");
843 T_EXPECT_EQ(footprint_after, footprint_expected,
844 "made volatile %lld dirty bytes: "
845 "footprint %lld -> %lld expected %lld delta %lld",
846 dirty_size, footprint_before, footprint_after,
847 footprint_expected, footprint_after - footprint_expected);
848 }
849
850 T_DECL(phys_footprint_ledger_owned,
851 "phys_footprint for ledger-tagged memory ownership",
852 T_META_LTEPHASE(LTE_POSTINIT))
853 {
854 uint64_t footprint_before, pagetable_before;
855 uint64_t footprint_after, pagetable_after;
856 uint64_t footprint_expected;
857 kern_return_t kr;
858 mach_vm_address_t pre_vm_addr, vm_addr;
859 mach_vm_size_t vm_size, dirty_size, me_size;
860 mach_port_t me_port;
861
862 /* pre-warm to account for page table expansion */
863 pre_vm_addr = pre_warm(MEM_SIZE);
864
865 /* making a memory entry... */
866 get_ledger_info(&footprint_before, &pagetable_before);
867 vm_size = MEM_SIZE;
868 me_size = vm_size;
869 me_port = MACH_PORT_NULL;
870 kr = mach_make_memory_entry_64(mach_task_self(),
871 &me_size,
872 0,
873 (MAP_MEM_NAMED_CREATE |
874 MAP_MEM_LEDGER_TAGGED |
875 VM_PROT_READ | VM_PROT_WRITE),
876 &me_port,
877 MACH_PORT_NULL);
878 T_QUIET;
879 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
880 kr, mach_error_string(kr));
881 T_QUIET;
882 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
883 /* ... should not change footprint */
884 get_ledger_info(&footprint_after, &pagetable_after);
885 footprint_expected = footprint_before;
886 footprint_expected += (pagetable_after - pagetable_before);
887 T_LOG("making a memory entry does not change phys_footprint");
888 T_EXPECT_EQ(footprint_after, footprint_expected,
889 "making a memory entry of %lld bytes: "
890 "footprint %lld -> %lld expected %lld delta %lld",
891 vm_size, footprint_before, footprint_after,
892 footprint_expected, footprint_after - footprint_expected);
893
894 /* mapping ledger-tagged virtual memory... */
895 get_ledger_info(&footprint_before, &pagetable_before);
896 vm_addr = 0;
897 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
898 0, /* mask */
899 VM_FLAGS_ANYWHERE,
900 me_port,
901 0, /* offset */
902 FALSE, /* copy */
903 VM_PROT_READ | VM_PROT_WRITE,
904 VM_PROT_READ | VM_PROT_WRITE,
905 VM_INHERIT_DEFAULT);
906 T_QUIET;
907 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
908 kr, mach_error_string(kr));
909 T_QUIET;
910 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
911 /* ... should not change footprint */
912 get_ledger_info(&footprint_after, &pagetable_after);
913 footprint_expected = footprint_before;
914 footprint_expected += (pagetable_after - pagetable_before);
915 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
916 T_EXPECT_EQ(footprint_after, footprint_expected,
917 "ledger-tagged mapping of %lld bytes: "
918 "footprint %lld -> %lld expected %lld delta %lld",
919 vm_size, footprint_before, footprint_after,
920 footprint_expected, footprint_after - footprint_expected);
921
922 /* touching memory... */
923 get_ledger_info(&footprint_before, &pagetable_before);
924 dirty_size = vm_size / 2;
925 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
926 /* ... should increase footprint */
927 get_ledger_info(&footprint_after, &pagetable_after);
928 footprint_expected = footprint_before + dirty_size;
929 footprint_expected += (pagetable_after - pagetable_before);
930 T_LOG("modifying ledger-tagged memory increases phys_footprint");
931 T_EXPECT_EQ(footprint_after, footprint_expected,
932 "touched %lld bytes: "
933 "footprint %lld -> %lld expected %lld delta %lld",
934 dirty_size, footprint_before, footprint_after,
935 footprint_expected, footprint_after - footprint_expected);
936
937 /* deallocating memory while holding memory entry... */
938 get_ledger_info(&footprint_before, &pagetable_before);
939 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
940 T_QUIET;
941 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
942 kr, mach_error_string(kr));
943 /* ... should not change footprint */
944 get_ledger_info(&footprint_after, &pagetable_after);
945 footprint_expected = footprint_before;
946 footprint_expected += (pagetable_after - pagetable_before);
947 T_LOG("deallocating owned memory while holding memory entry "
948 "does not change phys_footprint");
949 T_EXPECT_EQ(footprint_after, footprint_expected,
950 "deallocated %lld dirty bytes: "
951 "footprint %lld -> %lld expected %lld delta %lld",
952 dirty_size, footprint_before, footprint_after,
953 footprint_expected, footprint_after - footprint_expected);
954
955 /* releasing the memory entry... */
956 kr = mach_port_deallocate(mach_task_self(), me_port);
957 T_QUIET;
958 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
959 kr, mach_error_string(kr));
960 /* ... should decrease footprint */
961 get_ledger_info(&footprint_after, &pagetable_after);
962 footprint_expected = footprint_before - dirty_size;
963 footprint_expected += (pagetable_after - pagetable_before);
964 T_LOG("releasing memory entry decreases phys_footprint");
965 T_EXPECT_EQ(footprint_after, footprint_expected,
966 "made volatile %lld dirty bytes: "
967 "footprint %lld -> %lld expected %lld delta %lld",
968 dirty_size, footprint_before, footprint_after,
969 footprint_expected, footprint_after - footprint_expected);
970 }
971
972 T_DECL(phys_footprint_no_footprint_for_debug,
973 "phys_footprint for no_footprint_for_debug",
974 T_META_LTEPHASE(LTE_POSTINIT))
975 {
976 uint64_t footprint_before, pagetable_before;
977 uint64_t footprint_after, pagetable_after;
978 uint64_t footprint_expected;
979 kern_return_t kr;
980 mach_vm_address_t pre_vm_addr, vm_addr;
981 mach_vm_size_t vm_size, dirty_size, me_size;
982 mach_port_t me_port;
983 int new_value, ret;
984
985 /* pre-warm to account for page table expansion */
986 pre_vm_addr = pre_warm(MEM_SIZE);
987
988 /* making a memory entry... */
989 get_ledger_info(&footprint_before, &pagetable_before);
990 vm_size = MEM_SIZE;
991 me_size = vm_size;
992 me_port = MACH_PORT_NULL;
993 kr = mach_make_memory_entry_64(mach_task_self(),
994 &me_size,
995 0,
996 (MAP_MEM_NAMED_CREATE |
997 MAP_MEM_LEDGER_TAGGED |
998 VM_PROT_READ | VM_PROT_WRITE),
999 &me_port,
1000 MACH_PORT_NULL);
1001 T_QUIET;
1002 T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
1003 kr, mach_error_string(kr));
1004 T_QUIET;
1005 T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
1006 /* ... should not change footprint */
1007 get_ledger_info(&footprint_after, &pagetable_after);
1008 footprint_expected = footprint_before;
1009 footprint_expected += (pagetable_after - pagetable_before);
1010 T_LOG("making a memory entry does not change phys_footprint");
1011 T_EXPECT_EQ(footprint_after, footprint_expected,
1012 "making a memory entry of %lld bytes: "
1013 "footprint %lld -> %lld expected %lld delta %lld",
1014 vm_size, footprint_before, footprint_after,
1015 footprint_expected, footprint_after - footprint_expected);
1016
1017 /* trying to hide debug memory from footprint while not allowed */
1018 kr = mach_memory_entry_ownership(me_port,
1019 mach_task_self(),
1020 VM_LEDGER_TAG_DEFAULT,
1021 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1022 T_QUIET;
1023 T_EXPECT_EQ(kr, KERN_NO_ACCESS,
1024 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) fails without sysctl");
1025
1026 /* let's get permission to hide debug memory */
1027 new_value = 1;
1028 ret = sysctlbyname("vm.task_no_footprint_for_debug", NULL, NULL, &new_value, sizeof(new_value));
1029 if (ret == -1 && errno == ENOENT) {
1030 T_SKIP("sysctlbyname(vm.task_no_footprint_for_debug) ENOENT");
1031 }
1032 T_QUIET;
1033 T_ASSERT_POSIX_SUCCESS(ret, "sysctlbyname(vm.task_no_footprint_for_debug)");
1034
1035 /* trying to hide debug memory from footprint while allowed */
1036 kr = mach_memory_entry_ownership(me_port,
1037 mach_task_self(),
1038 VM_LEDGER_TAG_DEFAULT,
1039 VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
1040 T_QUIET;
1041 T_EXPECT_EQ(kr, KERN_SUCCESS,
1042 "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) succeeds after sysctl");
1043
1044 /* mapping ledger-tagged virtual memory... */
1045 get_ledger_info(&footprint_before, &pagetable_before);
1046 vm_addr = 0;
1047 kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1048 0, /* mask */
1049 VM_FLAGS_ANYWHERE,
1050 me_port,
1051 0, /* offset */
1052 FALSE, /* copy */
1053 VM_PROT_READ | VM_PROT_WRITE,
1054 VM_PROT_READ | VM_PROT_WRITE,
1055 VM_INHERIT_DEFAULT);
1056 T_QUIET;
1057 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
1058 kr, mach_error_string(kr));
1059 T_QUIET;
1060 T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
1061 /* ... should not change footprint */
1062 get_ledger_info(&footprint_after, &pagetable_after);
1063 footprint_expected = footprint_before;
1064 footprint_expected += (pagetable_after - pagetable_before);
1065 T_LOG("mapping ledger-tagged memory does not change phys_footprint");
1066 T_EXPECT_EQ(footprint_after, footprint_expected,
1067 "ledger-tagged mapping of %lld bytes: "
1068 "footprint %lld -> %lld expected %lld delta %lld",
1069 vm_size, footprint_before, footprint_after,
1070 footprint_expected, footprint_after - footprint_expected);
1071
1072 /* touching memory... */
1073 get_ledger_info(&footprint_before, &pagetable_before);
1074 dirty_size = vm_size / 2;
1075 memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
1076 /* ... should not increase footprint */
1077 get_ledger_info(&footprint_after, &pagetable_after);
1078 footprint_expected = footprint_before;
1079 footprint_expected += (pagetable_after - pagetable_before);
1080 T_LOG("modifying no_footprint_for_debug memory does not increase phys_footprint");
1081 T_EXPECT_EQ(footprint_after, footprint_expected,
1082 "touched %lld bytes: "
1083 "footprint %lld -> %lld expected %lld delta %lld",
1084 dirty_size, footprint_before, footprint_after,
1085 footprint_expected, footprint_after - footprint_expected);
1086
1087 /* deallocating memory while holding memory entry... */
1088 get_ledger_info(&footprint_before, &pagetable_before);
1089 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1090 T_QUIET;
1091 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1092 kr, mach_error_string(kr));
1093 /* ... should not change footprint */
1094 get_ledger_info(&footprint_after, &pagetable_after);
1095 footprint_expected = footprint_before;
1096 footprint_expected += (pagetable_after - pagetable_before);
1097 T_LOG("deallocating owned memory while holding memory entry "
1098 "does not change phys_footprint");
1099 T_EXPECT_EQ(footprint_after, footprint_expected,
1100 "deallocated %lld dirty bytes: "
1101 "footprint %lld -> %lld expected %lld delta %lld",
1102 dirty_size, footprint_before, footprint_after,
1103 footprint_expected, footprint_after - footprint_expected);
1104
1105 /* releasing the memory entry... */
1106 kr = mach_port_deallocate(mach_task_self(), me_port);
1107 T_QUIET;
1108 T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
1109 kr, mach_error_string(kr));
1110 /* ... should not change footprint */
1111 get_ledger_info(&footprint_after, &pagetable_after);
1112 footprint_expected = footprint_before;
1113 footprint_expected += (pagetable_after - pagetable_before);
1114 T_LOG("releasing memory entry does not change phys_footprint");
1115 T_EXPECT_EQ(footprint_after, footprint_expected,
1116 "made volatile %lld dirty bytes: "
1117 "footprint %lld -> %lld expected %lld delta %lld",
1118 dirty_size, footprint_before, footprint_after,
1119 footprint_expected, footprint_after - footprint_expected);
1120 }
1121 #endif /* MAP_MEM_LEDGER_TAGGED */
1122
1123 /* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
1124 #include <CoreFoundation/CoreFoundation.h>
1125 #include <IOSurface/IOSurface.h>
1126 #include <IOSurface/IOSurfacePrivate.h>
1127 static size_t
bytes_per_element(uint32_t format)1128 bytes_per_element(uint32_t format)
1129 {
1130 size_t bpe = 0;
1131 switch (format) {
1132 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1133 bpe = 4;
1134 break;
1135 default:
1136 bpe = 0;
1137 break;
1138 }
1139 return bpe;
1140 }
1141 static size_t
bytes_per_pixel(uint32_t format)1142 bytes_per_pixel(uint32_t format)
1143 {
1144 size_t bpe = 0;
1145 switch (format) {
1146 case 32: // kCVPixelFormatType_32ARGB (ARGB8)
1147 bpe = 4;
1148 break;
1149 default:
1150 bpe = 0;
1151 break;
1152 }
1153 return bpe;
1154 }
1155 static inline size_t
roundSizeToMultiple(size_t size,size_t mult)1156 roundSizeToMultiple(size_t size, size_t mult)
1157 {
1158 return ((size + mult - 1) / mult) * mult;
1159 }
1160 static inline void
setIntValue(CFMutableDictionaryRef dict,const CFStringRef key,int value)1161 setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
1162 {
1163 CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
1164 CFDictionarySetValue(dict, key, number);
1165 CFRelease(number);
1166 }
1167 static inline void
setBoolValue(CFMutableDictionaryRef dict,const CFStringRef key,bool value)1168 setBoolValue(CFMutableDictionaryRef dict, const CFStringRef key, bool value)
1169 {
1170 CFDictionarySetValue(dict, key, value ? kCFBooleanTrue : kCFBooleanFalse);
1171 }
1172 typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
1173 static IOReturn
SurfaceApplyPlaneBlock(IOSurfaceRef surface,SurfacePlaneBlock block)1174 SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
1175 {
1176 if (surface == nil || block == nil) {
1177 return kIOReturnBadArgument;
1178 }
1179
1180 IOReturn result = kIOReturnSuccess;
1181 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1182
1183 if (planeCount == 0) {
1184 result = IOSurfaceLock(surface, 0, NULL);
1185 if (result != kIOReturnSuccess) {
1186 return result;
1187 }
1188
1189 void* base = IOSurfaceGetBaseAddress(surface);
1190 size_t rb = IOSurfaceGetBytesPerRow(surface);
1191 size_t w = IOSurfaceGetWidth(surface);
1192 size_t h = IOSurfaceGetHeight(surface);
1193
1194 if (base && rb && w && h) {
1195 block(base, 0, w, h, rb);
1196 }
1197
1198 IOSurfaceUnlock(surface, 0, NULL);
1199 } else if (planeCount == 2) {
1200 for (size_t i = 0; i < planeCount; i++) {
1201 result = IOSurfaceLock(surface, 0, NULL);
1202 if (result != kIOReturnSuccess) {
1203 return result;
1204 }
1205
1206 void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
1207 size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
1208 size_t w = IOSurfaceGetWidthOfPlane(surface, i);
1209 size_t h = IOSurfaceGetHeightOfPlane(surface, i);
1210
1211 if (base && rb && w && h) {
1212 block(base, i, w, h, rb);
1213 }
1214
1215 IOSurfaceUnlock(surface, 0, NULL);
1216 }
1217 }
1218 return result;
1219 }
1220 static void
ClearSurface(IOSurfaceRef surface)1221 ClearSurface(IOSurfaceRef surface)
1222 {
1223 const int zero = 0;
1224 (void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
1225 {
1226 if (i == 0) {
1227 memset(p, zero, rb * h);
1228 } else {
1229 memset(p, 128, rb * h);
1230 }
1231 });
1232 }
1233 static size_t
SurfaceGetMemorySize(IOSurfaceRef surface)1234 SurfaceGetMemorySize(IOSurfaceRef surface)
1235 {
1236 size_t planeCount = IOSurfaceGetPlaneCount(surface);
1237
1238 if (planeCount == 0) {
1239 size_t rb = IOSurfaceGetBytesPerRow(surface);
1240 size_t h = IOSurfaceGetHeight(surface);
1241 return rb * h;
1242 } else if (planeCount == 2) {
1243 size_t rb0 = IOSurfaceGetBytesPerRowOfPlane(surface, 0);
1244 size_t h0 = IOSurfaceGetHeightOfPlane(surface, 0);
1245 size_t rb1 = IOSurfaceGetBytesPerRowOfPlane(surface, 1);
1246 size_t h1 = IOSurfaceGetHeightOfPlane(surface, 1);
1247 return rb0 * h0 + rb1 * h1;
1248 }
1249 return 0;
1250 }
1251 static IOSurfaceRef
CreateSurface(uint32_t pixelsWide,uint32_t pixelsHigh,uint32_t rowBytesAlignment,uint32_t fmt,bool purgeable,bool clear)1252 CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
1253 {
1254 IOSurfaceRef surface = nil;
1255
1256 if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
1257 return nil;
1258 }
1259
1260 size_t bpp = bytes_per_pixel(fmt);
1261 size_t bpe = bytes_per_element(fmt);
1262 if (bpp == 0 || bpe == 0) {
1263 return nil;
1264 }
1265
1266 size_t rowbytes = pixelsWide * bpp;
1267 if (rowBytesAlignment == 0) {
1268 rowBytesAlignment = 16;
1269 }
1270 rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);
1271
1272 CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1273 setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
1274 setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
1275 setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
1276 setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
1277 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1278 setBoolValue(props, kIOSurfaceNonPurgeable, !purgeable);
1279 #else /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1280 (void)purgeable;
1281 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1282 {
1283 if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
1284 setIntValue(props, kIOSurfaceElementWidth, 2);
1285 setIntValue(props, kIOSurfaceElementHeight, 1);
1286 }
1287 setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
1288 }
1289
1290 surface = IOSurfaceCreate(props);
1291
1292 if (clear) {
1293 ClearSurface(surface);
1294 }
1295
1296 CFRelease(props);
1297 return surface;
1298 }
1299 T_DECL(phys_footprint_purgeable_iokit,
1300 "phys_footprint for purgeable IOKit memory",
1301 T_META_LTEPHASE(LTE_POSTINIT))
1302 {
1303 uint64_t footprint_before, pagetable_before;
1304 uint64_t footprint_after, pagetable_after;
1305 uint64_t footprint_expected, footprint_delta_slop;
1306 int64_t footprint_delta;
1307 IOSurfaceRef surface;
1308 uint32_t old_state;
1309 uint64_t surface_size;
1310
1311 T_SETUPBEGIN;
1312 footprint_delta_slop = 8 * vm_kernel_page_size;
1313 ledger_init();
1314 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1315 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1316 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1317 CFRelease(surface);
1318 T_SETUPEND;
1319
1320 surface_size = 1024 * 1024 * 4;
1321
1322 /* create IOsurface: footprint grows */
1323 get_ledger_info(&footprint_before, &pagetable_before);
1324 surface = CreateSurface(1024, 1024, 0, 32, true, true);
1325 get_ledger_info(&footprint_after, &pagetable_after);
1326 if (legacy_footprint) {
1327 footprint_expected = footprint_before;
1328 footprint_expected += (pagetable_after - pagetable_before);
1329 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1330 T_LOG("LEGACY FOOTPRINT: creating purgeable IOSurface: no footprint impact");
1331 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1332 "create purgeable IOSurface %lld bytes: "
1333 "footprint %lld -> %lld expected %lld delta %lld",
1334 surface_size, footprint_before, footprint_after,
1335 footprint_expected, footprint_delta);
1336 } else {
1337 footprint_expected = footprint_before + surface_size;
1338 footprint_expected += (pagetable_after - pagetable_before);
1339 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1340 T_LOG("creating purgeable IOSurface increases phys_footprint");
1341 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1342 "create purgeable IOSurface %lld bytes: "
1343 "footprint %lld -> %lld expected %lld delta %lld",
1344 surface_size, footprint_before, footprint_after,
1345 footprint_expected, footprint_delta);
1346 }
1347
1348 /* make IOSurface volatile: footprint shrinks */
1349 get_ledger_info(&footprint_before, &pagetable_before);
1350 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1351 get_ledger_info(&footprint_after, &pagetable_after);
1352 if (legacy_footprint) {
1353 footprint_expected = footprint_before;
1354 footprint_expected += (pagetable_after - pagetable_before);
1355 T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
1356 T_EXPECT_EQ(footprint_after, footprint_expected,
1357 "volatile IOSurface %lld bytes: "
1358 "footprint %lld -> %lld expected %lld delta %lld",
1359 surface_size, footprint_before, footprint_after,
1360 footprint_expected, footprint_after - footprint_expected);
1361 } else {
1362 footprint_expected = footprint_before - surface_size;
1363 footprint_expected += (pagetable_after - pagetable_before);
1364 T_LOG("making IOSurface volatile decreases phys_footprint");
1365 T_EXPECT_EQ(footprint_after, footprint_expected,
1366 "made volatile %lld bytes: "
1367 "footprint %lld -> %lld expected %lld delta %lld",
1368 surface_size, footprint_before, footprint_after,
1369 footprint_expected, footprint_after - footprint_expected);
1370 }
1371
1372 /* make IOSurface non-volatile: footprint grows */
1373 get_ledger_info(&footprint_before, &pagetable_before);
1374 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
1375 get_ledger_info(&footprint_after, &pagetable_after);
1376 if (legacy_footprint) {
1377 footprint_expected = footprint_before;
1378 footprint_expected += (pagetable_after - pagetable_before);
1379 T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
1380 T_EXPECT_EQ(footprint_after, footprint_expected,
1381 "non-volatile IOSurface %lld bytes: "
1382 "footprint %lld -> %lld expected %lld delta %lld",
1383 surface_size, footprint_before, footprint_after,
1384 footprint_expected, footprint_after - footprint_expected);
1385 } else {
1386 footprint_expected = footprint_before + surface_size;
1387 footprint_expected += (pagetable_after - pagetable_before);
1388 T_LOG("making IOSurface non-volatile increases phys_footprint");
1389 T_EXPECT_EQ(footprint_after, footprint_expected,
1390 "made non-volatile %lld bytes: "
1391 "footprint %lld -> %lld expected %lld delta %lld",
1392 surface_size, footprint_before, footprint_after,
1393 footprint_expected, footprint_after - footprint_expected);
1394 }
1395
1396 /* accessing IOSurface re-mapping: no footprint impact */
1397
1398 /* deallocating IOSurface re-mapping: no footprint impact */
1399
1400 /* release IOSurface: footprint shrinks */
1401 get_ledger_info(&footprint_before, &pagetable_before);
1402 CFRelease(surface);
1403 get_ledger_info(&footprint_after, &pagetable_after);
1404 if (legacy_footprint) {
1405 footprint_expected = footprint_before;
1406 footprint_expected += (pagetable_after - pagetable_before);
1407 T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
1408 T_EXPECT_EQ(footprint_after, footprint_expected,
1409 "releasing IOSurface %lld bytes: "
1410 "footprint %lld -> %lld expected %lld delta %lld",
1411 surface_size, footprint_before, footprint_after,
1412 footprint_expected, footprint_after - footprint_expected);
1413 } else {
1414 footprint_expected = footprint_before - surface_size;
1415 footprint_expected += (pagetable_after - pagetable_before);
1416 T_LOG("releasing IOSurface decreases phys_footprint");
1417 T_EXPECT_EQ(footprint_after, footprint_expected,
1418 "released IOSurface %lld bytes: "
1419 "footprint %lld -> %lld expected %lld delta %lld",
1420 surface_size, footprint_before, footprint_after,
1421 footprint_expected, footprint_after - footprint_expected);
1422 }
1423 }
1424
1425 #if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
1426 T_DECL(phys_footprint_nonpurgeable_iokit,
1427 "phys_footprint for non-purgeable IOKit memory",
1428 T_META_LTEPHASE(LTE_POSTINIT))
1429 {
1430 uint64_t footprint_before, pagetable_before;
1431 uint64_t footprint_after, pagetable_after;
1432 uint64_t footprint_expected, footprint_delta_slop;
1433 int64_t footprint_delta;
1434 IOSurfaceRef surface;
1435 uint64_t surface_size;
1436 void *map_base;
1437 size_t map_size;
1438 mach_vm_address_t remap_addr;
1439 kern_return_t kr;
1440 vm_prot_t cur_prot, max_prot;
1441 uint32_t old_state;
1442
1443
1444 T_SETUPBEGIN;
1445 ledger_init();
1446 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1447 CFRelease(surface);
1448 footprint_delta_slop = 8 * vm_kernel_page_size;
1449 T_SETUPEND;
1450
1451 surface_size = 1024 * 1024 * 4;
1452
1453 /* create IOsurface: footprint grows */
1454 get_ledger_info(&footprint_before, &pagetable_before);
1455 surface = CreateSurface(1024, 1024, 0, 32, false, true);
1456 get_ledger_info(&footprint_after, &pagetable_after);
1457 footprint_expected = footprint_before + surface_size;
1458 footprint_expected += (pagetable_after - pagetable_before);
1459 footprint_delta = (int64_t)(footprint_after - footprint_expected);
1460 T_LOG("creating non-purgeable IOSurface increases phys_footprint");
1461 T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
1462 "create non-purgeable IOSurface %lld bytes: "
1463 "footprint %lld -> %lld expected %lld delta %lld",
1464 surface_size, footprint_before, footprint_after,
1465 footprint_expected, footprint_delta);
1466
1467 /* make IOSurface volatile: fail and no footprint impact */
1468 get_ledger_info(&footprint_before, &pagetable_before);
1469 IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
1470 get_ledger_info(&footprint_after, &pagetable_after);
1471 footprint_expected = footprint_before;
1472 footprint_expected += (pagetable_after - pagetable_before);
1473 T_LOG("making non-purgeable IOSurface volatile: no footprint impact");
1474 T_EXPECT_EQ(footprint_after, footprint_expected,
1475 "made volatile %lld non-purgeable bytes: "
1476 "footprint %lld -> %lld expected %lld delta %lld",
1477 surface_size, footprint_before, footprint_after,
1478 footprint_expected, footprint_after - footprint_expected);
1479
1480 /* re-mapping IOSurface: no footprint impact */
1481 get_ledger_info(&footprint_before, &pagetable_before);
1482 map_base = IOSurfaceGetBaseAddress(surface);
1483 map_size = SurfaceGetMemorySize(surface);
1484 // T_EXPECT_EQ(map_size, surface_size, "map_size %lld surface_size %lld",
1485 // map_size, surface_size);
1486 remap_addr = 0;
1487 kr = mach_vm_remap(mach_task_self(),
1488 &remap_addr,
1489 (mach_vm_size_t)surface_size,
1490 0,
1491 VM_FLAGS_ANYWHERE,
1492 mach_task_self(),
1493 (mach_vm_address_t)map_base,
1494 FALSE, /* copy */
1495 &cur_prot,
1496 &max_prot,
1497 VM_INHERIT_DEFAULT);
1498 T_QUIET;
1499 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_remap() error 0x%x (%s)",
1500 kr, mach_error_string(kr));
1501 get_ledger_info(&footprint_after, &pagetable_after);
1502 footprint_expected = footprint_before;
1503 footprint_expected += (pagetable_after - pagetable_before);
1504 T_LOG("re-mapping IOSurface does not impact phys_footprint");
1505 T_EXPECT_EQ(footprint_after, footprint_expected,
1506 "remapping IOSurface %lld bytes: "
1507 "footprint %lld -> %lld expected %lld delta %lld",
1508 surface_size, footprint_before, footprint_after,
1509 footprint_expected, footprint_after - footprint_expected);
1510
1511 /* accessing IOSurface re-mapping: footprint grows */
1512 get_ledger_info(&footprint_before, &pagetable_before);
1513 memset((char *)(uintptr_t)remap_addr, 'p', (size_t)surface_size);
1514 get_ledger_info(&footprint_after, &pagetable_after);
1515 footprint_expected = footprint_before + surface_size;
1516 footprint_expected += (pagetable_after - pagetable_before);
1517 T_LOG("accessing re-mapped IOSurface grows phys_footprint");
1518 T_EXPECT_EQ(footprint_after, footprint_expected,
1519 "accessing remapped IOSurface %lld bytes: "
1520 "footprint %lld -> %lld expected %lld delta %lld",
1521 surface_size, footprint_before, footprint_after,
1522 footprint_expected, footprint_after - footprint_expected);
1523
1524 /* deallocating IOSurface re-mapping: footprint shrinks */
1525 get_ledger_info(&footprint_before, &pagetable_before);
1526 kr = mach_vm_deallocate(mach_task_self(),
1527 remap_addr,
1528 (mach_vm_size_t)surface_size);
1529 T_QUIET;
1530 T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
1531 kr, mach_error_string(kr));
1532 get_ledger_info(&footprint_after, &pagetable_after);
1533 footprint_expected = footprint_before - surface_size;
1534 footprint_expected += (pagetable_after - pagetable_before);
1535 T_LOG("deallocating re-mapping of IOSurface shrinks phys_footprint");
1536 T_EXPECT_EQ(footprint_after, footprint_expected,
1537 "deallocating remapped IOSurface %lld bytes: "
1538 "footprint %lld -> %lld expected %lld delta %lld",
1539 surface_size, footprint_before, footprint_after,
1540 footprint_expected, footprint_after - footprint_expected);
1541
1542 /* release IOSurface: footprint shrinks */
1543 get_ledger_info(&footprint_before, &pagetable_before);
1544 CFRelease(surface);
1545 get_ledger_info(&footprint_after, &pagetable_after);
1546 footprint_expected = footprint_before - surface_size;
1547 footprint_expected += (pagetable_after - pagetable_before);
1548 T_LOG("releasing IOSurface decreases phys_footprint");
1549 T_EXPECT_EQ(footprint_after, footprint_expected,
1550 "released IOSurface %lld bytes: "
1551 "footprint %lld -> %lld expected %lld delta %lld",
1552 surface_size, footprint_before, footprint_after,
1553 footprint_expected, footprint_after - footprint_expected);
1554 }
1555 #endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
1556