#include <darwintest.h>
#include <darwintest_utils.h>

#include <mach/mach_error.h>
#include <mach/mach_init.h>
#include <mach/memory_entry.h>
#include <mach/mach_port.h>
#include <mach/mach_vm.h>
#include <mach/task.h>
#include <mach/task_info.h>
#include <mach/vm_map.h>

#include <sys/mman.h>
#include <sys/types.h>
#include <sys/sysctl.h>

#include <TargetConditionals.h>

#include <Kernel/kern/ledger.h>
extern int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3);

boolean_t legacy_footprint;

#if LEGACY_FOOTPRINT_ENTITLED && defined(__arm64__)
#define TEST_VM_NAMESPACE "xnu.vm_legacy"
#else /* ENTITLED && __arm64__ */
#define TEST_VM_NAMESPACE "xnu.vm"
#endif /* ENTITLED && __arm64__ */

T_GLOBAL_META(
	T_META_NAMESPACE(TEST_VM_NAMESPACE),
	T_META_RADAR_COMPONENT_NAME("xnu"),
	T_META_RADAR_COMPONENT_VERSION("VM"),
	T_META_RUN_CONCURRENTLY(true));

#define MEM_SIZE (100 * 1024 * 1024) /* 100 MB */

static int64_t ledger_count = -1;
static int footprint_index = -1;
static int pagetable_index = -1;
static struct ledger_entry_info *lei = NULL;

static void
ledger_init(void)
{
	static int                      ledger_inited = 0;
	struct ledger_info              li;
	struct ledger_template_info     *templateInfo;
	int64_t                         templateCnt;
	int                             i;
	int                             legacy_footprint_entitlement_mode;
	size_t                          oldlen;

	if (ledger_inited) {
		return;
	}
	ledger_inited = 1;

	T_SETUPBEGIN;

	legacy_footprint = FALSE;
#if LEGACY_FOOTPRINT_ENTITLED
	int ret;

	T_QUIET;
	T_WITH_ERRNO;
	oldlen = sizeof(legacy_footprint_entitlement_mode);
	ret = sysctlbyname("kern.legacy_footprint_entitlement_mode",
	    &legacy_footprint_entitlement_mode,
	    &oldlen,
	    NULL,
	    0);
	if (ret == 0 && legacy_footprint_entitlement_mode == 2) {
		legacy_footprint = TRUE;
	}
#endif /* LEGACY_FOOTPRINT_ENTITLED */

	T_QUIET;
	T_WITH_ERRNO;
	T_ASSERT_EQ(ledger(LEDGER_INFO,
	    (caddr_t)(uintptr_t)getpid(),
	    (caddr_t)&li,
	    NULL),
	    0,
	    "ledger(LEDGER_INFO)");

	templateCnt = li.li_entries;
	templateInfo = malloc((size_t)li.li_entries * sizeof(struct ledger_template_info));
	T_QUIET;
	T_WITH_ERRNO;
	T_ASSERT_NE(templateInfo, NULL, "malloc()");

	ledger_count = li.li_entries;
	footprint_index = -1;
	pagetable_index = -1;
	T_QUIET;
	T_WITH_ERRNO;
	T_ASSERT_GE(ledger(LEDGER_TEMPLATE_INFO,
	    (caddr_t)templateInfo,
	    (caddr_t)&templateCnt,
	    NULL),
	    0,
	    "ledger(LEDGER_TEMPLATE_INFO)");
	for (i = 0; i < templateCnt; i++) {
		if (!strncmp(templateInfo[i].lti_name,
		    "phys_footprint",
		    strlen("phys_footprint"))) {
			footprint_index = i;
		} else if (!strncmp(templateInfo[i].lti_name,
		    "page_table",
		    strlen("page_table"))) {
			pagetable_index = i;
		}
	}
	free(templateInfo);

	lei = (struct ledger_entry_info *)
	    malloc((size_t)ledger_count * sizeof(*lei));
	T_QUIET;
	T_WITH_ERRNO;
	T_ASSERT_NE(lei, NULL, "malloc(ledger_entry_info)");

	T_QUIET;
	T_ASSERT_NE(footprint_index, -1, "no footprint_index");
	T_QUIET;
	T_ASSERT_NE(pagetable_index, -1, "no pagetable_index");

	T_SETUPEND;
}

static void
get_ledger_info(
	uint64_t        *phys_footprint,
	uint64_t        *page_table)
{
	int64_t count;

	count = ledger_count;
	T_QUIET;
	T_WITH_ERRNO;
	T_ASSERT_GE(ledger(LEDGER_ENTRY_INFO,
	    (caddr_t)(uintptr_t)getpid(),
	    (caddr_t)lei,
	    (caddr_t)&count),
	    0,
	    "ledger(LEDGER_ENTRY_INFO)");
	T_QUIET;
	T_ASSERT_GT(count, (int64_t)footprint_index, "no entry for footprint");
	T_QUIET;
	T_ASSERT_GT(count, (int64_t)pagetable_index, "no entry for pagetable");
	if (phys_footprint) {
		*phys_footprint = (uint64_t)(lei[footprint_index].lei_balance);
	}
	if (page_table) {
		*page_table = (uint64_t)(lei[pagetable_index].lei_balance);
	}
}

static mach_vm_address_t
pre_warm(
	mach_vm_size_t  vm_size)
{
	kern_return_t           kr;
	mach_vm_address_t       vm_addr;
	unsigned char           BigBufOnStack[100 * 1024];
	uint64_t                footprint, page_table;

	/* make sure ledgers are ready to be queried */
	ledger_init();

	T_SETUPBEGIN;

	/*
	 * Touch a few pages ahead on the stack, to make
	 * sure we don't see a footprint increase due to
	 * an extra stack page later.
	 */
	memset(BigBufOnStack, 0xb, sizeof(BigBufOnStack));
	T_QUIET;
	T_EXPECT_EQ(BigBufOnStack[0], 0xb,
	    "BigBufOnStack[0] == 0x%x",
	    BigBufOnStack[0]);
	T_QUIET;
	T_EXPECT_EQ(BigBufOnStack[sizeof(BigBufOnStack) - 1], 0xb,
	    "BigBufOnStack[%lu] == 0x%x",
	    sizeof(BigBufOnStack),
	    BigBufOnStack[sizeof(BigBufOnStack) - 1]);

	/*
	 * Pre-allocate, touch and then release the same amount
	 * of memory we'll be allocating later during the test,
	 * to account for any memory overhead (page tables, global
	 * variables, ...).
	 */
	vm_addr = 0;
	kr = mach_vm_allocate(mach_task_self(),
	    &vm_addr,
	    vm_size,
	    VM_FLAGS_ANYWHERE);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate(%lld) error 0x%x (%s)",
	    vm_size, kr, mach_error_string(kr));
	memset((char *)(uintptr_t)vm_addr, 'p', (size_t)vm_size);
	kr = mach_vm_deallocate(mach_task_self(),
	    vm_addr,
	    vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));

	/*
	 * Exercise the ledger code to make sure it's ready to run
	 * without any extra memory overhead later.
	 */
	get_ledger_info(&footprint, &page_table);

	T_SETUPEND;

	/*
	 * Return the start of the virtual range we pre-warmed, so that the
	 * test can check that it's using the same range.
	 */
	return vm_addr;
}

T_DECL(phys_footprint_anonymous,
    "phys_footprint for anonymous memory",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	kern_return_t           kr;
	mach_vm_address_t       pre_vm_addr, vm_addr;
	mach_vm_size_t          vm_size, dirty_size;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(MEM_SIZE);

	/* allocating virtual memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_addr = 0;
	vm_size = MEM_SIZE;
	kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
	    VM_FLAGS_ANYWHERE);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("virtual allocation does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "virtual allocation of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = vm_size / 2;
	memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying anonymous memory increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating dirty anonymous memory decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocated %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}

#define TEMP_FILE_TEMPLATE "/tmp/phys_footprint_data.XXXXXXXX"
#define TEMP_FILE_SIZE  (1 * 1024 * 1024)

T_DECL(phys_footprint_file,
    "phys_footprint for mapped file",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	mach_vm_address_t       pre_vm_addr;
	int                     fd;
	char                    *map_addr;
	size_t                  map_size, dirty_size;
	ssize_t                 nbytes;
	char                    tmp_file_name[PATH_MAX] = TEMP_FILE_TEMPLATE;
	char                    *buf;
	size_t                  buf_size;

	T_SETUPBEGIN;
	buf_size = TEMP_FILE_SIZE;
	T_QUIET;
	T_ASSERT_NOTNULL(buf = (char *)malloc(buf_size),
	    "allocate %zu-byte buffer", buf_size);
	memset(buf, 'f', buf_size);
	T_WITH_ERRNO;
	T_QUIET;
	T_ASSERT_NOTNULL(mktemp(tmp_file_name),
	    "create temporary file name");
	T_WITH_ERRNO;
	T_QUIET;
	T_ASSERT_GE(fd = open(tmp_file_name, O_CREAT | O_RDWR),
	    0,
	    "create temp file");
	T_WITH_ERRNO;
	T_QUIET;
	T_ASSERT_EQ(nbytes = write(fd, buf, buf_size),
	    (ssize_t)buf_size,
	    "write %zu bytes", buf_size);
	free(buf);
	T_SETUPEND;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(TEMP_FILE_SIZE);

	/* mapping a file does not impact footprint... */
	get_ledger_info(&footprint_before, &pagetable_before);
	map_size = TEMP_FILE_SIZE;
	T_WITH_ERRNO;
	T_QUIET;
	T_ASSERT_NOTNULL(map_addr = (char *)mmap(NULL, map_size,
	    PROT_READ | PROT_WRITE,
	    MAP_FILE | MAP_SHARED, fd, 0),
	    "mmap()");
	T_QUIET;
	T_EXPECT_EQ((mach_vm_address_t)map_addr, pre_vm_addr,
	    "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("mapping file does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "mapping file with %zu bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    map_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching file-backed memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = map_size / 2;
	memset(map_addr, 'F', dirty_size);
	/* ... should not impact footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying file-backed memory does not impact phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %zu bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating file-backed memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	T_WITH_ERRNO;
	T_QUIET;
	T_ASSERT_EQ(munmap(map_addr, map_size),
	    0,
	    "unmap file");
	/* ... should not impact footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("unmapping file-backed memory does not impact phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "unmapped %zu dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_purgeable,
    "phys_footprint for purgeable memory",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	kern_return_t           kr;
	mach_vm_address_t       pre_vm_addr, vm_addr;
	mach_vm_size_t          vm_size, dirty_size;
	int                     state;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(MEM_SIZE);

	/* allocating purgeable virtual memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_addr = 0;
	vm_size = MEM_SIZE;
	kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
	    VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("purgeable virtual allocation does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "purgeable virtual allocation of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = vm_size / 2;
	memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying anonymous memory increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making it volatile... */
	get_ledger_info(&footprint_before, &pagetable_before);
	state = VM_PURGABLE_VOLATILE;
	T_QUIET;
	T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
	    vm_addr,
	    VM_PURGABLE_SET_STATE,
	    &state),
	    KERN_SUCCESS,
	    "vm_purgable_control(VOLATILE)");
	T_QUIET;
	T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
	    "memory was non-volatile");
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making volatile decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making it non-volatile... */
	get_ledger_info(&footprint_before, &pagetable_before);
	state = VM_PURGABLE_NONVOLATILE;
	T_QUIET;
	T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
	    vm_addr,
	    VM_PURGABLE_SET_STATE,
	    &state),
	    KERN_SUCCESS,
	    "vm_purgable_control(NONVOLATILE)");
	T_QUIET;
	T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
	    "memory was volatile");
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making non-volatile increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made non-volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating memory decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocated %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_purgeable_ownership,
    "phys_footprint for owned purgeable memory",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	kern_return_t           kr;
	mach_vm_address_t       pre_vm_addr, vm_addr;
	mach_vm_size_t          vm_size, dirty_size, me_size;
	int                     state;
	mach_port_t             me_port;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(MEM_SIZE);

	/* allocating purgeable virtual memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_addr = 0;
	vm_size = MEM_SIZE;
	kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
	    VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_allocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("purgeable virtual allocation does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "purgeable virtual allocation of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = vm_size / 2;
	memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying anonymous memory increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making it volatile... */
	get_ledger_info(&footprint_before, &pagetable_before);
	state = VM_PURGABLE_VOLATILE;
	T_QUIET;
	T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
	    vm_addr,
	    VM_PURGABLE_SET_STATE,
	    &state),
	    KERN_SUCCESS,
	    "vm_purgable_control(VOLATILE)");
	T_QUIET;
	T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
	    "memory was non-volatile");
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making volatile decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making it non-volatile... */
	get_ledger_info(&footprint_before, &pagetable_before);
	state = VM_PURGABLE_NONVOLATILE;
	T_QUIET;
	T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
	    vm_addr,
	    VM_PURGABLE_SET_STATE,
	    &state),
	    KERN_SUCCESS,
	    "vm_purgable_control(NONVOLATILE)");
	T_QUIET;
	T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
	    "memory was volatile");
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making non-volatile increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made non-volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making a memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	me_size = vm_size;
	me_port = MACH_PORT_NULL;
	kr = mach_make_memory_entry_64(mach_task_self(),
	    &me_size,
	    vm_addr,
	    VM_PROT_READ | VM_PROT_WRITE,
	    &me_port,
	    MACH_PORT_NULL);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making a memory entry does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "making a memory entry of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating memory while holding memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating owned memory while holding memory entry "
	    "does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocated %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* releasing the memory entry... */
	kr = mach_port_deallocate(mach_task_self(), me_port);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("releasing memory entry decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}

#ifdef MAP_MEM_LEDGER_TAGGED
T_DECL(phys_footprint_ledger_purgeable_owned,
    "phys_footprint for ledger-tagged purgeable memory ownership",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	kern_return_t           kr;
	mach_vm_address_t       pre_vm_addr, vm_addr;
	mach_vm_size_t          vm_size, dirty_size, me_size;
	int                     state;
	mach_port_t             me_port;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(MEM_SIZE);

	/* making a memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_size = MEM_SIZE;
	me_size = vm_size;
	me_port = MACH_PORT_NULL;
	kr = mach_make_memory_entry_64(mach_task_self(),
	    &me_size,
	    0,
	    (MAP_MEM_NAMED_CREATE |
	    MAP_MEM_LEDGER_TAGGED |
	    MAP_MEM_PURGABLE |
	    VM_PROT_READ | VM_PROT_WRITE),
	    &me_port,
	    MACH_PORT_NULL);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making a memory entry does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "making a memory entry of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* mapping ledger-tagged virtual memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_addr = 0;
	kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
	    0, /* mask */
	    VM_FLAGS_ANYWHERE,
	    me_port,
	    0, /* offset */
	    FALSE, /* copy */
	    VM_PROT_READ | VM_PROT_WRITE,
	    VM_PROT_READ | VM_PROT_WRITE,
	    VM_INHERIT_DEFAULT);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("mapping ledger-tagged memory does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "ledger-tagged mapping of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = vm_size / 2;
	memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying ledger-tagged memory increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making it volatile... */
	get_ledger_info(&footprint_before, &pagetable_before);
	state = VM_PURGABLE_VOLATILE;
	T_QUIET;
	T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
	    vm_addr,
	    VM_PURGABLE_SET_STATE,
	    &state),
	    KERN_SUCCESS,
	    "vm_purgable_control(VOLATILE)");
	T_QUIET;
	T_ASSERT_EQ(state, VM_PURGABLE_NONVOLATILE,
	    "memory was non-volatile");
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making volatile decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* making it non-volatile... */
	get_ledger_info(&footprint_before, &pagetable_before);
	state = VM_PURGABLE_NONVOLATILE;
	T_QUIET;
	T_ASSERT_EQ(mach_vm_purgable_control(mach_task_self(),
	    vm_addr,
	    VM_PURGABLE_SET_STATE,
	    &state),
	    KERN_SUCCESS,
	    "vm_purgable_control(NONVOLATILE)");
	T_QUIET;
	T_ASSERT_EQ(state, VM_PURGABLE_VOLATILE,
	    "memory was volatile");
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making non-volatile increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made non-volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating memory while holding memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating owned memory while holding memory entry "
	    "does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocated %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* releasing the memory entry... */
	kr = mach_port_deallocate(mach_task_self(), me_port);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("releasing memory entry decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_ledger_owned,
    "phys_footprint for ledger-tagged memory ownership",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	kern_return_t           kr;
	mach_vm_address_t       pre_vm_addr, vm_addr;
	mach_vm_size_t          vm_size, dirty_size, me_size;
	mach_port_t             me_port;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(MEM_SIZE);

	/* making a memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_size = MEM_SIZE;
	me_size = vm_size;
	me_port = MACH_PORT_NULL;
	kr = mach_make_memory_entry_64(mach_task_self(),
	    &me_size,
	    0,
	    (MAP_MEM_NAMED_CREATE |
	    MAP_MEM_LEDGER_TAGGED |
	    VM_PROT_READ | VM_PROT_WRITE),
	    &me_port,
	    MACH_PORT_NULL);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making a memory entry does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "making a memory entry of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* mapping ledger-tagged virtual memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_addr = 0;
	kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
	    0, /* mask */
	    VM_FLAGS_ANYWHERE,
	    me_port,
	    0, /* offset */
	    FALSE, /* copy */
	    VM_PROT_READ | VM_PROT_WRITE,
	    VM_PROT_READ | VM_PROT_WRITE,
	    VM_INHERIT_DEFAULT);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("mapping ledger-tagged memory does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "ledger-tagged mapping of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = vm_size / 2;
	memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
	/* ... should increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying ledger-tagged memory increases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating memory while holding memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating owned memory while holding memory entry "
	    "does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocated %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* releasing the memory entry... */
	kr = mach_port_deallocate(mach_task_self(), me_port);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should decrease footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - dirty_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("releasing memory entry decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}

T_DECL(phys_footprint_no_footprint_for_debug,
    "phys_footprint for no_footprint_for_debug",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t                footprint_before, pagetable_before;
	uint64_t                footprint_after, pagetable_after;
	uint64_t                footprint_expected;
	kern_return_t           kr;
	mach_vm_address_t       pre_vm_addr, vm_addr;
	mach_vm_size_t          vm_size, dirty_size, me_size;
	mach_port_t             me_port;
	int                     new_value, ret;

	/* pre-warm to account for page table expansion */
	pre_vm_addr = pre_warm(MEM_SIZE);

	/* making a memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_size = MEM_SIZE;
	me_size = vm_size;
	me_port = MACH_PORT_NULL;
	kr = mach_make_memory_entry_64(mach_task_self(),
	    &me_size,
	    0,
	    (MAP_MEM_NAMED_CREATE |
	    MAP_MEM_LEDGER_TAGGED |
	    VM_PROT_READ | VM_PROT_WRITE),
	    &me_port,
	    MACH_PORT_NULL);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "make_memory_entry() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(me_size, vm_size, "memory entry size mismatch");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making a memory entry does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "making a memory entry of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* trying to hide debug memory from footprint while not allowed */
	kr = mach_memory_entry_ownership(me_port,
	    mach_task_self(),
	    VM_LEDGER_TAG_DEFAULT,
	    VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_NO_ACCESS,
	    "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) fails without sysctl");

	/* let's get permission to hide debug memory */
	new_value = 1;
	ret = sysctlbyname("vm.task_no_footprint_for_debug", NULL, NULL, &new_value, sizeof(new_value));
	if (ret == -1 && errno == ENOENT) {
		T_SKIP("sysctlbyname(vm.task_no_footprint_for_debug) ENOENT");
	}
	T_QUIET;
	T_ASSERT_POSIX_SUCCESS(ret, "sysctlbyname(vm.task_no_footprint_for_debug)");

	/* trying to hide debug memory from footprint while allowed */
	kr = mach_memory_entry_ownership(me_port,
	    mach_task_self(),
	    VM_LEDGER_TAG_DEFAULT,
	    VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS,
	    "mach_memory_entry_ownership(NO_FOOTPRINT_FOR_DEBUG) succeeds after sysctl");

	/* mapping ledger-tagged virtual memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	vm_addr = 0;
	kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
	    0, /* mask */
	    VM_FLAGS_ANYWHERE,
	    me_port,
	    0, /* offset */
	    FALSE, /* copy */
	    VM_PROT_READ | VM_PROT_WRITE,
	    VM_PROT_READ | VM_PROT_WRITE,
	    VM_INHERIT_DEFAULT);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_map() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	T_QUIET;
	T_EXPECT_EQ(vm_addr, pre_vm_addr, "pre-warm mishap");
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("mapping ledger-tagged memory does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "ledger-tagged mapping of %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    vm_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* touching memory... */
	get_ledger_info(&footprint_before, &pagetable_before);
	dirty_size = vm_size / 2;
	memset((char *)(uintptr_t)vm_addr, 'x', (size_t)dirty_size);
	/* ... should not increase footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("modifying no_footprint_for_debug memory does not increase phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "touched %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating memory while holding memory entry... */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating owned memory while holding memory entry "
	    "does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocated %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* releasing the memory entry... */
	kr = mach_port_deallocate(mach_task_self(), me_port);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "mach_port_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	/* ... should not change footprint */
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("releasing memory entry does not change phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld dirty bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    dirty_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}
#endif /* MAP_MEM_LEDGER_TAGGED */

/* IOSurface code from: CoreImage/CoreImageTests/CIRender/SurfaceUtils.c */
#include <CoreFoundation/CoreFoundation.h>
#include <IOSurface/IOSurface.h>
#include <IOSurface/IOSurfacePrivate.h>
static size_t
bytes_per_element(uint32_t format)
{
	size_t bpe = 0;
	switch (format) {
	case 32:     // kCVPixelFormatType_32ARGB (ARGB8)
		bpe = 4;
		break;
	default:
		bpe = 0;
		break;
	}
	return bpe;
}
static size_t
bytes_per_pixel(uint32_t format)
{
	size_t bpe = 0;
	switch (format) {
	case 32:     // kCVPixelFormatType_32ARGB (ARGB8)
		bpe = 4;
		break;
	default:
		bpe = 0;
		break;
	}
	return bpe;
}
static inline size_t
roundSizeToMultiple(size_t size, size_t mult)
{
	return ((size + mult - 1) / mult) * mult;
}
static inline void
setIntValue(CFMutableDictionaryRef dict, const CFStringRef key, int value)
{
	CFNumberRef number = CFNumberCreate(0, kCFNumberIntType, &value);
	CFDictionarySetValue(dict, key, number);
	CFRelease(number);
}
static inline void
setBoolValue(CFMutableDictionaryRef dict, const CFStringRef key, bool value)
{
	CFDictionarySetValue(dict, key, value ? kCFBooleanTrue : kCFBooleanFalse);
}
typedef void (^SurfacePlaneBlock)(void *data, size_t planeIndex, size_t width, size_t height, size_t rowbytes);
static IOReturn
SurfaceApplyPlaneBlock(IOSurfaceRef surface, SurfacePlaneBlock block)
{
	if (surface == nil || block == nil) {
		return kIOReturnBadArgument;
	}

	IOReturn result = kIOReturnSuccess;
	size_t planeCount = IOSurfaceGetPlaneCount(surface);

	if (planeCount == 0) {
		result = IOSurfaceLock(surface, 0, NULL);
		if (result != kIOReturnSuccess) {
			return result;
		}

		void* base = IOSurfaceGetBaseAddress(surface);
		size_t rb = IOSurfaceGetBytesPerRow(surface);
		size_t w = IOSurfaceGetWidth(surface);
		size_t h = IOSurfaceGetHeight(surface);

		if (base && rb && w && h) {
			block(base, 0, w, h, rb);
		}

		IOSurfaceUnlock(surface, 0, NULL);
	} else if (planeCount == 2) {
		for (size_t i = 0; i < planeCount; i++) {
			result = IOSurfaceLock(surface, 0, NULL);
			if (result != kIOReturnSuccess) {
				return result;
			}

			void* base = IOSurfaceGetBaseAddressOfPlane(surface, i);
			size_t rb = IOSurfaceGetBytesPerRowOfPlane(surface, i);
			size_t w = IOSurfaceGetWidthOfPlane(surface, i);
			size_t h = IOSurfaceGetHeightOfPlane(surface, i);

			if (base && rb && w && h) {
				block(base, i, w, h, rb);
			}

			IOSurfaceUnlock(surface, 0, NULL);
		}
	}
	return result;
}
static void
ClearSurface(IOSurfaceRef surface)
{
	const int zero = 0;
	(void) SurfaceApplyPlaneBlock(surface, ^(void *p, size_t i, __unused size_t w, size_t h, size_t rb)
	{
		if (i == 0) {
		        memset(p, zero, rb * h);
		} else {
		        memset(p, 128, rb * h);
		}
	});
}
static size_t
SurfaceGetMemorySize(IOSurfaceRef surface)
{
	size_t planeCount = IOSurfaceGetPlaneCount(surface);

	if (planeCount == 0) {
		size_t rb = IOSurfaceGetBytesPerRow(surface);
		size_t h = IOSurfaceGetHeight(surface);
		return rb * h;
	} else if (planeCount == 2) {
		size_t rb0 = IOSurfaceGetBytesPerRowOfPlane(surface, 0);
		size_t h0 = IOSurfaceGetHeightOfPlane(surface, 0);
		size_t rb1 = IOSurfaceGetBytesPerRowOfPlane(surface, 1);
		size_t h1 = IOSurfaceGetHeightOfPlane(surface, 1);
		return rb0 * h0 + rb1 * h1;
	}
	return 0;
}
static IOSurfaceRef
CreateSurface(uint32_t pixelsWide, uint32_t pixelsHigh, uint32_t rowBytesAlignment, uint32_t fmt, bool purgeable, bool clear)
{
	IOSurfaceRef surface = nil;

	if (pixelsWide < 1 || pixelsHigh < 1 || fmt == 0) {
		return nil;
	}

	size_t bpp = bytes_per_pixel(fmt);
	size_t bpe = bytes_per_element(fmt);
	if (bpp == 0 || bpe == 0) {
		return nil;
	}

	size_t rowbytes = pixelsWide * bpp;
	if (rowBytesAlignment == 0) {
		rowBytesAlignment = 16;
	}
	rowbytes = roundSizeToMultiple(rowbytes, rowBytesAlignment);

	CFMutableDictionaryRef props = CFDictionaryCreateMutable(0, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
	setIntValue(props, kIOSurfaceBytesPerRow, (int)rowbytes);
	setIntValue(props, kIOSurfaceWidth, (int)pixelsWide);
	setIntValue(props, kIOSurfaceHeight, (int)pixelsHigh);
	setIntValue(props, kIOSurfacePixelFormat, (int)fmt);
#if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
	setBoolValue(props, kIOSurfaceNonPurgeable, !purgeable);
#else /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
	(void)purgeable;
#endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */
	{
		if (bpe != bpp) { // i.e. a 422 format such as 'yuvf' etc.
			setIntValue(props, kIOSurfaceElementWidth, 2);
			setIntValue(props, kIOSurfaceElementHeight, 1);
		}
		setIntValue(props, kIOSurfaceBytesPerElement, (int)bpe);
	}

	surface = IOSurfaceCreate(props);

	if (clear) {
		ClearSurface(surface);
	}

	CFRelease(props);
	return surface;
}
T_DECL(phys_footprint_purgeable_iokit,
    "phys_footprint for purgeable IOKit memory",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t        footprint_before, pagetable_before;
	uint64_t        footprint_after, pagetable_after;
	uint64_t        footprint_expected, footprint_delta_slop;
	int64_t         footprint_delta;
	IOSurfaceRef    surface;
	uint32_t        old_state;
	uint64_t        surface_size;

	T_SETUPBEGIN;
	footprint_delta_slop = 8 * vm_kernel_page_size;
	ledger_init();
	surface = CreateSurface(1024, 1024, 0, 32, true, true);
	IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
	IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
	CFRelease(surface);
	T_SETUPEND;

	surface_size = 1024 * 1024 * 4;

	/* create IOsurface: footprint grows */
	get_ledger_info(&footprint_before, &pagetable_before);
	surface = CreateSurface(1024, 1024, 0, 32, true, true);
	get_ledger_info(&footprint_after, &pagetable_after);
	if (legacy_footprint) {
		footprint_expected = footprint_before;
		footprint_expected += (pagetable_after - pagetable_before);
		footprint_delta = (int64_t)(footprint_after - footprint_expected);
		T_LOG("LEGACY FOOTPRINT: creating purgeable IOSurface: no footprint impact");
		T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
		    "create purgeable IOSurface %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_delta);
	} else {
		footprint_expected = footprint_before + surface_size;
		footprint_expected += (pagetable_after - pagetable_before);
		footprint_delta = (int64_t)(footprint_after - footprint_expected);
		T_LOG("creating purgeable IOSurface increases phys_footprint");
		T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
		    "create purgeable IOSurface %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_delta);
	}

	/* make IOSurface volatile: footprint shrinks */
	get_ledger_info(&footprint_before, &pagetable_before);
	IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
	get_ledger_info(&footprint_after, &pagetable_after);
	if (legacy_footprint) {
		footprint_expected = footprint_before;
		footprint_expected += (pagetable_after - pagetable_before);
		T_LOG("LEGACY FOOTPRINT: volatile IOSurface: no footprint impact");
		T_EXPECT_EQ(footprint_after, footprint_expected,
		    "volatile IOSurface %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_after - footprint_expected);
	} else {
		footprint_expected = footprint_before - surface_size;
		footprint_expected += (pagetable_after - pagetable_before);
		T_LOG("making IOSurface volatile decreases phys_footprint");
		T_EXPECT_EQ(footprint_after, footprint_expected,
		    "made volatile %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_after - footprint_expected);
	}

	/* make IOSurface non-volatile: footprint grows */
	get_ledger_info(&footprint_before, &pagetable_before);
	IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableNonVolatile, &old_state);
	get_ledger_info(&footprint_after, &pagetable_after);
	if (legacy_footprint) {
		footprint_expected = footprint_before;
		footprint_expected += (pagetable_after - pagetable_before);
		T_LOG("LEGACY FOOTPRINT: non-volatile IOSurface: no footprint impact");
		T_EXPECT_EQ(footprint_after, footprint_expected,
		    "non-volatile IOSurface %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_after - footprint_expected);
	} else {
		footprint_expected = footprint_before + surface_size;
		footprint_expected += (pagetable_after - pagetable_before);
		T_LOG("making IOSurface non-volatile increases phys_footprint");
		T_EXPECT_EQ(footprint_after, footprint_expected,
		    "made non-volatile %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_after - footprint_expected);
	}

	/* accessing IOSurface re-mapping: no footprint impact */

	/* deallocating IOSurface re-mapping: no footprint impact */

	/* release IOSurface: footprint shrinks */
	get_ledger_info(&footprint_before, &pagetable_before);
	CFRelease(surface);
	get_ledger_info(&footprint_after, &pagetable_after);
	if (legacy_footprint) {
		footprint_expected = footprint_before;
		footprint_expected += (pagetable_after - pagetable_before);
		T_LOG("LEGACY FOOTPRINT: release IOSurface: no footprint impact");
		T_EXPECT_EQ(footprint_after, footprint_expected,
		    "releasing IOSurface %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_after - footprint_expected);
	} else {
		footprint_expected = footprint_before - surface_size;
		footprint_expected += (pagetable_after - pagetable_before);
		T_LOG("releasing IOSurface decreases phys_footprint");
		T_EXPECT_EQ(footprint_after, footprint_expected,
		    "released IOSurface %lld bytes: "
		    "footprint %lld -> %lld expected %lld delta %lld",
		    surface_size, footprint_before, footprint_after,
		    footprint_expected, footprint_after - footprint_expected);
	}
}

#if (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR)
T_DECL(phys_footprint_nonpurgeable_iokit,
    "phys_footprint for non-purgeable IOKit memory",
    T_META_LTEPHASE(LTE_POSTINIT))
{
	uint64_t        footprint_before, pagetable_before;
	uint64_t        footprint_after, pagetable_after;
	uint64_t        footprint_expected, footprint_delta_slop;
	int64_t         footprint_delta;
	IOSurfaceRef    surface;
	uint64_t        surface_size;
	void            *map_base;
	size_t          map_size;
	mach_vm_address_t remap_addr;
	kern_return_t kr;
	vm_prot_t       cur_prot, max_prot;
	uint32_t        old_state;


	T_SETUPBEGIN;
	ledger_init();
	surface = CreateSurface(1024, 1024, 0, 32, false, true);
	CFRelease(surface);
	footprint_delta_slop = 8 * vm_kernel_page_size;
	T_SETUPEND;

	surface_size = 1024 * 1024 * 4;

	/* create IOsurface: footprint grows */
	get_ledger_info(&footprint_before, &pagetable_before);
	surface = CreateSurface(1024, 1024, 0, 32, false, true);
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + surface_size;
	footprint_expected += (pagetable_after - pagetable_before);
	footprint_delta = (int64_t)(footprint_after - footprint_expected);
	T_LOG("creating non-purgeable IOSurface increases phys_footprint");
	T_EXPECT_LE((uint64_t)llabs(footprint_delta), footprint_delta_slop,
	    "create non-purgeable IOSurface %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    surface_size, footprint_before, footprint_after,
	    footprint_expected, footprint_delta);

	/* make IOSurface volatile: fail and no footprint impact */
	get_ledger_info(&footprint_before, &pagetable_before);
	IOSurfaceSetPurgeable(surface, kIOSurfacePurgeableVolatile, &old_state);
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("making non-purgeable IOSurface volatile: no footprint impact");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "made volatile %lld non-purgeable bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    surface_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* re-mapping IOSurface: no footprint impact */
	get_ledger_info(&footprint_before, &pagetable_before);
	map_base = IOSurfaceGetBaseAddress(surface);
	map_size = SurfaceGetMemorySize(surface);
//	T_EXPECT_EQ(map_size, surface_size, "map_size %lld surface_size %lld",
//		    map_size, surface_size);
	remap_addr = 0;
	kr = mach_vm_remap(mach_task_self(),
	    &remap_addr,
	    (mach_vm_size_t)surface_size,
	    0,
	    VM_FLAGS_ANYWHERE,
	    mach_task_self(),
	    (mach_vm_address_t)map_base,
	    FALSE,                /* copy */
	    &cur_prot,
	    &max_prot,
	    VM_INHERIT_DEFAULT);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_remap() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("re-mapping IOSurface does not impact phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "remapping IOSurface %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    surface_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* accessing IOSurface re-mapping: footprint grows */
	get_ledger_info(&footprint_before, &pagetable_before);
	memset((char *)(uintptr_t)remap_addr, 'p', (size_t)surface_size);
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before + surface_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("accessing re-mapped IOSurface grows phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "accessing remapped IOSurface %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    surface_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* deallocating IOSurface re-mapping: footprint shrinks */
	get_ledger_info(&footprint_before, &pagetable_before);
	kr = mach_vm_deallocate(mach_task_self(),
	    remap_addr,
	    (mach_vm_size_t)surface_size);
	T_QUIET;
	T_EXPECT_EQ(kr, KERN_SUCCESS, "vm_deallocate() error 0x%x (%s)",
	    kr, mach_error_string(kr));
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - surface_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("deallocating re-mapping of IOSurface shrinks phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "deallocating remapped IOSurface %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    surface_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);

	/* release IOSurface: footprint shrinks */
	get_ledger_info(&footprint_before, &pagetable_before);
	CFRelease(surface);
	get_ledger_info(&footprint_after, &pagetable_after);
	footprint_expected = footprint_before - surface_size;
	footprint_expected += (pagetable_after - pagetable_before);
	T_LOG("releasing IOSurface decreases phys_footprint");
	T_EXPECT_EQ(footprint_after, footprint_expected,
	    "released IOSurface %lld bytes: "
	    "footprint %lld -> %lld expected %lld delta %lld",
	    surface_size, footprint_before, footprint_after,
	    footprint_expected, footprint_after - footprint_expected);
}
#endif /* (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR) */