xref: /xnu-11417.140.69/osfmk/arm64/sptm/pmap/pmap_pt_geometry.h (revision 43a90889846e00bfb5cf1d255cdc0a701a1e05a4)

/*
 * Copyright (c) 2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/**
 * PMAP Page Table Geometry.
 *
 * This header file is used to store the types, and inline functions related to
 * retrieving information about and parsing page table hierarchies.
 *
 * To prevent circular dependencies, this file shouldn't include any of the
 * other internal osfmk/arm/pmap/ header files.
 */
#ifndef _ARM_PMAP_PMAP_PT_GEOMETRY_H_
#define _ARM_PMAP_PMAP_PT_GEOMETRY_H_

#include <stdint.h>

#include <kern/debug.h>
#include <kern/locks.h>
#include <mach/vm_types.h>
#include <mach_assert.h>

#include <arm64/proc_reg.h>

/**
 * arm64/sptm/pmap/pmap.h is safe to be included in this file since it shouldn't rely on any
 * of the internal pmap header files (so no circular dependencies).
 */
#include <arm64/sptm/pmap/pmap.h>

/**
 * Structure representing parameters of a single page table level. An array of
 * these structures are used to represent the geometry for an entire page table
 * hierarchy.
 */
struct page_table_level_info {
	const uint64_t size;
	const uint64_t offmask;
	const uint64_t shift;
	const uint64_t index_mask;
	const uint64_t valid_mask;
	const uint64_t type_mask;
	const uint64_t type_block;
};

/**
 * Operations that are dependent on the type of page table. This is useful, for
 * instance, when dealing with stage 1 vs stage 2 pmaps.
 */
struct page_table_ops {
	bool (*alloc_id)(pmap_t pmap);
	void (*free_id)(pmap_t pmap);
	void (*flush_tlb_region_async)(vm_offset_t va, size_t length, pmap_t pmap, bool last_level_only);
	void (*flush_tlb_async)(pmap_t pmap);
	pt_entry_t (*wimg_to_pte)(unsigned int wimg, pmap_paddr_t pa);
};

/**
 * The Page Table Attribute structure is used for both parameterizing the
 * different possible page table geometries, but also for abstracting out the
 * differences between stage 1 and stage 2 page tables. This allows one set of
 * code to seamlessly handle the differences between various address space
 * layouts as well as stage 1 vs stage 2 page tables on the fly. See
 * doc/arm_pmap.md for more details.
 *
 * Instead of accessing the fields in this structure directly, it is recommended
 * to use the page table attribute getter functions defined below.
 */
struct page_table_attr {
	/* Sizes and offsets for each level in the page table hierarchy. */
	const struct page_table_level_info * const pta_level_info;

	/* Operations that are dependent on the type of page table. */
	const struct page_table_ops * const pta_ops;

	/**
	 * The Access Permissions bits have different layouts within a page table
	 * entry depending on whether it's an entry for a stage 1 or stage 2 pmap.
	 *
	 * These fields describe the correct PTE bits to set to get the wanted
	 * permissions for the page tables described by this attribute structure.
	 */
	const uintptr_t ap_ro;
	const uintptr_t ap_rw;
	const uintptr_t ap_rona;
	const uintptr_t ap_rwna;
	const uintptr_t ap_xn;
	const uintptr_t ap_x;

	/* The page table level at which the hierarchy begins. */
	const unsigned int pta_root_level;

	/* The page table level at which the commpage is nested into an address space. */
	const unsigned int pta_commpage_level;

	/* The last level in the page table hierarchy (ARM supports up to four levels). */
	const unsigned int pta_max_level;


	/**
	 * Value to set the Translation Control Register (TCR) to in order to inform
	 * the hardware of this page table geometry.
	 */
	const uint64_t pta_tcr_value;

	/* Page Table/Granule Size. */
	const uint64_t pta_page_size;

	/**
	 * How many bits to shift "1" by to get the page table size. Alternatively,
	 * could also be thought of as how many bits make up the page offset in a
	 * virtual address.
	 */
	const uint64_t pta_page_shift;

	/**
	 * SPTM page table geometry index.
	 */
	const uint8_t geometry_id;
};

typedef struct page_table_attr pt_attr_t;

/* The default page table attributes for a system. */
extern const struct page_table_attr * const native_pt_attr;

/**
 * Macros for getting pmap attributes/operations; not functions for const
 * propagation.
 */
#if ARM_PARAMETERIZED_PMAP

/* The page table attributes are linked to the pmap */
#define pmap_get_pt_attr(pmap) ((pmap)->pmap_pt_attr)
#define pmap_get_pt_ops(pmap) ((pmap)->pmap_pt_attr->pta_ops)

#else /* ARM_PARAMETERIZED_PMAP */

/* The page table attributes are fixed (to allow for const propagation) */
#define pmap_get_pt_attr(pmap) (native_pt_attr)
#define pmap_get_pt_ops(pmap) (&native_pt_ops)

#endif /* ARM_PARAMETERIZED_PMAP */

/* Defines representing a level in a page table hierarchy. */
#define PMAP_TT_L0_LEVEL 0x0
#define PMAP_TT_L1_LEVEL 0x1
#define PMAP_TT_L2_LEVEL 0x2
#define PMAP_TT_L3_LEVEL 0x3

/**
 * Inline functions exported for usage by other pmap modules.
 *
 * In an effort to not cause any performance regressions while breaking up the
 * pmap, I'm keeping all functions originally marked as "static inline", as
 * inline and moving them into header files to be shared across the pmap
 * modules. In reality, many of these functions probably don't need to be inline
 * and can be moved back into a .c file.
 *
 * TODO: rdar://70538514 (PMAP Cleanup: re-evaluate whether inline functions should actually be inline)
 */

/**
 * Keep the following in mind when looking at the available attribute getters:
 *
 * We tend to use standard terms to describe various levels in a page table
 * hierarchy. The "root" level is the top of a hierarchy. The root page table is
 * the one that will programmed into the Translation Table Base Register (TTBR)
 * to inform the hardware of where to begin when performing page table walks.
 * The "twig" level is always one up from the last level, and the "leaf" level
 * is the last page table level in a hierarchy. The leaf page tables always
 * contain block entries, but the higher levels can contain either table or
 * block entries.
 *
 * ARM supports up to four levels of page tables. The levels start at L0 and
 * increase to L3 the deeper into a hierarchy you get, although L0 isn't
 * necessarily always the root level. For example, in a four-level hierarchy,
 * the root would be L0, the twig would be L2, and the leaf would be L3. But for
 * a three-level hierarchy, the root would be L1, the twig would be L2, and the
 * leaf would be L3.
 */
/* Page size getter. */
static inline uint64_t
pt_attr_page_size(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_page_size;
}

/**
 * Return the size of the virtual address space covered by a single TTE at a
 * specified level in the hierarchy.
 */
__unused static inline uint64_t
pt_attr_ln_size(const pt_attr_t * const pt_attr, unsigned int level)
{
	return pt_attr->pta_level_info[level].size;
}

/**
 * Return the page descriptor shift for a specified level in the hierarchy. This
 * shift value can be used to get the index into a page table at this level in
 * the hierarchy from a given virtual address.
 */
__unused static inline uint64_t
pt_attr_ln_shift(const pt_attr_t * const pt_attr, unsigned int level)
{
	return pt_attr->pta_level_info[level].shift;
}

/**
 * Return a mask of the offset for a specified level in the hierarchy.
 *
 * This should be equivalent to the value returned by pt_attr_ln_size() - 1.
 */
static inline uint64_t
pt_attr_ln_offmask(const pt_attr_t * const pt_attr, unsigned int level)
{
	return pt_attr->pta_level_info[level].offmask;
}

/**
 * On ARMv7 systems, the leaf page table size (1KB) is smaller than the page
 * size (4KB). To simplify our code, leaf tables are operated on in bundles of
 * four, so that four leaf page tables can be allocated with a single page.
 * Because of that, each page of leaf tables takes up four root/twig entries.
 *
 * This function returns the offset mask for a given level with that taken into
 * consideration. On ARMv8 systems, the granule size is identical to the page
 * size so this doesn't need to be taken into account.
 *
 */
__unused static inline uint64_t
pt_attr_ln_pt_offmask(const pt_attr_t * const pt_attr, unsigned int level)
{
	return pt_attr_ln_offmask(pt_attr, level);
}

/**
 * Return the mask for getting a page table index out of a virtual address for a
 * specified level in the hierarchy. This can be combined with the value
 * returned by pt_attr_ln_shift() to get the index into a page table.
 */
__unused static inline uint64_t
pt_attr_ln_index_mask(const pt_attr_t * const pt_attr, unsigned int level)
{
	return pt_attr->pta_level_info[level].index_mask;
}

/**
 * Return the second to last page table level.
 */
static inline unsigned int
pt_attr_twig_level(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_max_level - 1;
}

/**
 * Return the first page table level. This is what will be programmed into the
 * Translation Table Base Register (TTBR) to inform the hardware of where to
 * begin page table walks.
 */
static inline unsigned int
pt_attr_root_level(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_root_level;
}

/**
 * Return the level at which to nest the commpage pmap into userspace pmaps.
 * Since the commpage is shared across all userspace address maps, memory is
 * saved by sharing the commpage page tables with every userspace pmap. The
 * level at which to nest the commpage is dependent on the page table geometry.
 *
 * Typically this is L1 for 4KB page tables, and L2 for 16KB page tables. In
 * this way, the commpage's L2/L3 page tables are reused in every 4KB task, and
 * the L3 page table is reused in every 16KB task.
 */
static inline unsigned int
pt_attr_commpage_level(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_commpage_level;
}

/**
 * Return the size of the virtual address space covered by a single PTE at the
 * leaf level.
 */
static __unused inline uint64_t
pt_attr_leaf_size(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level].size;
}

/**
 * Return a mask of the offset for a leaf table.
 *
 * This should be equivalent to the value returned by pt_attr_leaf_size() - 1.
 */
static __unused inline uint64_t
pt_attr_leaf_offmask(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level].offmask;
}

/**
 * Return the page descriptor shift for a leaf table entry. This shift value can
 * be used to get the index into a leaf page table from a given virtual address.
 */
static inline uint64_t
pt_attr_leaf_shift(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level].shift;
}

/**
 * Return the mask for getting a leaf table index out of a virtual address. This
 * can be combined with the value returned by pt_attr_leaf_shift() to get the
 * index into a leaf table.
 */
static __unused inline uint64_t
pt_attr_leaf_index_mask(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level].index_mask;
}

/**
 * Return the size of the virtual address space covered by a single TTE at the
 * twig level.
 */
static inline uint64_t
pt_attr_twig_size(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].size;
}

/**
 * Return a mask of the offset for a twig table.
 *
 * This should be equivalent to the value returned by pt_attr_twig_size() - 1.
 */
static inline uint64_t
pt_attr_twig_offmask(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].offmask;
}

/**
 * Return the page descriptor shift for a twig table entry. This shift value can
 * be used to get the index into a twig page table from a given virtual address.
 */
static inline uint64_t
pt_attr_twig_shift(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].shift;
}

/**
 * Return the mask for getting a twig table index out of a virtual address. This
 * can be combined with the value returned by pt_attr_twig_shift() to get the
 * index into a twig table.
 */
static __unused inline uint64_t
pt_attr_twig_index_mask(const pt_attr_t * const pt_attr)
{
	return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].index_mask;
}

/**
 * Return the amount of memory that a leaf table takes up. This is equivalent
 * to the amount of virtual address space covered by a single twig TTE.
 */
static inline uint64_t
pt_attr_leaf_table_size(const pt_attr_t * const pt_attr)
{
	return pt_attr_twig_size(pt_attr);
}

/**
 * Return the offset mask for the memory used by a leaf page table.
 *
 * This should be equivalent to the value returned by pt_attr_twig_size() - 1.
 */
static inline uint64_t
pt_attr_leaf_table_offmask(const pt_attr_t * const pt_attr)
{
	return pt_attr_twig_offmask(pt_attr);
}

/**
 * Return the Access Permissions bits required to specify User and Kernel
 * Read/Write permissions on a PTE in this type of page table hierarchy (stage 1
 * vs stage 2).
 */
static inline uintptr_t
pt_attr_leaf_rw(const pt_attr_t * const pt_attr)
{
	return pt_attr->ap_rw;
}

/**
 * Return the Access Permissions bits required to specify User and Kernel
 * Read-Only permissions on a PTE in this type of page table hierarchy (stage 1
 * vs stage 2).
 */
static inline uintptr_t
pt_attr_leaf_ro(const pt_attr_t * const pt_attr)
{
	return pt_attr->ap_ro;
}

/**
 * Return the Access Permissions bits required to specify just Kernel Read-Only
 * permissions on a PTE in this type of page table hierarchy (stage 1 vs stage
 * 2).
 */
static inline uintptr_t
pt_attr_leaf_rona(const pt_attr_t * const pt_attr)
{
	return pt_attr->ap_rona;
}

/**
 * Return the Access Permissions bits required to specify just Kernel Read/Write
 * permissions on a PTE in this type of page table hierarchy (stage 1 vs stage
 * 2).
 */
static inline uintptr_t
pt_attr_leaf_rwna(const pt_attr_t * const pt_attr)
{
	return pt_attr->ap_rwna;
}

/**
 * Return the mask of the page table entry bits required to set both the
 * privileged and unprivileged execute never bits.
 */
static inline uintptr_t
pt_attr_leaf_xn(const pt_attr_t * const pt_attr)
{
	return pt_attr->ap_xn;
}

/**
 * Return the mask of the page table entry bits required to set just the
 * privileged execute never bit.
 */
static inline uintptr_t
pt_attr_leaf_x(const pt_attr_t * const pt_attr)
{
	return pt_attr->ap_x;
}


/**
 * Return the last level in the page table hierarchy.
 */
static inline unsigned int
pt_attr_leaf_level(const pt_attr_t * const pt_attr)
{
	return pt_attr_twig_level(pt_attr) + 1;
}


/**
 * Return the index into a specific level of page table for a given virtual
 * address.
 *
 * @param pt_attr Page table attribute structure describing the hierarchy.
 * @param addr The virtual address to get the index from.
 * @param pt_level The page table whose index should be returned.
 */
static inline unsigned int
ttn_index(const pt_attr_t * const pt_attr, vm_map_address_t addr, unsigned int pt_level)
{
	const uint64_t index_unshifted = addr & pt_attr_ln_index_mask(pt_attr, pt_level);
	return (unsigned int)(index_unshifted >> pt_attr_ln_shift(pt_attr, pt_level));
}

/**
 * Return the index into a twig page table for a given virtual address.
 *
 * @param pt_attr Page table attribute structure describing the hierarchy.
 * @param addr The virtual address to get the index from.
 */
static inline unsigned int
tte_index(const pt_attr_t * const pt_attr, vm_map_address_t addr)
{
	return ttn_index(pt_attr, addr, PMAP_TT_L2_LEVEL);
}

/**
 * Return the index into a leaf page table for a given virtual address.
 *
 * @param pt_attr Page table attribute structure describing the hierarchy.
 * @param addr The virtual address to get the index from.
 */
static inline unsigned int
pte_index(const pt_attr_t * const pt_attr, vm_map_address_t addr)
{
	return ttn_index(pt_attr, addr, PMAP_TT_L3_LEVEL);
}

/**
 * Return true if a leaf-level PTE is valid.
 *
 * @note This will NOT work on non-leaf-level entries. Please use tte_is_valid()
 *       instead.
 */
static inline bool
pte_is_valid(pt_entry_t pte)
{
	return (pte & ARM_PTE_TYPE_MASK) == ARM_PTE_TYPE_VALID;
}

/**
 * Return true if a non-leaf-level TTE is valid and typed as a table.
 *
 * @note This will NOT work on leaf-level entries. Please use pte_is_valid()
 *       instead.
 *
 * @note This will return false if the TTE represents a non-leaf-level block
 *       mapping (instead of a table mapping).
 */
static inline bool
tte_is_valid_table(tt_entry_t tte)
{
	return (tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID);
}

/**
 * Return true if a non-leaf-level TTE is valid and typed as a block mapping.
 *
 * @note This will NOT work on leaf-level entries. Please use pte_is_valid()
 *       instead.
 *
 * @note This will return false if the TTE represents a non-leaf-level table
 *       mapping (instead of a block mapping).
 */
static inline bool
tte_is_valid_block(tt_entry_t tte)
{
	return (tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID);
}

/**
 * Return true if a non-leaf-level TTE is typed as a table (regardless of
 * validity).
 *
 * @note This will NOT work on leaf-level entries.
 */
static inline bool
tte_is_table(tt_entry_t tte)
{
	return (tte & (ARM_TTE_TYPE_MASK)) == (ARM_TTE_TYPE_TABLE);
}

/**
 * Return true if a non-leaf-level TTE is typed as a block mapping (regardless
 * of validity).
 *
 * @note This will NOT work on leaf-level entries.
 */
static inline bool
tte_is_block(tt_entry_t tte)
{
	return (tte & (ARM_TTE_TYPE_MASK)) == (ARM_TTE_TYPE_BLOCK);
}

/**
 * Given an address and a map, compute the address of the table entry at the
 * specified page table level. If the address is invalid with respect to the map
 * then TT_ENTRY_NULL is returned.
 *
 * @param pmap The pmap whose page tables to parse.
 * @param target_level The page table level at which to stop parsing the
 *                     hierarchy at.
 * @param addr The virtual address to calculate the table indices off of.
 */
static inline tt_entry_t *
pmap_ttne(pmap_t pmap, unsigned int target_level, vm_map_address_t addr)
{
	tt_entry_t *table_ttep = TT_ENTRY_NULL;
	tt_entry_t *ttep = TT_ENTRY_NULL;
	tt_entry_t tte = ARM_TTE_EMPTY;
	unsigned int cur_level;

	const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);

	if (__improbable((addr < pmap->min) || (addr >= pmap->max))) {
		return TT_ENTRY_NULL;
	}
	/* Start parsing at the root page table. */
	table_ttep = pmap->tte;

	assert(target_level <= pt_attr->pta_max_level);

	for (cur_level = pt_attr->pta_root_level; cur_level <= target_level; cur_level++) {
		ttep = &table_ttep[ttn_index(pt_attr, addr, cur_level)];

		if (cur_level == target_level) {
			break;
		}

		tte = *ttep;

#if MACH_ASSERT
		if (tte_is_valid_block(tte)) {
			panic("%s: Attempt to demote L%u block, tte=0x%llx, pmap=%p, target_level=%u, addr=%p",
			    __func__, cur_level, tte, pmap, target_level, (void*)addr);
		}
#endif
		if (!tte_is_valid_table(tte)) {
			return TT_ENTRY_NULL;
		}

		table_ttep = (tt_entry_t*)phystokv(tte & ARM_TTE_TABLE_MASK);
	}

	return ttep;
}

/**
 * Given an address and a map, compute the address of the level 1 translation
 * table entry. If the address is invalid with respect to the map then
 * TT_ENTRY_NULL is returned.
 *
 * @param pmap The pmap whose page tables to parse.
 * @param addr The virtual address to calculate the table indices off of.
 */
static inline tt_entry_t *
pmap_tt1e(pmap_t pmap, vm_map_address_t addr)
{
	return pmap_ttne(pmap, PMAP_TT_L1_LEVEL, addr);
}

/**
 * Given an address and a map, compute the address of the level 2 translation
 * table entry. If the address is invalid with respect to the map then
 * TT_ENTRY_NULL is returned.
 *
 * @param pmap The pmap whose page tables to parse.
 * @param addr The virtual address to calculate the table indices off of.
 */
static inline tt_entry_t *
pmap_tt2e(pmap_t pmap, vm_map_address_t addr)
{
	return pmap_ttne(pmap, PMAP_TT_L2_LEVEL, addr);
}

/**
 * Given an address and a map, compute the address of the level 3 page table
 * entry. If the address is invalid with respect to the map then PT_ENTRY_NULL
 * is returned.
 *
 * @param pmap The pmap whose page tables to parse.
 * @param addr The virtual address to calculate the table indices off of.
 */
static inline pt_entry_t *
pmap_tt3e(pmap_t pmap, vm_map_address_t addr)
{
	return (pt_entry_t*)pmap_ttne(pmap, PMAP_TT_L3_LEVEL, addr);
}

/**
 * Given an address and a map, compute the address of the twig translation table
 * entry. If the address is invalid with respect to the map then TT_ENTRY_NULL
 * is returned.
 *
 * @param pmap The pmap whose page tables to parse.
 * @param addr The virtual address to calculate the table indices off of.
 */
static inline tt_entry_t *
pmap_tte(pmap_t pmap, vm_map_address_t addr)
{
	return pmap_tt2e(pmap, addr);
}

/**
 * Given an address and a map, compute the address of the leaf page table entry.
 * If the address is invalid with respect to the map then PT_ENTRY_NULL is
 * returned.
 *
 * @param pmap The pmap whose page tables to parse.
 * @param addr The virtual address to calculate the table indices off of.
 */
static inline pt_entry_t *
pmap_pte(pmap_t pmap, vm_map_address_t addr)
{
	return pmap_tt3e(pmap, addr);
}

/**
 * Given a virtual address and a page hierarchy level, align the address such that
 * it targets a TTE index that is page ratio-aligned. Normally used prior to
 * calling SPTM table operations (map/unmap/nest/unnest), since the SPTM enforces
 * this requirement.
 *
 * @param pt_attr Page table attribute structure associated with the address space at hand.
 * @param level Page table level for which to align the address.
 * @param va Virtual address to align.
 *
 * @return Aligned virtual address.
 */
static inline vm_map_address_t
pt_attr_align_va(const pt_attr_t * const pt_attr, unsigned int level, vm_map_address_t va)
{
	const uint64_t page_ratio = PAGE_SIZE / pt_attr_page_size(pt_attr);
	const uint64_t ln_shift = pt_attr_ln_shift(pt_attr, level);

	return va & ~((page_ratio - 1) << ln_shift);
}
#endif /* _ARM_PMAP_PMAP_PT_GEOMETRY_H_ */