1 /*
2 * Copyright (c) 2007-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /**
29 * Machine-dependent structures for the physical map module.
30 *
31 * This header file contains the types and prototypes that make up the public
32 * pmap API that's exposed to the rest of the kernel. Any types/prototypes used
33 * strictly by the pmap itself should be placed into one of the osfmk/arm/pmap/
34 * header files.
35 *
36 * To prevent circular dependencies and exposing anything not needed by the
37 * rest of the kernel, this file shouldn't include ANY of the internal
38 * osfmk/arm/pmap/ header files.
39 */
40 #ifndef _ARM_PMAP_H_
41 #define _ARM_PMAP_H_
42
43 #include <mach_assert.h>
44
45 #include <arm64/proc_reg.h>
46
47 #ifndef ASSEMBLER
48
49 #include <stdatomic.h>
50 #include <stdbool.h>
51 #include <libkern/section_keywords.h>
52 #include <mach/kern_return.h>
53 #include <mach/machine/vm_types.h>
54 #include <arm64/sptm/pmap/pmap_public.h>
55 #include <kern/ast.h>
56 #include <mach/arm/thread_status.h>
57 #include <os/refcnt.h>
58
59 #include <arm64/tlb.h>
60
61
62 /* Shift for 2048 max virtual ASIDs (2048 pmaps). */
63 #define ASID_SHIFT (11)
64
65 /* Max supported ASIDs (can be virtual). */
66 #define MAX_ASIDS (1 << ASID_SHIFT)
67
68 /* Shift for the maximum ARM ASID value (256 or 65536) */
69 #ifndef ARM_ASID_SHIFT
70 #if HAS_16BIT_ASID
71 #define ARM_ASID_SHIFT (16)
72 #else
73 #define ARM_ASID_SHIFT (8)
74 #endif /* HAS_16BIT_ASID */
75 #endif /* ARM_ASID_SHIFT */
76
77 /* Max ASIDs supported by the hardware. */
78 #define ARM_MAX_ASIDS (1 << ARM_ASID_SHIFT)
79
80 /* Number of bits in a byte. */
81 #define NBBY (8)
82
83 /**
84 * The maximum number of hardware ASIDs used by the pmap for user address spaces.
85 *
86 * One ASID is always dedicated to the kernel (ASID 0). On systems with software-
87 * based spectre/meltdown mitigations, each address space technically uses two
88 * hardware ASIDs (one for EL1 and one for EL0) so the total number of available
89 * ASIDs a user process can use is halved on those systems.
90 */
91 #if __ARM_KERNEL_PROTECT__
92 #define MAX_HW_ASIDS (ARM_MAX_ASIDS >> 1)
93 #else
94 #define MAX_HW_ASIDS ARM_MAX_ASIDS
95 #endif /* __ARM_KERNEL_PROTECT__ */
96
97 /**
98 * Maximum number of Virtual Machine IDs.
99 *
100 * All even number physical VMIDs are reserved for SK usage. Thus only 128
101 * logical VMIDs are available. Software will convert the logical VMID to
102 * the proper odd numbered physical VMID when allocating/freeing VMIDs.
103 */
104 #ifndef ARM_VMID_SHIFT
105 #define ARM_VMID_SHIFT (7)
106 #endif /* ARM_VMID_SHIFT */
107 #define ARM_MAX_VMIDS (1 << ARM_VMID_SHIFT)
108
109 /* XPRR virtual register map */
110
111 /* Maximum number of CPU windows per-cpu. */
112 #define CPUWINDOWS_MAX 4
113
114
115 #if defined(ARM_LARGE_MEMORY)
116 /*
117 * 2 L1 tables (Linear KVA and V=P), plus 2*16 L2 tables map up to (16*64GB) 1TB of DRAM
118 * Upper limit on how many pages can be consumed by bootstrap page tables
119 */
120 #define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 34)
121 #else /* defined(ARM_LARGE_MEMORY) */
122 #define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 8)
123 #endif /* defined(ARM_LARGE_MEMORY) */
124
125 typedef uint64_t tt_entry_t; /* translation table entry type */
126 typedef uint64_t pt_entry_t; /* page table entry type */
127
128 /* Used to represent a NULL page/translation table entry pointer. */
129 #define PT_ENTRY_NULL ((pt_entry_t *) 0)
130 #define TT_ENTRY_NULL ((tt_entry_t *) 0)
131
132 /**
133 * Number of PTE pointers in a single PVE. This must be 2, since the algorithm
134 * has been optimized to that case. Should this change in the future, both
135 * enter_pv() and remove_pv() will need to be modified accordingly. In addition
136 * to this, the documentation and the LLDB macros that walk PV lists will also
137 * need to be adapted.
138 */
139 #define PTE_PER_PVE 2
140 _Static_assert(PTE_PER_PVE == 2, "PTE_PER_PVE is not 2");
141
142 /**
143 * Structure to track the active mappings for a given page. This structure is
144 * used in the pv_head_table when a physical page has more than one mapping to
145 * it. Each entry in this linked list of structures can represent
146 * up to PTE_PER_PVE mappings.
147 */
148 typedef struct pv_entry {
149 /* Linked list to the next mapping of the physical page. */
150 struct pv_entry *pve_next;
151
152 /* Pointer to the page table entry for this mapping. */
153 pt_entry_t *pve_ptep[PTE_PER_PVE];
154 } pv_entry_t;
155
156 /**
157 * Structure that tracks free pv_entry nodes for the pv_head_table. Each one
158 * of these nodes represents a single mapping to a physical page, so a new node
159 * is allocated whenever a new mapping is created.
160 */
161 typedef struct {
162 pv_entry_t *list;
163 uint32_t count;
164 } pv_free_list_t;
165
166 /**
167 * Forward declaration of the structure that controls page table geometry and
168 * TTE/PTE format.
169 */
170 struct page_table_attr;
171
172 struct pmap_cpu_data {
173 unsigned int cpu_number;
174 bool copywindow_strong_sync[CPUWINDOWS_MAX];
175 bool inflight_disconnect;
176 pv_free_list_t pv_free;
177 pv_entry_t *pv_free_spill_marker;
178 };
179 typedef struct pmap_cpu_data pmap_cpu_data_t;
180
181 #include <mach/vm_prot.h>
182 #include <mach/vm_statistics.h>
183 #include <mach/machine/vm_param.h>
184 #include <kern/kern_types.h>
185 #include <kern/thread.h>
186 #include <kern/queue.h>
187
188
189 #include <sys/cdefs.h>
190
191 /* Base address for low globals. */
192 #if defined(ARM_LARGE_MEMORY)
193 #define LOW_GLOBAL_BASE_ADDRESS 0xfffffe0000000000ULL
194 #else /* defined(ARM_LARGE_MEMORY) */
195 #define LOW_GLOBAL_BASE_ADDRESS 0xfffffff000000000ULL
196 #endif /* defined(ARM_LARGE_MEMORY) */
197
198 /*
199 * This indicates (roughly) where there is free space for the VM
200 * to use for the heap; this does not need to be precise.
201 */
202 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
203 #if defined(ARM_LARGE_MEMORY)
204 #define KERNEL_PMAP_HEAP_RANGE_START (VM_MIN_KERNEL_AND_KEXT_ADDRESS+ARM_TT_L1_SIZE)
205 #else /* defined(ARM_LARGE_MEMORY) */
206 #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
207 #endif /* defined(ARM_LARGE_MEMORY) */
208 #else /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) */
209 #if defined(ARM_LARGE_MEMORY)
210 /* For large memory systems with no KTRR/CTRR such as virtual machines */
211 #define KERNEL_PMAP_HEAP_RANGE_START (VM_MIN_KERNEL_AND_KEXT_ADDRESS+ARM_TT_L1_SIZE)
212 #else
213 #define KERNEL_PMAP_HEAP_RANGE_START LOW_GLOBAL_BASE_ADDRESS
214 #endif
215 #endif /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) */
216
217 /**
218 * For setups where the VM page size does not match the hardware page size (the
219 * VM page size must be a multiple of the hardware page size), we will need to
220 * determine what the page ratio is.
221 */
222 #define PAGE_RATIO ((1 << PAGE_SHIFT) >> ARM_PGSHIFT)
223 #define TEST_PAGE_RATIO_4 (PAGE_RATIO == 4)
224
225
226
227 /* superpages */
228 #define SUPERPAGE_NBASEPAGES 1 /* No superpages support */
229
230 /* Convert addresses to pages and vice versa. No rounding is used. */
231 #define arm_atop(x) (((vm_map_address_t)(x)) >> ARM_PGSHIFT)
232 #define arm_ptoa(x) (((vm_map_address_t)(x)) << ARM_PGSHIFT)
233
234 /**
235 * Round off or truncate to the nearest page. These will work for either
236 * addresses or counts (i.e. 1 byte rounds to 1 page bytes).
237 */
238 #define arm_round_page(x) ((((vm_map_address_t)(x)) + ARM_PGMASK) & ~ARM_PGMASK)
239 #define arm_trunc_page(x) (((vm_map_address_t)(x)) & ~ARM_PGMASK)
240
241 extern void flush_mmu_tlb_region(vm_offset_t va, unsigned length);
242
243 extern uint64_t get_mmu_control(void);
244 extern uint64_t get_aux_control(void);
245 extern void set_aux_control(uint64_t);
246 extern void set_mmu_ttb(uint64_t);
247 extern void set_mmu_ttb_alternate(uint64_t);
248 extern uint64_t get_tcr(void);
249 extern void set_tcr(uint64_t);
250 extern uint64_t pmap_get_arm64_prot(pmap_t, vm_offset_t);
251
252
253 extern pmap_paddr_t get_mmu_ttb(void);
254 extern pmap_paddr_t mmu_kvtop(vm_offset_t va);
255 extern pmap_paddr_t mmu_kvtop_wpreflight(vm_offset_t va);
256 extern pmap_paddr_t mmu_uvtop(vm_offset_t va);
257
258
259 /* Convert address offset to translation table index */
260 #define ttel0num(a) ((a & ARM_TTE_L0_MASK) >> ARM_TT_L0_SHIFT)
261 #define ttel1num(a) ((a & ARM_TTE_L1_MASK) >> ARM_TT_L1_SHIFT)
262 #define ttel2num(a) ((a & ARM_TTE_L2_MASK) >> ARM_TT_L2_SHIFT)
263
264 #define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK)
265 #define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK)
266
267 #define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK)
268 #define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK)
269 #define pte_to_ap(p) (((p) & ARM_PTE_APMASK) >> ARM_PTE_APSHIFT)
270 #define pte_increment_pa(p) ((p) += ptoa(1))
271
272 #define TLBFLUSH_SIZE (ARM_TTE_MAX/((sizeof(unsigned int))*BYTE_SIZE))
273
274
275 #define pmap_cs_log(level, fmt, args...)
276 #define pmap_cs_log_debug(fmt, args...)
277 #define pmap_cs_log_info(fmt, args...)
278 #define pmap_cs_log_error(fmt, args...)
279 #define pmap_cs_log_force(level, fmt, args...)
280
281
282 /**
283 * Wrapper struct that represents a locked entry in the PV head table.
284 * This struct should only be obtained as the return value from pvh_try_lock()
285 * or the pvh_lock* functions.
286 */
287 typedef struct {
288 /* The pv_head_table entry obtained from the lock operation. */
289 uintptr_t pvh;
290 /**
291 * Token obtained from thread_priority_floor_start(), the lock was
292 * placed in sleep mode using pvh_lock_enter_sleep_mode().
293 */
294 thread_pri_floor_t pri_token;
295 /* The index of the locked physical page. */
296 unsigned int pai;
297 } locked_pvh_t;
298
299
300
301 /* Convert translation/page table entry to kernel virtual address. */
302 #define ttetokv(a) (phystokv(tte_to_pa(a)))
303 #define ptetokv(a) (phystokv(pte_to_pa(a)))
304
305 struct pmap {
306 /* Pointer to the root translation table. */
307 tt_entry_t *tte;
308
309 /* Physical page of the root translation table. */
310 pmap_paddr_t ttep;
311
312 /*
313 * The min and max fields represent the lowest and highest addressable VAs
314 * as dictated strictly by the paging hierarchy (root level + root table size)
315 * in conjunction with whether the root table is used with TTBR0, TTBR1, or VTTBR.
316 * These fields do not encapsulate any higher-level address-space partitioning
317 * policies.
318 */
319
320 /* Lowest supported VA (inclusive) */
321 vm_map_address_t min;
322
323 /* Highest supported VA (exclusive) */
324 vm_map_address_t max;
325
326 #if ARM_PARAMETERIZED_PMAP
327 /* Details about the page table layout. */
328 const struct page_table_attr * pmap_pt_attr;
329 #endif /* ARM_PARAMETERIZED_PMAP */
330
331 /* Ledger tracking phys mappings */
332 ledger_t ledger;
333
334 decl_lck_rw_data(, rwlock);
335
336 /* Global list of pmaps */
337 queue_chain_t pmaps;
338
339 /* Information representing the "nested" (shared) region in this pmap. */
340 struct pmap *nested_pmap;
341 vm_map_address_t nested_region_addr;
342 vm_map_offset_t nested_region_size;
343 vm_map_offset_t nested_region_true_start;
344 vm_map_offset_t nested_region_true_end;
345 bitmap_t *nested_region_unnested_table_bitmap;
346
347 /* PMAP reference count */
348 os_ref_atomic_t ref_count;
349
350 /* Number of pmaps that nested this pmap without bounds set. */
351 uint32_t nested_no_bounds_refcnt;
352
353 union {
354 /**
355 * Represents the address space identifier (ASID) for this pmap.
356 * The value 0 is reserved for the kernel pmap; this field will
357 * also be 0 for nested pmaps as those pmaps are never directly
358 * activated on a CPU. This represents a virtual ASID that
359 * is used to globally identify an address space on
360 * the system. Depending upon hardware configuration, this
361 * identifier may have a 1:1 correspondence with the hardware
362 * ASID.
363 */
364 uint16_t asid;
365
366 /**
367 * Represents the virtual machine identifier (VMID) for this pmap.
368 * The value 0 is reserved.
369 */
370 uint16_t vmid;
371 };
372
373 #if MACH_ASSERT
374 int pmap_pid;
375 char pmap_procname[17];
376 #endif /* MACH_ASSERT */
377
378 bool reserved0;
379
380 bool pmap_vm_map_cs_enforced;
381
382 bool reserved1;
383 unsigned int reserved2;
384 unsigned int reserved3;
385
386 #if defined(CONFIG_ROSETTA)
387 /* Whether the pmap is used for Rosetta. */
388 bool is_rosetta;
389 #else
390 bool reserved4;
391 #endif /* defined(CONFIG_ROSETTA) */
392
393 #if DEVELOPMENT || DEBUG
394 bool footprint_suspended;
395 bool footprint_was_suspended;
396 #endif /* DEVELOPMENT || DEBUG */
397
398 /* Whether the No-Execute functionality is enabled. */
399 bool nx_enabled;
400
401 /* Whether this pmap represents a 64-bit address space. */
402 bool is_64bit;
403
404 /* Nested a pmap when the bounds were not set. */
405 bool nested_has_no_bounds_ref;
406
407 /* The nesting bounds have been set. */
408 bool nested_bounds_set;
409
410 #if HAS_APPLE_PAC
411 bool disable_jop;
412 #else
413 bool reserved5;
414 #endif /* HAS_APPLE_PAC */
415
416 bool reserved6;
417
418 #define PMAP_TYPE_USER 0 /* ordinary pmap */
419 #define PMAP_TYPE_KERNEL 1 /* kernel pmap */
420 #define PMAP_TYPE_COMMPAGE 2 /* commpage pmap */
421 #define PMAP_TYPE_NESTED 3 /* pmap nested within another pmap */
422 uint8_t type;
423
424 /*
425 * TrustedExecutionMonitor manages its own address space data structure and
426 * the PMAP is used as the owning structure for keeping this structure.
427 */
428 uint32_t reserved7[4];
429 void *reserved8;
430 uint8_t reserved9;
431
432 /* The ID of the vm_map that this pmap is backing, if any */
433 vm_map_serial_t associated_vm_map_serial_id;
434 };
435
436 #define PMAP_VASID(pmap) ((pmap)->asid)
437 #define PMAP_HWASID(pmap) ((pmap)->asid & (MAX_HW_ASIDS - 1))
438
439 #if VM_DEBUG
440 extern int pmap_list_resident_pages(
441 pmap_t pmap,
442 vm_offset_t *listp,
443 int space);
444 #else /* VM_DEBUG */
445 #define pmap_list_resident_pages(pmap, listp, space) (0)
446 #endif /* VM_DEBUG */
447
448 extern int copysafe(vm_map_address_t from, vm_map_address_t to, uint32_t cnt, int type, uint32_t *bytes_copied);
449
450 /* Globals shared between arm_vm_init and pmap */
451 extern tt_entry_t *cpu_tte; /* First CPUs translation table (shared with kernel pmap) */
452 extern pmap_paddr_t cpu_ttep; /* Physical translation table addr */
453
454 extern void *ropagetable_begin;
455 extern void *ropagetable_end;
456
457
458 extern tt_entry_t *invalid_tte; /* Global invalid translation table */
459 extern pmap_paddr_t invalid_ttep; /* Physical invalid translation table addr */
460
461 #define PMAP_CONTEXT(pmap, thread)
462
463 /**
464 * Platform dependent Prototypes
465 */
466 extern void pmap_clear_user_ttb(void);
467 extern void pmap_bootstrap(vm_offset_t);
468 extern vm_map_address_t pmap_ptov(pmap_t, ppnum_t);
469 extern pmap_paddr_t pmap_find_pa(pmap_t map, addr64_t va);
470 extern pmap_paddr_t pmap_find_pa_nofault(pmap_t map, addr64_t va);
471 extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
472 extern ppnum_t pmap_find_phys_nofault(pmap_t map, addr64_t va);
473 extern void pmap_switch_user(thread_t th, vm_map_t map);
474 extern void pmap_set_pmap(pmap_t pmap, thread_t thread);
475 extern void pmap_gc(void);
476 #if HAS_APPLE_PAC
477 extern void * pmap_sign_user_ptr(void *value, ptrauth_key key, uint64_t data, uint64_t jop_key);
478 extern void * pmap_auth_user_ptr(void *value, ptrauth_key key, uint64_t data, uint64_t jop_key);
479 #endif /* HAS_APPLE_PAC */
480
481 /**
482 * Interfaces implemented as macros.
483 */
484
485 #define PMAP_SWITCH_USER(th, new_map, my_cpu) pmap_switch_user((th), (new_map))
486
487 #define pmap_kernel() (kernel_pmap)
488
489 #define pmap_kernel_va(VA) \
490 (((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS))
491
492 #define pmap_attribute(pmap, addr, size, attr, value) (KERN_INVALID_ADDRESS)
493
494 #define copyinmsg(from, to, cnt) copyin(from, to, cnt)
495 #define copyoutmsg(from, to, cnt) copyout(from, to, cnt)
496
497 /* Unimplemented interfaces. */
498 #define MACRO_NOOP
499 #define pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) MACRO_NOOP
500 #define pmap_pageable(pmap, start, end, pageable) MACRO_NOOP
501
502 extern pmap_paddr_t kvtophys(vm_offset_t va);
503 extern pmap_paddr_t kvtophys_nofail(vm_offset_t va);
504 extern vm_map_address_t phystokv(pmap_paddr_t pa);
505 extern vm_map_address_t phystokv_range(pmap_paddr_t pa, vm_size_t *max_len);
506
507 extern vm_map_address_t pmap_map(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, unsigned int flags);
508 extern vm_map_address_t pmap_map_high_window_bd( vm_offset_t pa, vm_size_t len, vm_prot_t prot);
509 extern kern_return_t pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
510 extern kern_return_t pmap_map_block_addr(pmap_t pmap, addr64_t va, pmap_paddr_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
511 extern void pmap_map_globals(void);
512
513 #define PMAP_MAP_BD_DEVICE 0x0
514 #define PMAP_MAP_BD_WCOMB 0x1
515 #define PMAP_MAP_BD_POSTED 0x2
516 #define PMAP_MAP_BD_POSTED_REORDERED 0x3
517 #define PMAP_MAP_BD_POSTED_COMBINED_REORDERED 0x4
518 #define PMAP_MAP_BD_MASK 0x7
519
520 extern vm_map_address_t pmap_map_bd_with_options(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, int32_t options);
521 extern vm_map_address_t pmap_map_bd(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot);
522
523 extern void pmap_init_pte_page(pmap_t, pt_entry_t *, vm_offset_t, unsigned int ttlevel, boolean_t alloc_ptd);
524
525 extern boolean_t pmap_valid_address(pmap_paddr_t addr);
526 extern void pmap_disable_NX(pmap_t pmap);
527 extern void pmap_set_nested(pmap_t pmap);
528 extern void pmap_create_commpages(vm_map_address_t *kernel_data_addr, vm_map_address_t *kernel_text_addr,
529 vm_map_address_t *kernel_ro_data_addr, vm_map_address_t *user_text_addr);
530 extern void pmap_insert_commpage(pmap_t pmap);
531
532 extern vm_offset_t pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index);
533 extern unsigned int pmap_map_cpu_windows_copy(ppnum_t pn, vm_prot_t prot, unsigned int wimg_bits);
534 extern void pmap_unmap_cpu_windows_copy(unsigned int index);
535
536 extern vm_offset_t pmap_ro_zone_align(vm_offset_t);
537 extern void pmap_ro_zone_memcpy(zone_id_t zid, vm_offset_t va, vm_offset_t offset,
538 vm_offset_t new_data, vm_size_t new_data_size);
539 extern uint64_t pmap_ro_zone_atomic_op(zone_id_t zid, vm_offset_t va, vm_offset_t offset,
540 uint32_t op, uint64_t value);
541 extern void pmap_ro_zone_bzero(zone_id_t zid, vm_offset_t va, vm_offset_t offset, vm_size_t size);
542
543 extern boolean_t pmap_valid_page(ppnum_t pn);
544 extern boolean_t pmap_bootloader_page(ppnum_t pn);
545
546 extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
547
548
549
550 #define ARM_PMAP_MAX_OFFSET_DEFAULT 0x01
551 #define ARM_PMAP_MAX_OFFSET_MIN 0x02
552 #define ARM_PMAP_MAX_OFFSET_MAX 0x04
553 #define ARM_PMAP_MAX_OFFSET_DEVICE 0x08
554 #define ARM_PMAP_MAX_OFFSET_JUMBO 0x10
555 #if XNU_PLATFORM_iPhoneOS && EXTENDED_USER_VA_SUPPORT
556 #define ARM_PMAP_MAX_OFFSET_EXTRA_JUMBO 0x20
557 #endif /* XNU_PLATFORM_iPhoneOS && EXTENDED_USER_VA_SUPPORT */
558
559 extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option);
560 extern vm_map_offset_t pmap_max_64bit_offset(unsigned int option);
561 extern vm_map_offset_t pmap_max_32bit_offset(unsigned int option);
562
563 boolean_t pmap_virtual_region(unsigned int region_select, vm_map_offset_t *startp, vm_map_size_t *size);
564
565 boolean_t pmap_enforces_execute_only(pmap_t pmap);
566
567 void pmap_abandon_measurement(void);
568
569
570
571 /* pmap dispatch indices */
572 #define ARM_FAST_FAULT_INDEX 0
573 #define ARM_FORCE_FAST_FAULT_INDEX 1
574 #define MAPPING_FREE_PRIME_INDEX 2
575 #define MAPPING_REPLENISH_INDEX 3
576 #define PHYS_ATTRIBUTE_CLEAR_INDEX 4
577 #define PHYS_ATTRIBUTE_SET_INDEX 5
578 #define PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX 6
579 #define PMAP_CHANGE_WIRING_INDEX 7
580 #define PMAP_CREATE_INDEX 8
581 #define PMAP_DESTROY_INDEX 9
582 #define PMAP_ENTER_OPTIONS_INDEX 10
583 /* #define PMAP_EXTRACT_INDEX 11 -- Not used*/
584 #define PMAP_FIND_PA_INDEX 12
585 #define PMAP_INSERT_COMMPAGE_INDEX 13
586 #define PMAP_IS_EMPTY_INDEX 14
587 #define PMAP_MAP_CPU_WINDOWS_COPY_INDEX 15
588 #define PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX 16
589 #define PMAP_NEST_INDEX 17
590 #define PMAP_PAGE_PROTECT_OPTIONS_INDEX 18
591 #define PMAP_PROTECT_OPTIONS_INDEX 19
592 #define PMAP_QUERY_PAGE_INFO_INDEX 20
593 #define PMAP_QUERY_RESIDENT_INDEX 21
594 #define PMAP_REFERENCE_INDEX 22
595 #define PMAP_REMOVE_OPTIONS_INDEX 23
596 #define PMAP_SET_CACHE_ATTRIBUTES_INDEX 25
597 #define PMAP_SET_NESTED_INDEX 26
598 #define PMAP_SET_PROCESS_INDEX 27
599 #define PMAP_SWITCH_INDEX 28
600 #define PMAP_SWITCH_USER_TTB_INDEX 29
601 #define PMAP_CLEAR_USER_TTB_INDEX 30
602 #define PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX 31
603 #define PMAP_UNNEST_OPTIONS_INDEX 32
604 #define PMAP_FOOTPRINT_SUSPEND_INDEX 33
605 #define PMAP_CPU_DATA_INIT_INDEX 34
606 #define PMAP_RELEASE_PAGES_TO_KERNEL_INDEX 35
607 #define PMAP_SET_JIT_ENTITLED_INDEX 36
608
609
610 #define PMAP_UPDATE_COMPRESSOR_PAGE_INDEX 55
611 #define PMAP_TRIM_INDEX 56
612 #define PMAP_LEDGER_VERIFY_SIZE_INDEX 57
613 #define PMAP_LEDGER_ALLOC_INDEX 58
614 #define PMAP_LEDGER_FREE_INDEX 59
615
616 #if HAS_APPLE_PAC
617 #define PMAP_SIGN_USER_PTR 60
618 #define PMAP_AUTH_USER_PTR 61
619 #endif /* HAS_APPLE_PAC */
620
621 #define PHYS_ATTRIBUTE_CLEAR_RANGE_INDEX 66
622
623
624 #if __has_feature(ptrauth_calls) && (defined(XNU_TARGET_OS_OSX) || (DEVELOPMENT || DEBUG))
625 #define PMAP_DISABLE_USER_JOP_INDEX 69
626 #endif /* __has_feature(ptrauth_calls) && (defined(XNU_TARGET_OS_OSX) || (DEVELOPMENT || DEBUG)) */
627
628
629
630 #define PMAP_SET_VM_MAP_CS_ENFORCED_INDEX 72
631
632 #define PMAP_SET_COMPILATION_SERVICE_CDHASH_INDEX 73
633 #define PMAP_MATCH_COMPILATION_SERVICE_CDHASH_INDEX 74
634 #define PMAP_NOP_INDEX 75
635
636 #define PMAP_RO_ZONE_MEMCPY_INDEX 76
637 #define PMAP_RO_ZONE_ATOMIC_OP_INDEX 77
638
639 #if DEVELOPMENT || DEBUG
640 #define PMAP_TEST_TEXT_CORRUPTION_INDEX 79
641 #endif /* DEVELOPMENT || DEBUG */
642
643
644
645 #define PMAP_SET_LOCAL_SIGNING_PUBLIC_KEY_INDEX 84
646 #define PMAP_UNRESTRICT_LOCAL_SIGNING_INDEX 85
647
648
649
650 #define PMAP_RO_ZONE_BZERO_INDEX 90
651
652
653
654
655 #define PMAP_SET_TPRO_INDEX 98
656
657 #define PMAP_COUNT 99
658
659 /**
660 * Value used when initializing pmap per-cpu data to denote that the structure
661 * hasn't been initialized with its associated CPU number yet.
662 */
663 #define PMAP_INVALID_CPU_NUM (~0U)
664
665 /**
666 * Align the pmap per-cpu data to the L2 cache size for each individual CPU's
667 * data. This prevents accesses from one CPU affecting another, especially
668 * when atomically updating fields.
669 */
670 struct pmap_cpu_data_array_entry {
671 pmap_cpu_data_t cpu_data;
672 } __attribute__((aligned(MAX_L2_CLINE_BYTES)));
673
674 /* Initialize the pmap per-CPU data for the current CPU. */
675 extern void pmap_cpu_data_init(void);
676
677 /* Get the pmap per-CPU data for the current CPU. */
678 extern pmap_cpu_data_t *pmap_get_cpu_data(void);
679
680 /* Get the pmap per-CPU data for an arbitrary CPU. */
681 extern pmap_cpu_data_t *pmap_get_remote_cpu_data(unsigned int cpu);
682
683 /*
684 * For long-running PV list operations, we pick a reasonable maximum chunk size
685 * beyond which we will exit to preemptible context to avoid excessive preemption
686 * latency and PVH lock timeouts.
687 */
688 #define PMAP_MAX_PV_LIST_CHUNK_SIZE 64
689
690 /*
691 * For most batched page operations, we pick a sane default page count
692 * interval at which to check for pending preemption and exit the PPL if found.
693 */
694 #define PMAP_DEFAULT_PREEMPTION_CHECK_PAGE_INTERVAL 64
695
696 static inline bool
_pmap_pending_preemption_real(void)697 _pmap_pending_preemption_real(void)
698 {
699 return !!(*((volatile ast_t*)ast_pending()) & AST_URGENT);
700 }
701
702 #if SCHED_HYGIENE_DEBUG && (DEBUG || DEVELOPMENT)
703 bool pmap_pending_preemption(void); // more complicated, so externally defined
704 #else /* SCHED_HYGIENE_DEBUG && (DEBUG || DEVELOPMENT) */
705 #define pmap_pending_preemption _pmap_pending_preemption_real
706 #endif /* SCHED_HYGIENE_DEBUG && (DEBUG || DEVELOPMENT) */
707
708 #define MARK_AS_PMAP_TEXT
709 #define MARK_AS_PMAP_DATA
710 #define MARK_AS_PMAP_RODATA
711
712 extern void pmap_nop(pmap_t);
713
714 extern lck_grp_t pmap_lck_grp;
715
716 extern void CleanPoC_DcacheRegion_Force_nopreempt_nohid_nobarrier(vm_offset_t va, size_t length);
717
718 #define pmap_force_dcache_clean(va, sz) CleanPoC_DcacheRegion_Force(va, sz)
719 #define pmap_simple_lock(l) simple_lock(l, &pmap_lck_grp)
720 #define pmap_simple_unlock(l) simple_unlock(l)
721 #define pmap_simple_lock_try(l) simple_lock_try(l, &pmap_lck_grp)
722 #define pmap_simple_lock_assert(l, t) simple_lock_assert(l, t)
723
724 #if DEVELOPMENT || DEBUG
725 extern kern_return_t pmap_test_text_corruption(pmap_paddr_t);
726 #endif /* DEVELOPMENT || DEBUG */
727
728 /* Check if a page has any mappings. */
729 extern bool pmap_is_page_free(pmap_paddr_t paddr);
730
731 #endif /* #ifndef ASSEMBLER */
732
733 #if __ARM_KERNEL_PROTECT__
734 /*
735 * The exception vector mappings start at the middle of the kernel page table
736 * range (so that the EL0 mapping can be located at the base of the range).
737 */
738 #define ARM_KERNEL_PROTECT_EXCEPTION_START ((~((ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK) / 2ULL)) + 1ULL)
739 #endif /* __ARM_KERNEL_PROTECT__ */
740
741 #endif /* #ifndef _ARM_PMAP_H_ */
742