1 /*
2 * Copyright (c) 2007-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /**
29 * Machine-dependent structures for the physical map module.
30 *
31 * This header file contains the types and prototypes that make up the public
32 * pmap API that's exposed to the rest of the kernel. Any types/prototypes used
33 * strictly by the pmap itself should be placed into one of the osfmk/arm/pmap/
34 * header files.
35 *
36 * To prevent circular dependencies and exposing anything not needed by the
37 * rest of the kernel, this file shouldn't include ANY of the internal
38 * osfmk/arm/pmap/ header files.
39 */
40 #ifndef _ARM_PMAP_H_
41 #define _ARM_PMAP_H_
42
43 #include <mach_assert.h>
44
45 #include <arm/proc_reg.h>
46 #if defined(__arm64__)
47 #include <arm64/proc_reg.h>
48 #endif /* defined(__arm64__) */
49
50 #ifndef ASSEMBLER
51
52 #include <stdatomic.h>
53 #include <stdbool.h>
54 #include <libkern/section_keywords.h>
55 #include <mach/kern_return.h>
56 #include <mach/machine/vm_types.h>
57 #include <arm/pmap_public.h>
58 #include <kern/ast.h>
59 #include <mach/arm/thread_status.h>
60
61 #if defined(__arm64__)
62 #include <arm64/tlb.h>
63 #else /* defined(__arm64__) */
64 #include <arm/tlb.h>
65 #endif /* defined(__arm64__) */
66
67
68 /* Shift for 2048 max virtual ASIDs (2048 pmaps). */
69 #define ASID_SHIFT (11)
70
71 /* Max supported ASIDs (can be virtual). */
72 #define MAX_ASIDS (1 << ASID_SHIFT)
73
74 /* Shift for the maximum ARM ASID value (256) */
75 #ifndef ARM_ASID_SHIFT
76 #define ARM_ASID_SHIFT (8)
77 #endif /* ARM_ASID_SHIFT */
78
79 /* Max ASIDs supported by the hardware. */
80 #define ARM_MAX_ASIDS (1 << ARM_ASID_SHIFT)
81
82 /* Number of bits in a byte. */
83 #define NBBY (8)
84
85 /**
86 * The maximum number of hardware ASIDs used by the pmap for user address spaces.
87 *
88 * One ASID is always dedicated to the kernel (ASID 0). On systems with software-
89 * based spectre/meltdown mitigations, each address space technically uses two
90 * hardware ASIDs (one for EL1 and one for EL0) so the total number of available
91 * ASIDs a user process can use is halved on those systems.
92 */
93 #if __ARM_KERNEL_PROTECT__
94 #define MAX_HW_ASIDS ((ARM_MAX_ASIDS >> 1) - 1)
95 #else /* __ARM_KERNEL_PROTECT__ */
96 #define MAX_HW_ASIDS (ARM_MAX_ASIDS - 1)
97 #endif /* __ARM_KERNEL_PROTECT__ */
98
99 /* Maximum number of Virtual Machine IDs */
100 #ifndef ARM_VMID_SHIFT
101 #define ARM_VMID_SHIFT (8)
102 #endif /* ARM_VMID_SHIFT */
103 #define ARM_MAX_VMIDS (1 << ARM_VMID_SHIFT)
104
105 /* XPRR virtual register map */
106
107 /* Maximum number of CPU windows per-cpu. */
108 #define CPUWINDOWS_MAX 4
109
110 #if defined(__arm64__)
111
112 #if defined(ARM_LARGE_MEMORY)
113 /*
114 * 2 L1 tables (Linear KVA and V=P), plus 2*16 L2 tables map up to (16*64GB) 1TB of DRAM
115 * Upper limit on how many pages can be consumed by bootstrap page tables
116 */
117 #define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 34)
118 #else /* defined(ARM_LARGE_MEMORY) */
119 #define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 8)
120 #endif /* defined(ARM_LARGE_MEMORY) */
121
122 typedef uint64_t tt_entry_t; /* translation table entry type */
123 typedef uint64_t pt_entry_t; /* page table entry type */
124 #elif defined(__arm__)
125 typedef uint32_t tt_entry_t; /* translation table entry type */
126 typedef uint32_t pt_entry_t; /* page table entry type */
127 #else /* defined(__arm64__) */
128 #error unknown arch
129 #endif /* defined(__arm64__) */
130
131 /* Used to represent a NULL page/translation table entry pointer. */
132 #define PT_ENTRY_NULL ((pt_entry_t *) 0)
133 #define TT_ENTRY_NULL ((tt_entry_t *) 0)
134
135 /**
136 * Number of PTE pointers in a single PVE. This must be 2, since the algorithm
137 * has been optimized to that case. Should this change in the future, both
138 * enter_pv() and remove_pv() will need to be modified accordingly. In addition
139 * to this, the documentation and the LLDB macros that walk PV lists will also
140 * need to be adapted.
141 */
142 #define PTE_PER_PVE 2
143 _Static_assert(PTE_PER_PVE == 2, "PTE_PER_PVE is not 2");
144
145 /**
146 * Structure to track the active mappings for a given page. This structure is
147 * used in the pv_head_table when a physical page has more than one mapping to
148 * it. Each entry in this linked list of structures can represent
149 * up to PTE_PER_PVE mappings.
150 */
151 typedef struct pv_entry {
152 /* Linked list to the next mapping of the physical page. */
153 struct pv_entry *pve_next;
154
155 /* Pointer to the page table entry for this mapping. */
156 pt_entry_t *pve_ptep[PTE_PER_PVE];
157 }
158 #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
159 /**
160 * For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers
161 * are 32-bit: since pt_desc is 64-bit aligned and we cast often from pv_entry
162 * to pt_desc.
163 */
164 __attribute__ ((aligned(8))) pv_entry_t;
165 #else /* __arm__ && (__BIGGEST_ALIGNMENT__ > 4) */
166 pv_entry_t;
167 #endif /* __arm__ && (__BIGGEST_ALIGNMENT__ > 4) */
168
169 /**
170 * Structure that tracks free pv_entry nodes for the pv_head_table. Each one
171 * of these nodes represents a single mapping to a physical page, so a new node
172 * is allocated whenever a new mapping is created.
173 */
174 typedef struct {
175 pv_entry_t *list;
176 uint32_t count;
177 } pv_free_list_t;
178
179 /**
180 * Forward declaration of the structure that controls page table geometry and
181 * TTE/PTE format.
182 */
183 struct page_table_attr;
184
185 struct pmap_cpu_data {
186 #if XNU_MONITOR
187 const volatile struct pmap * _Atomic active_pmap;
188 const volatile struct pmap * _Atomic inflight_pmap;
189 uint64_t pvh_info[4];
190 void *ppl_kern_saved_sp;
191 void *ppl_stack;
192 arm_context_t *save_area;
193 unsigned int ppl_state;
194 #endif /* XNU_MONITOR */
195 pmap_t cpu_nested_pmap;
196 #if __ARM_MIXED_PAGE_SIZE__
197 uint64_t commpage_page_shift;
198 #endif
199 #if defined(__arm64__)
200 const struct page_table_attr *cpu_nested_pmap_attr;
201 vm_map_address_t cpu_nested_region_addr;
202 vm_map_offset_t cpu_nested_region_size;
203 #else /* defined(__arm64__) */
204 pmap_t cpu_user_pmap;
205 unsigned int cpu_user_pmap_stamp;
206 #endif /* defined(__arm64__) */
207 unsigned int cpu_number;
208 bool copywindow_strong_sync[CPUWINDOWS_MAX];
209 bool inflight_disconnect;
210 pv_free_list_t pv_free;
211 pv_entry_t *pv_free_spill_marker;
212
213 /*
214 * This supports overloading of ARM ASIDs by the pmap. The field needs
215 * to be wide enough to cover all the virtual bits in a virtual ASID.
216 * With 256 physical ASIDs, 8-bit fields let us support up to 65536
217 * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global
218 * ASID).
219 *
220 * If we were to use bitfield shenanigans here, we could save a bit of
221 * memory by only having enough bits to support MAX_ASIDS. However, such
222 * an implementation would be more error prone.
223 */
224 uint8_t cpu_sw_asids[MAX_HW_ASIDS];
225 };
226 typedef struct pmap_cpu_data pmap_cpu_data_t;
227
228 #include <mach/vm_prot.h>
229 #include <mach/vm_statistics.h>
230 #include <mach/machine/vm_param.h>
231 #include <kern/kern_types.h>
232 #include <kern/thread.h>
233 #include <kern/queue.h>
234
235
236 #include <sys/cdefs.h>
237
238 /* Base address for low globals. */
239 #if defined(ARM_LARGE_MEMORY)
240 #define LOW_GLOBAL_BASE_ADDRESS 0xfffffe0000000000ULL
241 #else /* defined(ARM_LARGE_MEMORY) */
242 #define LOW_GLOBAL_BASE_ADDRESS 0xfffffff000000000ULL
243 #endif /* defined(ARM_LARGE_MEMORY) */
244
245 /*
246 * This indicates (roughly) where there is free space for the VM
247 * to use for the heap; this does not need to be precise.
248 */
249 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
250 #if defined(ARM_LARGE_MEMORY)
251 #define KERNEL_PMAP_HEAP_RANGE_START (VM_MIN_KERNEL_AND_KEXT_ADDRESS+ARM_TT_L1_SIZE)
252 #else /* defined(ARM_LARGE_MEMORY) */
253 #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
254 #endif /* defined(ARM_LARGE_MEMORY) */
255 #else /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) */
256 #if defined(ARM_LARGE_MEMORY)
257 /* For large memory systems with no KTRR/CTRR such as virtual machines */
258 #define KERNEL_PMAP_HEAP_RANGE_START (VM_MIN_KERNEL_AND_KEXT_ADDRESS+ARM_TT_L1_SIZE)
259 #else
260 #define KERNEL_PMAP_HEAP_RANGE_START LOW_GLOBAL_BASE_ADDRESS
261 #endif
262 #endif /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) */
263
264 /**
265 * For setups where the VM page size does not match the hardware page size (the
266 * VM page size must be a multiple of the hardware page size), we will need to
267 * determine what the page ratio is.
268 */
269 #define PAGE_RATIO ((1 << PAGE_SHIFT) >> ARM_PGSHIFT)
270 #define TEST_PAGE_RATIO_4 (PAGE_RATIO == 4)
271
272
273 /* superpages */
274 #define SUPERPAGE_NBASEPAGES 1 /* No superpages support */
275
276 /* Convert addresses to pages and vice versa. No rounding is used. */
277 #define arm_atop(x) (((vm_map_address_t)(x)) >> ARM_PGSHIFT)
278 #define arm_ptoa(x) (((vm_map_address_t)(x)) << ARM_PGSHIFT)
279
280 /**
281 * Round off or truncate to the nearest page. These will work for either
282 * addresses or counts (i.e. 1 byte rounds to 1 page bytes).
283 */
284 #define arm_round_page(x) ((((vm_map_address_t)(x)) + ARM_PGMASK) & ~ARM_PGMASK)
285 #define arm_trunc_page(x) (((vm_map_address_t)(x)) & ~ARM_PGMASK)
286
287 #if __arm__
288 /* Convert address offset to page table index */
289 #define ptenum(a) ((((a) & ARM_TT_LEAF_INDEX_MASK) >> ARM_TT_LEAF_SHIFT))
290 #endif /* __arm__ */
291
292 extern void flush_mmu_tlb_region(vm_offset_t va, unsigned length);
293
294 #if defined(__arm64__)
295 extern uint64_t get_mmu_control(void);
296 extern uint64_t get_aux_control(void);
297 extern void set_aux_control(uint64_t);
298 extern void set_mmu_ttb(uint64_t);
299 extern void set_mmu_ttb_alternate(uint64_t);
300 extern uint64_t get_tcr(void);
301 extern void set_tcr(uint64_t);
302 extern uint64_t pmap_get_arm64_prot(pmap_t, vm_offset_t);
303 #else /* defined(__arm64__) */
304 extern uint32_t get_mmu_control(void);
305 extern void set_mmu_control(uint32_t);
306 extern uint32_t get_aux_control(void);
307 extern void set_aux_control(uint32_t);
308 extern void set_mmu_ttb(pmap_paddr_t);
309 extern void set_mmu_ttb_alternate(pmap_paddr_t);
310 extern void set_context_id(uint32_t);
311 #endif /* defined(__arm64__) */
312
313 extern pmap_paddr_t get_mmu_ttb(void);
314 extern pmap_paddr_t mmu_kvtop(vm_offset_t va);
315 extern pmap_paddr_t mmu_kvtop_wpreflight(vm_offset_t va);
316 extern pmap_paddr_t mmu_uvtop(vm_offset_t va);
317
318 #if (__ARM_VMSA__ <= 7)
319 /* Convert address offset to translation table index */
320 #define ttenum(a) ((a) >> ARM_TT_L1_SHIFT)
321
322 /* Convert translation table index to user virtual address */
323 #define tteitova(a) ((a) << ARM_TT_L1_SHIFT)
324
325 #define pa_to_suptte(a) ((a) & ARM_TTE_SUPER_L1_MASK)
326 #define suptte_to_pa(p) ((p) & ARM_TTE_SUPER_L1_MASK)
327
328 #define pa_to_sectte(a) ((a) & ARM_TTE_BLOCK_L1_MASK)
329 #define sectte_to_pa(p) ((p) & ARM_TTE_BLOCK_L1_MASK)
330
331 #define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK)
332 #define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK)
333
334 #define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK)
335 #define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK)
336 #define pte_increment_pa(p) ((p) += ptoa(1))
337
338 #define ARM_NESTING_SIZE_MIN ((PAGE_SIZE/0x1000)*4*ARM_TT_L1_SIZE)
339
340 #else /* __ARM_VMSA__ <= 7 */
341
342 /* Convert address offset to translation table index */
343 #define ttel0num(a) ((a & ARM_TTE_L0_MASK) >> ARM_TT_L0_SHIFT)
344 #define ttel1num(a) ((a & ARM_TTE_L1_MASK) >> ARM_TT_L1_SHIFT)
345 #define ttel2num(a) ((a & ARM_TTE_L2_MASK) >> ARM_TT_L2_SHIFT)
346
347 #define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK)
348 #define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK)
349
350 #define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK)
351 #define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK)
352 #define pte_to_ap(p) (((p) & ARM_PTE_APMASK) >> ARM_PTE_APSHIFT)
353 #define pte_increment_pa(p) ((p) += ptoa(1))
354
355 #define TLBFLUSH_SIZE (ARM_TTE_MAX/((sizeof(unsigned int))*BYTE_SIZE))
356
357 #endif /* __ARM_VMSA__ <= 7 */
358
359 /* Status flags used by the pmap garbage collection code. */
360 #define PMAP_GC_INFLIGHT 1
361 #define PMAP_GC_WAIT 2
362
363
364 #define pmap_cs_log(level, fmt, args...)
365 #define pmap_cs_log_debug(fmt, args...)
366 #define pmap_cs_log_info(fmt, args...)
367 #define pmap_cs_log_error(fmt, args...)
368 #define pmap_cs_log_force(level, fmt, args...)
369
370
371
372
373 /* Convert translation/page table entry to kernel virtual address. */
374 #define ttetokv(a) (phystokv(tte_to_pa(a)))
375 #define ptetokv(a) (phystokv(pte_to_pa(a)))
376
377 struct pmap {
378 /* Pointer to the root translation table. */
379 tt_entry_t *tte;
380
381 /* Physical page of the root translation table. */
382 pmap_paddr_t ttep;
383
384 /*
385 * The min and max fields represent the lowest and highest addressable VAs
386 * as dictated strictly by the paging hierarchy (root level + root table size)
387 * in conjunction with whether the root table is used with TTBR0, TTBR1, or VTTBR.
388 * These fields do not encapsulate any higher-level address-space partitioning
389 * policies.
390 */
391
392 /* Lowest supported VA (inclusive) */
393 vm_map_address_t min;
394
395 /* Highest supported VA (exclusive) */
396 vm_map_address_t max;
397
398 #if ARM_PARAMETERIZED_PMAP
399 /* Details about the page table layout. */
400 const struct page_table_attr * pmap_pt_attr;
401 #endif /* ARM_PARAMETERIZED_PMAP */
402
403 /* Ledger tracking phys mappings */
404 ledger_t ledger;
405
406 decl_lck_rw_data(, rwlock);
407
408 /* Global list of pmaps */
409 queue_chain_t pmaps;
410
411 /* Free list of translation table pages. */
412 tt_entry_t *tt_entry_free;
413
414 /* Information representing the "nested" (shared) region in this pmap. */
415 struct pmap *nested_pmap;
416 vm_map_address_t nested_region_addr;
417 vm_map_offset_t nested_region_size;
418 vm_map_offset_t nested_region_true_start;
419 vm_map_offset_t nested_region_true_end;
420 unsigned int *nested_region_asid_bitmap;
421 unsigned int nested_region_asid_bitmap_size;
422
423 #if (__ARM_VMSA__ <= 7)
424 /* Maximum number of translation table entries being used in the root table. */
425 unsigned int tte_index_max;
426 #endif /* (__ARM_VMSA__ <= 7) */
427
428 void * reserved0;
429 void * reserved1;
430 uint64_t reserved2;
431 uint64_t reserved3;
432
433 /* Creation stamp. Incremented by one for each pmap in the system. */
434 unsigned int stamp;
435
436 /* PMAP reference count */
437 _Atomic int32_t ref_count;
438
439 #if XNU_MONITOR
440 /* number of pmaps in which this pmap is nested */
441 _Atomic int32_t nested_count;
442 #endif
443
444 /* Garbage Collection status */
445 unsigned int gc_status;
446
447 /* Number of pmaps that nested this pmap without bounds set. */
448 uint32_t nested_no_bounds_refcnt;
449
450 /**
451 * Represents the real hardware ASID inserted into each TLB entry within
452 * this address space.
453 */
454 uint16_t hw_asid;
455
456 /**
457 * Represents the virtual "software" ASID. Any real hardware ASID can have
458 * multiple software ASIDs associated with it. This is used to know when to
459 * perform TLB flushes during context switches.
460 */
461 uint8_t sw_asid;
462
463 #if MACH_ASSERT
464 int pmap_pid;
465 char pmap_procname[17];
466 bool pmap_stats_assert;
467 #endif /* MACH_ASSERT */
468
469 bool reserved4;
470
471 bool pmap_vm_map_cs_enforced;
472
473 boolean_t reserved5;
474 unsigned int reserved6;
475 unsigned int reserved7;
476
477 bool reserved8;
478 bool reserved9;
479
480 #if DEVELOPMENT || DEBUG
481 bool footprint_suspended;
482 bool footprint_was_suspended;
483 #endif /* DEVELOPMENT || DEBUG */
484
485 /* Whether the No-Execute functionality is enabled. */
486 bool nx_enabled;
487
488 /* Whether this pmap represents a 64-bit address space. */
489 bool is_64bit;
490
491 /* Nested a pmap when the bounds were not set. */
492 bool nested_has_no_bounds_ref;
493
494 /* The nesting bounds have been set. */
495 bool nested_bounds_set;
496
497 #if HAS_APPLE_PAC
498 bool disable_jop;
499 #else
500 bool reserved10;
501 #endif /* HAS_APPLE_PAC */
502
503 #define PMAP_TYPE_USER 0 /* ordinary pmap */
504 #define PMAP_TYPE_KERNEL 1 /* kernel pmap */
505 #define PMAP_TYPE_COMMPAGE 2 /* commpage pmap */
506 #define PMAP_TYPE_NESTED 3 /* pmap nested within another pmap */
507 uint8_t type;
508 };
509
510 #define PMAP_VASID(pmap) (((uint32_t)((pmap)->sw_asid) << 16) | pmap->hw_asid)
511
512 #if VM_DEBUG
513 extern int pmap_list_resident_pages(
514 pmap_t pmap,
515 vm_offset_t *listp,
516 int space);
517 #else /* VM_DEBUG */
518 #define pmap_list_resident_pages(pmap, listp, space) (0)
519 #endif /* VM_DEBUG */
520
521 extern int copysafe(vm_map_address_t from, vm_map_address_t to, uint32_t cnt, int type, uint32_t *bytes_copied);
522
523 /* Globals shared between arm_vm_init and pmap */
524 extern tt_entry_t *cpu_tte; /* First CPUs translation table (shared with kernel pmap) */
525 extern pmap_paddr_t cpu_ttep; /* Physical translation table addr */
526
527 #if __arm64__
528 extern void *ropagetable_begin;
529 extern void *ropagetable_end;
530 #endif /* __arm64__ */
531
532 #if __arm64__
533 extern tt_entry_t *invalid_tte; /* Global invalid translation table */
534 extern pmap_paddr_t invalid_ttep; /* Physical invalid translation table addr */
535 #endif /* __arm64__ */
536
537 #define PMAP_CONTEXT(pmap, thread)
538
539 /**
540 * Platform dependent Prototypes
541 */
542 extern void pmap_clear_user_ttb(void);
543 extern void pmap_bootstrap(vm_offset_t);
544 extern vm_map_address_t pmap_ptov(pmap_t, ppnum_t);
545 extern pmap_paddr_t pmap_find_pa(pmap_t map, addr64_t va);
546 extern pmap_paddr_t pmap_find_pa_nofault(pmap_t map, addr64_t va);
547 extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va);
548 extern ppnum_t pmap_find_phys_nofault(pmap_t map, addr64_t va);
549 extern void pmap_switch_user(thread_t th, vm_map_t map);
550 extern void pmap_set_pmap(pmap_t pmap, thread_t thread);
551 extern void pmap_collect(pmap_t pmap);
552 extern void pmap_gc(void);
553 #if HAS_APPLE_PAC
554 extern void * pmap_sign_user_ptr(void *value, ptrauth_key key, uint64_t data, uint64_t jop_key);
555 extern void * pmap_auth_user_ptr(void *value, ptrauth_key key, uint64_t data, uint64_t jop_key);
556 #endif /* HAS_APPLE_PAC */
557
558 /**
559 * Interfaces implemented as macros.
560 */
561
562 #define PMAP_SWITCH_USER(th, new_map, my_cpu) pmap_switch_user((th), (new_map))
563
564 #define pmap_kernel() (kernel_pmap)
565
566 #define pmap_kernel_va(VA) \
567 (((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS))
568
569 #define pmap_attribute(pmap, addr, size, attr, value) (KERN_INVALID_ADDRESS)
570
571 #define copyinmsg(from, to, cnt) copyin(from, to, cnt)
572 #define copyoutmsg(from, to, cnt) copyout(from, to, cnt)
573
574 /* Unimplemented interfaces. */
575 #define MACRO_NOOP
576 #define pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) MACRO_NOOP
577 #define pmap_pageable(pmap, start, end, pageable) MACRO_NOOP
578
579 extern pmap_paddr_t kvtophys(vm_offset_t va);
580 extern pmap_paddr_t kvtophys_nofail(vm_offset_t va);
581 extern vm_map_address_t phystokv(pmap_paddr_t pa);
582 extern vm_map_address_t phystokv_range(pmap_paddr_t pa, vm_size_t *max_len);
583
584 extern vm_map_address_t pmap_map(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, unsigned int flags);
585 extern vm_map_address_t pmap_map_high_window_bd( vm_offset_t pa, vm_size_t len, vm_prot_t prot);
586 extern kern_return_t pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
587 extern kern_return_t pmap_map_block_addr(pmap_t pmap, addr64_t va, pmap_paddr_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
588 extern void pmap_map_globals(void);
589
590 #define PMAP_MAP_BD_DEVICE 0x0
591 #define PMAP_MAP_BD_WCOMB 0x1
592 #define PMAP_MAP_BD_POSTED 0x2
593 #define PMAP_MAP_BD_POSTED_REORDERED 0x3
594 #define PMAP_MAP_BD_POSTED_COMBINED_REORDERED 0x4
595 #define PMAP_MAP_BD_MASK 0x7
596
597 extern vm_map_address_t pmap_map_bd_with_options(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, int32_t options);
598 extern vm_map_address_t pmap_map_bd(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot);
599
600 extern void pmap_init_pte_page(pmap_t, pt_entry_t *, vm_offset_t, unsigned int ttlevel, boolean_t alloc_ptd);
601
602 extern boolean_t pmap_valid_address(pmap_paddr_t addr);
603 extern void pmap_disable_NX(pmap_t pmap);
604 extern void pmap_set_nested(pmap_t pmap);
605 extern void pmap_create_sharedpages(vm_map_address_t *kernel_data_addr, vm_map_address_t *kernel_text_addr, vm_map_address_t *user_text_addr);
606 extern void pmap_insert_sharedpage(pmap_t pmap);
607 extern void pmap_protect_sharedpage(void);
608
609 extern vm_offset_t pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index);
610 extern unsigned int pmap_map_cpu_windows_copy(ppnum_t pn, vm_prot_t prot, unsigned int wimg_bits);
611 extern void pmap_unmap_cpu_windows_copy(unsigned int index);
612
613 extern void pmap_ro_zone_memcpy(zone_id_t zid, vm_offset_t va, vm_offset_t offset,
614 vm_offset_t new_data, vm_size_t new_data_size);
615 extern uint64_t pmap_ro_zone_atomic_op(zone_id_t zid, vm_offset_t va, vm_offset_t offset,
616 uint32_t op, uint64_t value);
617 extern void pmap_ro_zone_bzero(zone_id_t zid, vm_offset_t va, vm_offset_t offset, vm_size_t size);
618
619 #if XNU_MONITOR
620 /* exposed for use by the HMAC SHA driver */
621 extern void pmap_invoke_with_page(ppnum_t page_number, void *ctx,
622 void (*callback)(void *ctx, ppnum_t page_number, const void *page));
623 extern void pmap_hibernate_invoke(void *ctx, void (*callback)(void *ctx, uint64_t addr, uint64_t len));
624 extern void pmap_set_ppl_hashed_flag(const pmap_paddr_t addr);
625 extern void pmap_clear_ppl_hashed_flag_all(void);
626 extern void pmap_check_ppl_hashed_flag_all(void);
627 #endif /* XNU_MONITOR */
628
629 extern boolean_t pmap_valid_page(ppnum_t pn);
630 extern boolean_t pmap_bootloader_page(ppnum_t pn);
631
632 extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end);
633
634 #define ARM_PMAP_MAX_OFFSET_DEFAULT 0x01
635 #define ARM_PMAP_MAX_OFFSET_MIN 0x02
636 #define ARM_PMAP_MAX_OFFSET_MAX 0x04
637 #define ARM_PMAP_MAX_OFFSET_DEVICE 0x08
638 #define ARM_PMAP_MAX_OFFSET_JUMBO 0x10
639
640 extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option);
641 extern vm_map_offset_t pmap_max_64bit_offset(unsigned int option);
642 extern vm_map_offset_t pmap_max_32bit_offset(unsigned int option);
643
644 boolean_t pmap_virtual_region(unsigned int region_select, vm_map_offset_t *startp, vm_map_size_t *size);
645
646 boolean_t pmap_enforces_execute_only(pmap_t pmap);
647
648
649
650 /* pmap dispatch indices */
651 #define ARM_FAST_FAULT_INDEX 0
652 #define ARM_FORCE_FAST_FAULT_INDEX 1
653 #define MAPPING_FREE_PRIME_INDEX 2
654 #define MAPPING_REPLENISH_INDEX 3
655 #define PHYS_ATTRIBUTE_CLEAR_INDEX 4
656 #define PHYS_ATTRIBUTE_SET_INDEX 5
657 #define PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX 6
658 #define PMAP_CHANGE_WIRING_INDEX 7
659 #define PMAP_CREATE_INDEX 8
660 #define PMAP_DESTROY_INDEX 9
661 #define PMAP_ENTER_OPTIONS_INDEX 10
662 /* #define PMAP_EXTRACT_INDEX 11 -- Not used*/
663 #define PMAP_FIND_PA_INDEX 12
664 #define PMAP_INSERT_SHAREDPAGE_INDEX 13
665 #define PMAP_IS_EMPTY_INDEX 14
666 #define PMAP_MAP_CPU_WINDOWS_COPY_INDEX 15
667 #define PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX 16
668 #define PMAP_NEST_INDEX 17
669 #define PMAP_PAGE_PROTECT_OPTIONS_INDEX 18
670 #define PMAP_PROTECT_OPTIONS_INDEX 19
671 #define PMAP_QUERY_PAGE_INFO_INDEX 20
672 #define PMAP_QUERY_RESIDENT_INDEX 21
673 #define PMAP_REFERENCE_INDEX 22
674 #define PMAP_REMOVE_OPTIONS_INDEX 23
675 #define PMAP_SET_CACHE_ATTRIBUTES_INDEX 25
676 #define PMAP_SET_NESTED_INDEX 26
677 #define PMAP_SET_PROCESS_INDEX 27
678 #define PMAP_SWITCH_INDEX 28
679 #define PMAP_SWITCH_USER_TTB_INDEX 29
680 #define PMAP_CLEAR_USER_TTB_INDEX 30
681 #define PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX 31
682 #define PMAP_UNNEST_OPTIONS_INDEX 32
683 #define PMAP_FOOTPRINT_SUSPEND_INDEX 33
684 #define PMAP_CPU_DATA_INIT_INDEX 34
685 #define PMAP_RELEASE_PAGES_TO_KERNEL_INDEX 35
686 #define PMAP_SET_JIT_ENTITLED_INDEX 36
687
688
689 #define PMAP_UPDATE_COMPRESSOR_PAGE_INDEX 55
690 #define PMAP_TRIM_INDEX 56
691 #define PMAP_LEDGER_VERIFY_SIZE_INDEX 57
692 #define PMAP_LEDGER_ALLOC_INDEX 58
693 #define PMAP_LEDGER_FREE_INDEX 59
694
695 #if HAS_APPLE_PAC
696 #define PMAP_SIGN_USER_PTR 60
697 #define PMAP_AUTH_USER_PTR 61
698 #endif /* HAS_APPLE_PAC */
699
700 #define PHYS_ATTRIBUTE_CLEAR_RANGE_INDEX 66
701
702
703 #if __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX)
704 #define PMAP_DISABLE_USER_JOP_INDEX 69
705 #endif /* __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX) */
706
707
708
709 #define PMAP_SET_VM_MAP_CS_ENFORCED_INDEX 72
710
711 #define PMAP_SET_COMPILATION_SERVICE_CDHASH_INDEX 73
712 #define PMAP_MATCH_COMPILATION_SERVICE_CDHASH_INDEX 74
713 #define PMAP_NOP_INDEX 75
714
715 #define PMAP_RO_ZONE_MEMCPY_INDEX 76
716 #define PMAP_RO_ZONE_ATOMIC_OP_INDEX 77
717
718 #if DEVELOPMENT || DEBUG
719 #define PMAP_TEST_TEXT_CORRUPTION_INDEX 79
720 #endif /* DEVELOPMENT || DEBUG */
721
722
723
724 #define PMAP_SET_LOCAL_SIGNING_PUBLIC_KEY_INDEX 84
725 #define PMAP_UNRESTRICT_LOCAL_SIGNING_INDEX 85
726
727
728
729 #define PMAP_RO_ZONE_BZERO_INDEX 90
730
731
732 #define PMAP_COUNT 96
733
734 /**
735 * Value used when initializing pmap per-cpu data to denote that the structure
736 * hasn't been initialized with its associated CPU number yet.
737 */
738 #define PMAP_INVALID_CPU_NUM (~0U)
739
740 /**
741 * Align the pmap per-cpu data to the L2 cache size for each individual CPU's
742 * data. This prevents accesses from one CPU affecting another, especially
743 * when atomically updating fields.
744 */
745 struct pmap_cpu_data_array_entry {
746 pmap_cpu_data_t cpu_data;
747 } __attribute__((aligned(MAX_L2_CLINE_BYTES)));
748
749 /* Initialize the pmap per-CPU data for the current CPU. */
750 extern void pmap_cpu_data_init(void);
751
752 /* Get the pmap per-CPU data for the current CPU. */
753 extern pmap_cpu_data_t *pmap_get_cpu_data(void);
754
755 /* Get the pmap per-CPU data for an arbitrary CPU. */
756 extern pmap_cpu_data_t *pmap_get_remote_cpu_data(unsigned int cpu);
757
758 /*
759 * For long-running PV list operations, we pick a reasonable maximum chunk size
760 * beyond which we will exit to preemptible context to avoid excessive preemption
761 * latency and PVH lock timeouts.
762 */
763 #define PMAP_MAX_PV_LIST_CHUNK_SIZE 64
764
765 /*
766 * For most batched page operations, we pick a sane default page count
767 * interval at which to check for pending preemption and exit the PPL if found.
768 */
769 #define PMAP_DEFAULT_PREEMPTION_CHECK_PAGE_INTERVAL 64
770
771 static inline bool
_pmap_pending_preemption_real(void)772 _pmap_pending_preemption_real(void)
773 {
774 return !!(*((volatile ast_t*)ast_pending()) & AST_URGENT);
775 }
776
777 #if SCHED_PREEMPTION_DISABLE_DEBUG && (DEBUG || DEVELOPMENT)
778 bool pmap_pending_preemption(void); // more complicated, so externally defined
779 #else /* SCHED_PREEMPTION_DISABLE_DEBUG && (DEBUG || DEVELOPMENT) */
780 #define pmap_pending_preemption _pmap_pending_preemption_real
781 #endif /* SCHED_PREEMPTION_DISABLE_DEBUG && (DEBUG || DEVELOPMENT) */
782
783 #if XNU_MONITOR
784 extern boolean_t pmap_ppl_locked_down;
785
786 /*
787 * Denotes the bounds of the PPL stacks. These are visible so that other code
788 * can check if addresses are part of the PPL stacks.
789 */
790 extern void *pmap_stacks_start;
791 extern void *pmap_stacks_end;
792
793 /* Asks if a page belongs to the monitor. */
794 extern boolean_t pmap_is_monitor(ppnum_t pn);
795
796 /*
797 * Indicates that we are done with our static bootstrap
798 * allocations, so the monitor may now mark the pages
799 * that it owns.
800 */
801 extern void pmap_static_allocations_done(void);
802
803 /*
804 * Indicates that we are done mutating sensitive state in the system, and that
805 * the PPL may now restict write access to PPL owned mappings.
806 */
807 extern void pmap_lockdown_ppl(void);
808
809
810 #ifdef KASAN
811 #define PPL_STACK_SIZE (PAGE_SIZE << 2)
812 #else /* KASAN */
813 #define PPL_STACK_SIZE PAGE_SIZE
814 #endif /* KASAN */
815
816 /* One stack for each CPU, plus a guard page below each stack and above the last stack */
817 #define PPL_STACK_REGION_SIZE ((MAX_CPUS * (PPL_STACK_SIZE + ARM_PGBYTES)) + ARM_PGBYTES)
818
819 #define PPL_DATA_SEGMENT_SECTION_NAME "__PPLDATA,__data"
820 #define PPL_TEXT_SEGMENT_SECTION_NAME "__PPLTEXT,__text,regular,pure_instructions"
821 #define PPL_DATACONST_SEGMENT_SECTION_NAME "__PPLDATA,__const"
822
823 #define MARK_AS_PMAP_DATA \
824 __PLACE_IN_SECTION(PPL_DATA_SEGMENT_SECTION_NAME)
825 #define MARK_AS_PMAP_TEXT \
826 __attribute__((used, section(PPL_TEXT_SEGMENT_SECTION_NAME), noinline))
827 #define MARK_AS_PMAP_RODATA \
828 __PLACE_IN_SECTION(PPL_DATACONST_SEGMENT_SECTION_NAME)
829
830 #else /* XNU_MONITOR */
831
832 #define MARK_AS_PMAP_TEXT
833 #define MARK_AS_PMAP_DATA
834 #define MARK_AS_PMAP_RODATA
835
836 #endif /* XNU_MONITOR */
837
838
839 extern void pmap_nop(pmap_t);
840
841 extern lck_grp_t pmap_lck_grp;
842
843 #if XNU_MONITOR
844 extern void CleanPoC_DcacheRegion_Force_nopreempt(vm_offset_t va, size_t length);
845 #define pmap_force_dcache_clean(va, sz) CleanPoC_DcacheRegion_Force_nopreempt(va, sz)
846 #define pmap_simple_lock(l) simple_lock_nopreempt(l, &pmap_lck_grp)
847 #define pmap_simple_unlock(l) simple_unlock_nopreempt(l)
848 #define pmap_simple_lock_try(l) simple_lock_try_nopreempt(l, &pmap_lck_grp)
849 #define pmap_simple_lock_assert(l, t) simple_lock_assert(l, t)
850 #define pmap_lock_bit(l, i) hw_lock_bit_nopreempt(l, i, &pmap_lck_grp)
851 #define pmap_unlock_bit(l, i) hw_unlock_bit_nopreempt(l, i)
852 #else /* XNU_MONITOR */
853 #define pmap_force_dcache_clean(va, sz) CleanPoC_DcacheRegion_Force(va, sz)
854 #define pmap_simple_lock(l) simple_lock(l, &pmap_lck_grp)
855 #define pmap_simple_unlock(l) simple_unlock(l)
856 #define pmap_simple_lock_try(l) simple_lock_try(l, &pmap_lck_grp)
857 #define pmap_simple_lock_assert(l, t) simple_lock_assert(l, t)
858 #define pmap_lock_bit(l, i) hw_lock_bit(l, i, &pmap_lck_grp)
859 #define pmap_unlock_bit(l, i) hw_unlock_bit(l, i)
860 #endif /* XNU_MONITOR */
861
862 #if DEVELOPMENT || DEBUG
863 extern kern_return_t pmap_test_text_corruption(pmap_paddr_t);
864 #endif /* DEVELOPMENT || DEBUG */
865
866 #endif /* #ifndef ASSEMBLER */
867
868 #if __ARM_KERNEL_PROTECT__
869 /*
870 * The exception vector mappings start at the middle of the kernel page table
871 * range (so that the EL0 mapping can be located at the base of the range).
872 */
873 #define ARM_KERNEL_PROTECT_EXCEPTION_START ((~((ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK) / 2ULL)) + 1ULL)
874 #endif /* __ARM_KERNEL_PROTECT__ */
875
876 #endif /* #ifndef _ARM_PMAP_H_ */
877