1 /*
2 * Copyright (c) 2007-2011 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_kdp.h>
30 #include <debug.h>
31
32 #include <kern/assert.h>
33 #include <kern/misc_protos.h>
34 #include <kern/monotonic.h>
35 #include <mach/vm_types.h>
36 #include <mach/vm_param.h>
37 #include <vm/vm_kern.h>
38 #include <vm/vm_page_internal.h>
39 #include <vm/pmap.h>
40
41 #include <machine/atomic.h>
42 #include <arm64/proc_reg.h>
43 #include <arm64/lowglobals.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/misc_protos.h>
46 #include <pexpert/arm64/boot.h>
47 #include <pexpert/device_tree.h>
48
49 #include <libkern/kernel_mach_header.h>
50 #include <libkern/section_keywords.h>
51
52 #include <san/kasan.h>
53
54 #if __ARM_KERNEL_PROTECT__
55 /*
56 * If we want to support __ARM_KERNEL_PROTECT__, we need a sufficient amount of
57 * mappable space preceeding the kernel (as we unmap the kernel by cutting the
58 * range covered by TTBR1 in half). This must also cover the exception vectors.
59 */
60 static_assert(KERNEL_PMAP_HEAP_RANGE_START > ARM_KERNEL_PROTECT_EXCEPTION_START);
61
62 /* The exception vectors and the kernel cannot share root TTEs. */
63 static_assert((KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_ROOT_OFFMASK) > ARM_KERNEL_PROTECT_EXCEPTION_START);
64
65 /*
66 * We must have enough space in the TTBR1_EL1 range to create the EL0 mapping of
67 * the exception vectors.
68 */
69 static_assert((((~ARM_KERNEL_PROTECT_EXCEPTION_START) + 1) * 2ULL) <= (ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK));
70 #endif /* __ARM_KERNEL_PROTECT__ */
71
72 #define ARM_DYNAMIC_TABLE_XN (ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN)
73
74 #if KASAN
75 extern vm_offset_t shadow_pbase;
76 extern vm_offset_t shadow_ptop;
77 extern vm_offset_t physmap_vbase;
78 extern vm_offset_t physmap_vtop;
79 #endif
80
81 /*
82 * We explicitly place this in const, as it is not const from a language
83 * perspective, but it is only modified before we actually switch away from
84 * the bootstrap page tables.
85 */
86 SECURITY_READ_ONLY_LATE(uint8_t) bootstrap_pagetables[BOOTSTRAP_TABLE_SIZE] __attribute__((aligned(ARM_PGBYTES)));
87
88 /*
89 * Denotes the end of xnu.
90 */
91 extern void *last_kernel_symbol;
92
93 extern void arm64_replace_bootstack(cpu_data_t*);
94 extern void PE_slide_devicetree(vm_offset_t);
95
96 /*
97 * KASLR parameters
98 */
99 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_base;
100 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_top;
101 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_base;
102 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_top;
103 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_stext;
104 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_etext;
105 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slide;
106 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_base;
107 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_top;
108
109 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_stext;
110 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_etext;
111 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sdata;
112 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_edata;
113 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sinfo;
114 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_einfo;
115 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_slinkedit;
116 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_elinkedit;
117
118 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_builtinkmod_text;
119 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_builtinkmod_text_end;
120
121 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernelcache_base;
122 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernelcache_top;
123
124 /* Used by <mach/arm/vm_param.h> */
125 SECURITY_READ_ONLY_LATE(unsigned long) gVirtBase;
126 SECURITY_READ_ONLY_LATE(unsigned long) gPhysBase;
127 SECURITY_READ_ONLY_LATE(unsigned long) gPhysSize;
128 SECURITY_READ_ONLY_LATE(unsigned long) gT0Sz = T0SZ_BOOT;
129 SECURITY_READ_ONLY_LATE(unsigned long) gT1Sz = T1SZ_BOOT;
130
131 /* 23543331 - step 1 of kext / kernel __TEXT and __DATA colocation is to move
132 * all kexts before the kernel. This is only for arm64 devices and looks
133 * something like the following:
134 * -- vmaddr order --
135 * 0xffffff8004004000 __PRELINK_TEXT
136 * 0xffffff8007004000 __TEXT (xnu)
137 * 0xffffff80075ec000 __DATA (xnu)
138 * 0xffffff80076dc000 __KLD (xnu)
139 * 0xffffff80076e0000 __LAST (xnu)
140 * 0xffffff80076e4000 __LINKEDIT (xnu)
141 * 0xffffff80076e4000 __PRELINK_DATA (not used yet)
142 * 0xffffff800782c000 __PRELINK_INFO
143 * 0xffffff80078e4000 -- End of kernelcache
144 */
145
146 /* 24921709 - make XNU ready for KTRR
147 *
148 * Two possible kernel cache layouts, depending on which kcgen is being used.
149 * VAs increasing downwards.
150 * Old KCGEN:
151 *
152 * __PRELINK_TEXT
153 * __TEXT
154 * __DATA_CONST
155 * __TEXT_EXEC
156 * __KLD
157 * __LAST
158 * __DATA
159 * __PRELINK_DATA (expected empty)
160 * __LINKEDIT
161 * __PRELINK_INFO
162 *
163 * New kcgen:
164 *
165 * __PRELINK_TEXT <--- First KTRR (ReadOnly) segment
166 * __PLK_DATA_CONST
167 * __PLK_TEXT_EXEC
168 * __TEXT
169 * __DATA_CONST
170 * __TEXT_EXEC
171 * __KLD
172 * __LAST <--- Last KTRR (ReadOnly) segment
173 * __DATA
174 * __BOOTDATA (if present)
175 * __LINKEDIT
176 * __PRELINK_DATA (expected populated now)
177 * __PLK_LINKEDIT
178 * __PRELINK_INFO
179 *
180 */
181
182 vm_offset_t mem_size; /* Size of actual physical memory present
183 * minus any performance buffer and possibly
184 * limited by mem_limit in bytes */
185 uint64_t mem_actual; /* The "One True" physical memory size
186 * actually, it's the highest physical
187 * address + 1 */
188 uint64_t max_mem; /* Size of physical memory (bytes), adjusted
189 * by maxmem */
190 uint64_t max_mem_actual; /* Actual size of physical memory (bytes),
191 * adjusted by the maxmem boot-arg */
192 uint64_t sane_size; /* Memory size to use for defaults
193 * calculations */
194 /* This no longer appears to be used; kill it? */
195 addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel
196 * virtual address known
197 * to the VM system */
198
199 SECURITY_READ_ONLY_LATE(vm_offset_t) segEXTRADATA;
200 SECURITY_READ_ONLY_LATE(unsigned long) segSizeEXTRADATA;
201
202 /* Trust cache portion of EXTRADATA (if within it) */
203 SECURITY_READ_ONLY_LATE(vm_offset_t) segTRUSTCACHE;
204 SECURITY_READ_ONLY_LATE(unsigned long) segSizeTRUSTCACHE;
205
206 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTTEXT;
207 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWEST;
208 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTRO;
209 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTRO;
210
211 /* Only set when booted from MH_FILESET kernel collections */
212 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTKC;
213 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTKC;
214 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTROKC;
215 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTROKC;
216 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTAuxKC;
217 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTAuxKC;
218 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTROAuxKC;
219 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTROAuxKC;
220 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTRXAuxKC;
221 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTRXAuxKC;
222 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTNLEAuxKC;
223
224 SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTB;
225 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXT;
226
227 #if XNU_MONITOR
228 SECURITY_READ_ONLY_LATE(vm_offset_t) segPPLTEXTB;
229 SECURITY_READ_ONLY_LATE(unsigned long) segSizePPLTEXT;
230
231 SECURITY_READ_ONLY_LATE(vm_offset_t) segPPLTRAMPB;
232 SECURITY_READ_ONLY_LATE(unsigned long) segSizePPLTRAMP;
233
234 SECURITY_READ_ONLY_LATE(vm_offset_t) segPPLDATACONSTB;
235 SECURITY_READ_ONLY_LATE(unsigned long) segSizePPLDATACONST;
236 SECURITY_READ_ONLY_LATE(void *) pmap_stacks_start = NULL;
237 SECURITY_READ_ONLY_LATE(void *) pmap_stacks_end = NULL;
238 #if HAS_GUARDED_IO_FILTER
239 SECURITY_READ_ONLY_LATE(void *) iofilter_stacks_start = NULL;
240 SECURITY_READ_ONLY_LATE(void *) iofilter_stacks_end = NULL;
241 #endif
242 #endif
243
244 SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATACONSTB;
245 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATACONST;
246
247 SECURITY_READ_ONLY_LATE(vm_offset_t) segTEXTEXECB;
248 SECURITY_READ_ONLY_LATE(unsigned long) segSizeTEXTEXEC;
249
250 SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATAB;
251 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATA;
252
253 #if XNU_MONITOR
254 SECURITY_READ_ONLY_LATE(vm_offset_t) segPPLDATAB;
255 SECURITY_READ_ONLY_LATE(unsigned long) segSizePPLDATA;
256 #endif
257
258 SECURITY_READ_ONLY_LATE(vm_offset_t) segBOOTDATAB;
259 SECURITY_READ_ONLY_LATE(unsigned long) segSizeBOOTDATA;
260 extern vm_offset_t intstack_low_guard;
261 extern vm_offset_t intstack_high_guard;
262 extern vm_offset_t excepstack_high_guard;
263
264 SECURITY_READ_ONLY_LATE(vm_offset_t) segLINKB;
265 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLINK;
266
267 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDB;
268 SECURITY_READ_ONLY_LATE(unsigned long) segSizeKLD;
269 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDDATAB;
270 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKLDDATA;
271 SECURITY_READ_ONLY_LATE(vm_offset_t) segLASTB;
272 SECURITY_READ_ONLY_LATE(unsigned long) segSizeLAST;
273 SECURITY_READ_ONLY_LATE(vm_offset_t) segLASTDATACONSTB;
274 SECURITY_READ_ONLY_LATE(unsigned long) segSizeLASTDATACONST;
275
276 SECURITY_READ_ONLY_LATE(vm_offset_t) sectHIBTEXTB;
277 SECURITY_READ_ONLY_LATE(unsigned long) sectSizeHIBTEXT;
278 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIBDATAB;
279 SECURITY_READ_ONLY_LATE(unsigned long) segSizeHIBDATA;
280 SECURITY_READ_ONLY_LATE(vm_offset_t) sectHIBDATACONSTB;
281 SECURITY_READ_ONLY_LATE(unsigned long) sectSizeHIBDATACONST;
282
283 SECURITY_READ_ONLY_LATE(vm_offset_t) segPRELINKTEXTB;
284 SECURITY_READ_ONLY_LATE(unsigned long) segSizePRELINKTEXT;
285
286 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKTEXTEXECB;
287 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKTEXTEXEC;
288
289 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKDATACONSTB;
290 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKDATACONST;
291
292 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKDATAB;
293 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKDATA;
294
295 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLLVMCOVB = 0;
296 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLLVMCOV = 0;
297
298 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLINKEDITB;
299 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLINKEDIT;
300
301 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKINFOB;
302 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKINFO;
303
304 /* Only set when booted from MH_FILESET primary kernel collection */
305 SECURITY_READ_ONLY_LATE(vm_offset_t) segKCTEXTEXECB;
306 SECURITY_READ_ONLY_LATE(unsigned long) segSizeKCTEXTEXEC;
307 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKCDATACONSTB;
308 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKCDATACONST;
309 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKCDATAB;
310 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKCDATA;
311
312 SECURITY_READ_ONLY_LATE(static boolean_t) use_contiguous_hint = TRUE;
313
314 SECURITY_READ_ONLY_LATE(int) PAGE_SHIFT_CONST;
315
316 SECURITY_READ_ONLY_LATE(vm_offset_t) end_kern;
317 SECURITY_READ_ONLY_LATE(vm_offset_t) etext;
318 SECURITY_READ_ONLY_LATE(vm_offset_t) sdata;
319 SECURITY_READ_ONLY_LATE(vm_offset_t) edata;
320
321 SECURITY_READ_ONLY_LATE(static vm_offset_t) auxkc_mh, auxkc_base, auxkc_right_above;
322
323 vm_offset_t alloc_ptpage(boolean_t map_static);
324 SECURITY_READ_ONLY_LATE(vm_offset_t) ropage_next;
325 extern int dtrace_keep_kernel_symbols(void);
326
327 /*
328 * Bootstrap the system enough to run with virtual memory.
329 * Map the kernel's code and data, and allocate the system page table.
330 * Page_size must already be set.
331 *
332 * Parameters:
333 * first_avail: first available physical page -
334 * after kernel page tables
335 * avail_start: PA of first physical page
336 * avail_end: PA of last physical page
337 */
338 SECURITY_READ_ONLY_LATE(vm_offset_t) first_avail;
339 SECURITY_READ_ONLY_LATE(vm_offset_t) static_memory_end;
340 SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_start;
341 SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_end;
342 SECURITY_READ_ONLY_LATE(pmap_paddr_t) real_avail_end;
343 SECURITY_READ_ONLY_LATE(unsigned long) real_phys_size;
344 SECURITY_READ_ONLY_LATE(vm_map_address_t) physmap_base = (vm_map_address_t)0;
345 SECURITY_READ_ONLY_LATE(vm_map_address_t) physmap_end = (vm_map_address_t)0;
346
347 /**
348 * First physical address freely available to xnu.
349 */
350 SECURITY_READ_ONLY_LATE(addr64_t) first_avail_phys = 0;
351
352 /*
353 * Bounds of the kernelcache; used for accounting.
354 */
355 SECURITY_READ_ONLY_LATE(vm_offset_t) arm_vm_kernelcache_phys_start;
356 SECURITY_READ_ONLY_LATE(vm_offset_t) arm_vm_kernelcache_phys_end;
357
358 #if __ARM_KERNEL_PROTECT__
359 extern void ExceptionVectorsBase;
360 extern void ExceptionVectorsEnd;
361 #endif /* __ARM_KERNEL_PROTECT__ */
362
363 typedef struct {
364 pmap_paddr_t pa;
365 vm_map_address_t va;
366 vm_size_t len;
367 } ptov_table_entry;
368
369 #define PTOV_TABLE_SIZE 8
370
371 SECURITY_READ_ONLY_LATE(static ptov_table_entry) ptov_table[PTOV_TABLE_SIZE];
372 SECURITY_READ_ONLY_LATE(static boolean_t) kva_active = FALSE;
373
374 #define ARM64_PAGE_UNGUARDED (0)
375 #define ARM64_PAGE_GUARDED (1)
376
377 /* "physical to kernel virtual" - given a physical address, return the corresponding physical aperture address */
378 vm_map_address_t
phystokv(pmap_paddr_t pa)379 phystokv(pmap_paddr_t pa)
380 {
381
382 for (size_t i = 0; (i < PTOV_TABLE_SIZE) && (ptov_table[i].len != 0); i++) {
383 if ((pa >= ptov_table[i].pa) && (pa < (ptov_table[i].pa + ptov_table[i].len))) {
384 return pa - ptov_table[i].pa + ptov_table[i].va;
385 }
386 }
387 if (__improbable((pa < gPhysBase) || ((pa - gPhysBase) >= real_phys_size))) {
388 panic("%s: illegal PA: 0x%llx; phys base 0x%llx, size 0x%llx", __func__,
389 (unsigned long long)pa, (unsigned long long)gPhysBase, (unsigned long long)real_phys_size);
390 }
391 return pa - gPhysBase + gVirtBase;
392 }
393
394 vm_map_address_t
phystokv_range(pmap_paddr_t pa,vm_size_t * max_len)395 phystokv_range(pmap_paddr_t pa, vm_size_t *max_len)
396 {
397
398 vm_size_t len;
399 for (size_t i = 0; (i < PTOV_TABLE_SIZE) && (ptov_table[i].len != 0); i++) {
400 if ((pa >= ptov_table[i].pa) && (pa < (ptov_table[i].pa + ptov_table[i].len))) {
401 len = ptov_table[i].len - (pa - ptov_table[i].pa);
402 if (*max_len > len) {
403 *max_len = len;
404 }
405 return pa - ptov_table[i].pa + ptov_table[i].va;
406 }
407 }
408 len = PAGE_SIZE - (pa & PAGE_MASK);
409 if (*max_len > len) {
410 *max_len = len;
411 }
412 if (__improbable((pa < gPhysBase) || ((pa - gPhysBase) >= real_phys_size))) {
413 panic("%s: illegal PA: 0x%llx; phys base 0x%llx, size 0x%llx", __func__,
414 (unsigned long long)pa, (unsigned long long)gPhysBase, (unsigned long long)real_phys_size);
415 }
416 return pa - gPhysBase + gVirtBase;
417 }
418
419 vm_offset_t
ml_static_vtop(vm_offset_t va)420 ml_static_vtop(vm_offset_t va)
421 {
422 for (size_t i = 0; (i < PTOV_TABLE_SIZE) && (ptov_table[i].len != 0); i++) {
423 if ((va >= ptov_table[i].va) && (va < (ptov_table[i].va + ptov_table[i].len))) {
424 return va - ptov_table[i].va + ptov_table[i].pa;
425 }
426 }
427 if (__improbable((va < gVirtBase) || (((vm_address_t)(va) - gVirtBase) >= gPhysSize))) {
428 panic("%s: illegal VA: %p; virt base 0x%llx, size 0x%llx", __func__,
429 (void*)va, (unsigned long long)gVirtBase, (unsigned long long)gPhysSize);
430 }
431 return (vm_address_t)(va) - gVirtBase + gPhysBase;
432 }
433
434 /*
435 * This rounds the given address up to the nearest boundary for a PTE contiguous
436 * hint.
437 */
438 static vm_offset_t
round_up_pte_hint_address(vm_offset_t address)439 round_up_pte_hint_address(vm_offset_t address)
440 {
441 vm_offset_t hint_size = ARM_PTE_SIZE << ARM_PTE_HINT_ENTRIES_SHIFT;
442 return (address + (hint_size - 1)) & ~(hint_size - 1);
443 }
444
445 /* allocate a page for a page table: we support static and dynamic mappings.
446 *
447 * returns a virtual address for the allocated page
448 *
449 * for static mappings, we allocate from the region ropagetable_begin to ro_pagetable_end-1,
450 * which is defined in the DATA_CONST segment and will be protected RNX when vm_prot_finalize runs.
451 *
452 * for dynamic mappings, we allocate from avail_start, which should remain RWNX.
453 */
454
455 vm_offset_t
alloc_ptpage(boolean_t map_static)456 alloc_ptpage(boolean_t map_static)
457 {
458 vm_offset_t vaddr;
459
460 #if !(defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR))
461 map_static = FALSE;
462 #endif
463
464 if (!ropage_next) {
465 ropage_next = (vm_offset_t)&ropagetable_begin;
466 }
467
468 if (map_static) {
469 assert(ropage_next < (vm_offset_t)&ropagetable_end);
470
471 vaddr = ropage_next;
472 ropage_next += ARM_PGBYTES;
473
474 return vaddr;
475 } else {
476 vaddr = phystokv(avail_start);
477 avail_start += ARM_PGBYTES;
478
479 return vaddr;
480 }
481 }
482
483 #if DEBUG
484
485 void dump_kva_l2(vm_offset_t tt_base, tt_entry_t *tt, int indent, uint64_t *rosz_out, uint64_t *rwsz_out);
486
487 void
dump_kva_l2(vm_offset_t tt_base,tt_entry_t * tt,int indent,uint64_t * rosz_out,uint64_t * rwsz_out)488 dump_kva_l2(vm_offset_t tt_base, tt_entry_t *tt, int indent, uint64_t *rosz_out, uint64_t *rwsz_out)
489 {
490 unsigned int i;
491 boolean_t cur_ro, prev_ro = 0;
492 int start_entry = -1;
493 tt_entry_t cur, prev = 0;
494 pmap_paddr_t robegin = kvtophys((vm_offset_t)&ropagetable_begin);
495 pmap_paddr_t roend = kvtophys((vm_offset_t)&ropagetable_end);
496 boolean_t tt_static = kvtophys((vm_offset_t)tt) >= robegin &&
497 kvtophys((vm_offset_t)tt) < roend;
498
499 for (i = 0; i < TTE_PGENTRIES; i++) {
500 int tte_type = tt[i] & ARM_TTE_TYPE_MASK;
501 cur = tt[i] & ARM_TTE_TABLE_MASK;
502
503 if (tt_static) {
504 /* addresses mapped by this entry are static if it is a block mapping,
505 * or the table was allocated from the RO page table region */
506 cur_ro = (tte_type == ARM_TTE_TYPE_BLOCK) || (cur >= robegin && cur < roend);
507 } else {
508 cur_ro = 0;
509 }
510
511 if ((cur == 0 && prev != 0) || (cur_ro != prev_ro && prev != 0)) { // falling edge
512 uintptr_t start, end, sz;
513
514 start = (uintptr_t)start_entry << ARM_TT_L2_SHIFT;
515 start += tt_base;
516 end = ((uintptr_t)i << ARM_TT_L2_SHIFT) - 1;
517 end += tt_base;
518
519 sz = end - start + 1;
520 printf("%*s0x%08x_%08x-0x%08x_%08x %s (%luMB)\n",
521 indent * 4, "",
522 (uint32_t)(start >> 32), (uint32_t)start,
523 (uint32_t)(end >> 32), (uint32_t)end,
524 prev_ro ? "Static " : "Dynamic",
525 (sz >> 20));
526
527 if (prev_ro) {
528 *rosz_out += sz;
529 } else {
530 *rwsz_out += sz;
531 }
532 }
533
534 if ((prev == 0 && cur != 0) || cur_ro != prev_ro) { // rising edge: set start
535 start_entry = i;
536 }
537
538 prev = cur;
539 prev_ro = cur_ro;
540 }
541 }
542
543 void
dump_kva_space()544 dump_kva_space()
545 {
546 uint64_t tot_rosz = 0, tot_rwsz = 0;
547 int ro_ptpages, rw_ptpages;
548 pmap_paddr_t robegin = kvtophys((vm_offset_t)&ropagetable_begin);
549 pmap_paddr_t roend = kvtophys((vm_offset_t)&ropagetable_end);
550 boolean_t root_static = kvtophys((vm_offset_t)cpu_tte) >= robegin &&
551 kvtophys((vm_offset_t)cpu_tte) < roend;
552 uint64_t kva_base = ~((1ULL << (64 - T1SZ_BOOT)) - 1);
553
554 printf("Root page table: %s\n", root_static ? "Static" : "Dynamic");
555
556 for (unsigned int i = 0; i < TTE_PGENTRIES; i++) {
557 pmap_paddr_t cur;
558 boolean_t cur_ro;
559 uintptr_t start, end;
560 uint64_t rosz = 0, rwsz = 0;
561
562 if ((cpu_tte[i] & ARM_TTE_VALID) == 0) {
563 continue;
564 }
565
566 cur = cpu_tte[i] & ARM_TTE_TABLE_MASK;
567 start = (uint64_t)i << ARM_TT_L1_SHIFT;
568 start = start + kva_base;
569 end = start + (ARM_TT_L1_SIZE - 1);
570 cur_ro = cur >= robegin && cur < roend;
571
572 printf("0x%08x_%08x-0x%08x_%08x %s\n",
573 (uint32_t)(start >> 32), (uint32_t)start,
574 (uint32_t)(end >> 32), (uint32_t)end,
575 cur_ro ? "Static " : "Dynamic");
576
577 dump_kva_l2(start, (tt_entry_t*)phystokv(cur), 1, &rosz, &rwsz);
578 tot_rosz += rosz;
579 tot_rwsz += rwsz;
580 }
581
582 printf("L2 Address space mapped: Static %lluMB Dynamic %lluMB Total %lluMB\n",
583 tot_rosz >> 20,
584 tot_rwsz >> 20,
585 (tot_rosz >> 20) + (tot_rwsz >> 20));
586
587 ro_ptpages = (int)((ropage_next - (vm_offset_t)&ropagetable_begin) >> ARM_PGSHIFT);
588 rw_ptpages = (int)(lowGlo.lgStaticSize >> ARM_PGSHIFT);
589 printf("Pages used: static %d dynamic %d\n", ro_ptpages, rw_ptpages);
590 }
591
592 #endif /* DEBUG */
593
594 #if __ARM_KERNEL_PROTECT__ || XNU_MONITOR
595 /*
596 * arm_vm_map:
597 * root_ttp: The kernel virtual address for the root of the target page tables
598 * vaddr: The target virtual address
599 * pte: A page table entry value (may be ARM_PTE_EMPTY)
600 *
601 * This function installs pte at vaddr in root_ttp. Any page table pages needed
602 * to install pte will be allocated by this function.
603 */
604 static void
arm_vm_map(tt_entry_t * root_ttp,vm_offset_t vaddr,pt_entry_t pte)605 arm_vm_map(tt_entry_t * root_ttp, vm_offset_t vaddr, pt_entry_t pte)
606 {
607 vm_offset_t ptpage = 0;
608 tt_entry_t * ttp = root_ttp;
609
610 tt_entry_t * l1_ttep = NULL;
611 tt_entry_t l1_tte = 0;
612
613 tt_entry_t * l2_ttep = NULL;
614 tt_entry_t l2_tte = 0;
615 pt_entry_t * ptep = NULL;
616 pt_entry_t cpte = 0;
617
618 /*
619 * Walk the target page table to find the PTE for the given virtual
620 * address. Allocate any page table pages needed to do this.
621 */
622 l1_ttep = ttp + ((vaddr & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
623 l1_tte = *l1_ttep;
624
625 if (l1_tte == ARM_TTE_EMPTY) {
626 ptpage = alloc_ptpage(TRUE);
627 bzero((void *)ptpage, ARM_PGBYTES);
628 l1_tte = kvtophys(ptpage);
629 l1_tte &= ARM_TTE_TABLE_MASK;
630 l1_tte |= ARM_TTE_VALID | ARM_TTE_TYPE_TABLE | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA);
631 *l1_ttep = l1_tte;
632 ptpage = 0;
633 }
634
635 ttp = (tt_entry_t *)phystokv(l1_tte & ARM_TTE_TABLE_MASK);
636
637 l2_ttep = ttp + ((vaddr & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
638 l2_tte = *l2_ttep;
639
640 if (l2_tte == ARM_TTE_EMPTY) {
641 ptpage = alloc_ptpage(TRUE);
642 bzero((void *)ptpage, ARM_PGBYTES);
643 l2_tte = kvtophys(ptpage);
644 l2_tte &= ARM_TTE_TABLE_MASK;
645 l2_tte |= ARM_TTE_VALID | ARM_TTE_TYPE_TABLE;
646 *l2_ttep = l2_tte;
647 ptpage = 0;
648 }
649
650 ttp = (tt_entry_t *)phystokv(l2_tte & ARM_TTE_TABLE_MASK);
651
652 ptep = ttp + ((vaddr & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT);
653 cpte = *ptep;
654
655 /*
656 * If the existing PTE is not empty, then we are replacing a valid
657 * mapping.
658 */
659 if (cpte != ARM_PTE_EMPTY) {
660 panic("%s: cpte=%#llx is not empty, "
661 "vaddr=%#lx, pte=%#llx",
662 __FUNCTION__, cpte,
663 vaddr, pte);
664 }
665
666 *ptep = pte;
667 }
668
669 #endif // __ARM_KERNEL_PROTECT || XNU_MONITOR
670
671 #if __ARM_KERNEL_PROTECT__
672
673 /*
674 * arm_vm_kernel_el0_map:
675 * vaddr: The target virtual address
676 * pte: A page table entry value (may be ARM_PTE_EMPTY)
677 *
678 * This function installs pte at vaddr for the EL0 kernel mappings.
679 */
680 static void
arm_vm_kernel_el0_map(vm_offset_t vaddr,pt_entry_t pte)681 arm_vm_kernel_el0_map(vm_offset_t vaddr, pt_entry_t pte)
682 {
683 /* Calculate where vaddr will be in the EL1 kernel page tables. */
684 vm_offset_t kernel_pmap_vaddr = vaddr - ((ARM_TT_ROOT_INDEX_MASK + ARM_TT_ROOT_SIZE) / 2ULL);
685 arm_vm_map(cpu_tte, kernel_pmap_vaddr, pte);
686 }
687
688 /*
689 * arm_vm_kernel_el1_map:
690 * vaddr: The target virtual address
691 * pte: A page table entry value (may be ARM_PTE_EMPTY)
692 *
693 * This function installs pte at vaddr for the EL1 kernel mappings.
694 */
695 static void
arm_vm_kernel_el1_map(vm_offset_t vaddr,pt_entry_t pte)696 arm_vm_kernel_el1_map(vm_offset_t vaddr, pt_entry_t pte)
697 {
698 arm_vm_map(cpu_tte, vaddr, pte);
699 }
700
701 /*
702 * arm_vm_kernel_pte:
703 * vaddr: The target virtual address
704 *
705 * This function returns the PTE value for the given vaddr from the kernel page
706 * tables. If the region has been been block mapped, we return what an
707 * equivalent PTE value would be (as regards permissions and flags). We also
708 * remove the HINT bit (as we are not necessarily creating contiguous mappings.
709 */
710 static pt_entry_t
arm_vm_kernel_pte(vm_offset_t vaddr)711 arm_vm_kernel_pte(vm_offset_t vaddr)
712 {
713 tt_entry_t * ttp = cpu_tte;
714 tt_entry_t * ttep = NULL;
715 tt_entry_t tte = 0;
716 pt_entry_t * ptep = NULL;
717 pt_entry_t pte = 0;
718
719 ttep = ttp + ((vaddr & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
720 tte = *ttep;
721
722 assert(tte & ARM_TTE_VALID);
723
724 if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
725 /* This is a block mapping; return the equivalent PTE value. */
726 pte = (pt_entry_t)(tte & ~ARM_TTE_TYPE_MASK);
727 pte |= ARM_PTE_TYPE_VALID;
728 pte |= vaddr & ((ARM_TT_L1_SIZE - 1) & ARM_PTE_PAGE_MASK);
729 pte &= ~ARM_PTE_HINT_MASK;
730 return pte;
731 }
732
733 ttp = (tt_entry_t *)phystokv(tte & ARM_TTE_TABLE_MASK);
734 ttep = ttp + ((vaddr & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
735 tte = *ttep;
736
737 assert(tte & ARM_TTE_VALID);
738
739 if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
740 /* This is a block mapping; return the equivalent PTE value. */
741 pte = (pt_entry_t)(tte & ~ARM_TTE_TYPE_MASK);
742 pte |= ARM_PTE_TYPE_VALID;
743 pte |= vaddr & ((ARM_TT_L2_SIZE - 1) & ARM_PTE_PAGE_MASK);
744 pte &= ~ARM_PTE_HINT_MASK;
745 return pte;
746 }
747
748 ttp = (tt_entry_t *)phystokv(tte & ARM_TTE_TABLE_MASK);
749
750 ptep = ttp + ((vaddr & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT);
751 pte = *ptep;
752 pte &= ~ARM_PTE_HINT_MASK;
753 return pte;
754 }
755
756 /*
757 * arm_vm_prepare_kernel_el0_mappings:
758 * alloc_only: Indicates if PTE values should be copied from the EL1 kernel
759 * mappings.
760 *
761 * This function expands the kernel page tables to support the EL0 kernel
762 * mappings, and conditionally installs the PTE values for the EL0 kernel
763 * mappings (if alloc_only is false).
764 */
765 static void
arm_vm_prepare_kernel_el0_mappings(bool alloc_only)766 arm_vm_prepare_kernel_el0_mappings(bool alloc_only)
767 {
768 pt_entry_t pte = 0;
769 vm_offset_t start = ((vm_offset_t)&ExceptionVectorsBase) & ~PAGE_MASK;
770 vm_offset_t end = (((vm_offset_t)&ExceptionVectorsEnd) + PAGE_MASK) & ~PAGE_MASK;
771 vm_offset_t cur = 0;
772 vm_offset_t cur_fixed = 0;
773
774 /* Expand for/map the exceptions vectors in the EL0 kernel mappings. */
775 for (cur = start, cur_fixed = ARM_KERNEL_PROTECT_EXCEPTION_START; cur < end; cur += ARM_PGBYTES, cur_fixed += ARM_PGBYTES) {
776 /*
777 * We map the exception vectors at a different address than that
778 * of the kernelcache to avoid sharing page table pages with the
779 * kernelcache (as this may cause issues with TLB caching of
780 * page table pages.
781 */
782 if (!alloc_only) {
783 pte = arm_vm_kernel_pte(cur);
784 }
785
786 arm_vm_kernel_el1_map(cur_fixed, pte);
787 arm_vm_kernel_el0_map(cur_fixed, pte);
788 }
789
790 __builtin_arm_dmb(DMB_ISH);
791 __builtin_arm_isb(ISB_SY);
792
793 if (!alloc_only) {
794 /*
795 * If we have created the alternate exception vector mappings,
796 * the boot CPU may now switch over to them.
797 */
798 set_vbar_el1(ARM_KERNEL_PROTECT_EXCEPTION_START);
799 __builtin_arm_isb(ISB_SY);
800 }
801 }
802
803 /*
804 * arm_vm_populate_kernel_el0_mappings:
805 *
806 * This function adds all required mappings to the EL0 kernel mappings.
807 */
808 static void
arm_vm_populate_kernel_el0_mappings(void)809 arm_vm_populate_kernel_el0_mappings(void)
810 {
811 arm_vm_prepare_kernel_el0_mappings(FALSE);
812 }
813
814 /*
815 * arm_vm_expand_kernel_el0_mappings:
816 *
817 * This function expands the kernel page tables to accomodate the EL0 kernel
818 * mappings.
819 */
820 static void
arm_vm_expand_kernel_el0_mappings(void)821 arm_vm_expand_kernel_el0_mappings(void)
822 {
823 arm_vm_prepare_kernel_el0_mappings(TRUE);
824 }
825 #endif /* __ARM_KERNEL_PROTECT__ */
826
827 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
828 extern void bootstrap_instructions;
829
830 /*
831 * arm_replace_identity_map takes the V=P map that we construct in start.s
832 * and repurposes it in order to have it map only the page we need in order
833 * to turn on the MMU. This prevents us from running into issues where
834 * KTRR will cause us to fault on executable block mappings that cross the
835 * KTRR boundary.
836 */
837 static void
arm_replace_identity_map(void)838 arm_replace_identity_map(void)
839 {
840 vm_offset_t addr;
841 pmap_paddr_t paddr;
842
843 pmap_paddr_t l1_ptp_phys = 0;
844 tt_entry_t *l1_ptp_virt = NULL;
845 tt_entry_t *tte1 = NULL;
846 pmap_paddr_t l2_ptp_phys = 0;
847 tt_entry_t *l2_ptp_virt = NULL;
848 tt_entry_t *tte2 = NULL;
849 pmap_paddr_t l3_ptp_phys = 0;
850 pt_entry_t *l3_ptp_virt = NULL;
851 pt_entry_t *ptep = NULL;
852
853 addr = ((vm_offset_t)&bootstrap_instructions) & ~ARM_PGMASK;
854 paddr = kvtophys(addr);
855
856 /*
857 * Grab references to the V=P page tables, and allocate an L3 page.
858 */
859 l1_ptp_phys = kvtophys((vm_offset_t)&bootstrap_pagetables);
860 l1_ptp_virt = (tt_entry_t *)phystokv(l1_ptp_phys);
861 tte1 = &l1_ptp_virt[L1_TABLE_INDEX(paddr)];
862
863 l2_ptp_virt = L2_TABLE_VA(tte1);
864 l2_ptp_phys = (*tte1) & ARM_TTE_TABLE_MASK;
865 tte2 = &l2_ptp_virt[L2_TABLE_INDEX(paddr)];
866
867 l3_ptp_virt = (pt_entry_t *)alloc_ptpage(TRUE);
868 l3_ptp_phys = kvtophys((vm_offset_t)l3_ptp_virt);
869 ptep = &l3_ptp_virt[L3_TABLE_INDEX(paddr)];
870
871 /*
872 * Replace the large V=P mapping with a mapping that provides only the
873 * mappings needed to turn on the MMU.
874 */
875
876 bzero(l1_ptp_virt, ARM_PGBYTES);
877 *tte1 = ARM_TTE_BOOT_TABLE | (l2_ptp_phys & ARM_TTE_TABLE_MASK);
878
879 bzero(l2_ptp_virt, ARM_PGBYTES);
880 *tte2 = ARM_TTE_BOOT_TABLE | (l3_ptp_phys & ARM_TTE_TABLE_MASK);
881
882 *ptep = (paddr & ARM_PTE_MASK) |
883 ARM_PTE_TYPE_VALID |
884 ARM_PTE_SH(SH_OUTER_MEMORY) |
885 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) |
886 ARM_PTE_AF |
887 ARM_PTE_AP(AP_RONA) |
888 ARM_PTE_NX;
889 }
890 #endif /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) */
891
892 tt_entry_t *arm_kva_to_tte(vm_offset_t);
893
894 tt_entry_t *
arm_kva_to_tte(vm_offset_t va)895 arm_kva_to_tte(vm_offset_t va)
896 {
897 tt_entry_t *tte1, *tte2;
898 tte1 = cpu_tte + L1_TABLE_INDEX(va);
899 tte2 = L2_TABLE_VA(tte1) + L2_TABLE_INDEX(va);
900
901 return tte2;
902 }
903
904 #if XNU_MONITOR
905
906 static inline pt_entry_t *
arm_kva_to_pte(vm_offset_t va)907 arm_kva_to_pte(vm_offset_t va)
908 {
909 tt_entry_t *tte2 = arm_kva_to_tte(va);
910 return L3_TABLE_VA(tte2) + L3_TABLE_INDEX(va);
911 }
912
913 #endif
914
915 #define ARM64_GRANULE_ALLOW_BLOCK (1 << 0)
916 #define ARM64_GRANULE_ALLOW_HINT (1 << 1)
917
918 /**
919 * Updates a translation table entry (TTE) with the supplied value, unless doing so might render
920 * the pagetable region read-only before subsequent updates have finished. In that case, the TTE
921 * value will be saved off for deferred processing.
922 *
923 * @param ttep address of the TTE to update
924 * @param entry the value to store in ttep
925 * @param pa the base physical address mapped by the TTE
926 * @param ttebase L3-page- or L2-block-aligned base virtual address of the pagetable region
927 * @param granule mask indicating whether L2 block or L3 hint mappings are allowed for this segment
928 * @param deferred_ttep_pair 2-element array of addresses of deferred TTEs
929 * @param deferred_tte_pair 2-element array containing TTE values for deferred assignment to
930 * corresponding elements of deferred_ttep_pair
931 */
932 static void
update_or_defer_tte(tt_entry_t * ttep,tt_entry_t entry,pmap_paddr_t pa,vm_map_address_t ttebase,unsigned granule __unused,tt_entry_t ** deferred_ttep_pair,tt_entry_t * deferred_tte_pair)933 update_or_defer_tte(tt_entry_t *ttep, tt_entry_t entry, pmap_paddr_t pa, vm_map_address_t ttebase,
934 unsigned granule __unused, tt_entry_t **deferred_ttep_pair, tt_entry_t *deferred_tte_pair)
935 {
936 /*
937 * If we're trying to assign an entry that maps the current TTE region (identified by ttebase),
938 * and the pagetable is already live (indicated by kva_active), defer assignment of the current
939 * entry and possibly the entry after it until all other mappings in the segment have been
940 * updated. Otherwise we may end up immediately marking the pagetable region read-only
941 * leading to a fault later on a later assignment if we manage to outrun the TLB. This can
942 * happen on KTRR/CTRR-enabled devices when marking segDATACONST read-only, as the pagetables
943 * that map that segment must come from the segment itself. We therefore store the initial
944 * recursive TTE in deferred_ttep_pair[0] and its value in deferred_tte_pair[0]. We may also
945 * defer assignment of the TTE following that recursive TTE and store its value in
946 * deferred_tte_pair[1], because the TTE region following the current one may also contain
947 * pagetables and we must avoid marking that region read-only before updating those tables.
948 *
949 * We require that such recursive mappings must exist in regions that can be mapped with L2
950 * block entries if they are sufficiently large. This is what allows us to assume that no
951 * more than 2 deferred TTEs will be required, because:
952 * --If more than 2 adjacent L3 PTEs were required to map our pagetables, that would mean
953 * we would have at least one full L3 pagetable page and would instead use an L2 block.
954 * --If more than 2 adjacent L2 blocks were required to map our pagetables, that would
955 * mean we would have at least one full L2-block-sized region of TTEs and something
956 * is very wrong because no segment should be that large.
957 */
958 if ((deferred_ttep_pair != NULL) && (deferred_ttep_pair[0] != NULL) && (ttep == (deferred_ttep_pair[0] + 1))) {
959 assert(deferred_tte_pair[1] == 0);
960 deferred_ttep_pair[1] = ttep;
961 deferred_tte_pair[1] = entry;
962 } else if (kva_active && (phystokv(pa) == ttebase)) {
963 assert(deferred_ttep_pair != NULL);
964 assert(granule & ARM64_GRANULE_ALLOW_BLOCK);
965 if (deferred_ttep_pair[0] == NULL) {
966 deferred_ttep_pair[0] = ttep;
967 deferred_tte_pair[0] = entry;
968 } else {
969 assert(deferred_ttep_pair[1] == NULL);
970 deferred_ttep_pair[1] = ttep;
971 deferred_tte_pair[1] = entry;
972 }
973 } else {
974 *ttep = entry;
975 }
976 }
977
978 /*
979 * arm_vm_page_granular_helper updates protections at the L3 level. It will (if
980 * neccessary) allocate a page for the L3 table and update the corresponding L2
981 * entry. Then, it will iterate over the L3 table, updating protections as necessary.
982 * This expects to be invoked on a L2 entry or sub L2 entry granularity, so this should
983 * not be invoked from a context that does not do L2 iteration separately (basically,
984 * don't call this except from arm_vm_page_granular_prot).
985 *
986 * unsigned granule: 0 => force to page granule, or a combination of
987 * ARM64_GRANULE_* flags declared above.
988 *
989 * unsigned int guarded => flag indicating whether this range should be
990 * considered an ARM "guarded" page. This enables BTI enforcement for a region.
991 */
992
993 static void
arm_vm_page_granular_helper(vm_offset_t start,vm_offset_t _end,vm_offset_t va,pmap_paddr_t pa_offset,int pte_prot_APX,int pte_prot_XN,unsigned granule,__unused unsigned int guarded,tt_entry_t ** deferred_ttep_pair,tt_entry_t * deferred_tte_pair)994 arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, pmap_paddr_t pa_offset,
995 int pte_prot_APX, int pte_prot_XN, unsigned granule, __unused unsigned int guarded,
996 tt_entry_t **deferred_ttep_pair, tt_entry_t *deferred_tte_pair)
997 {
998 if (va & ARM_TT_L2_OFFMASK) { /* ragged edge hanging over a ARM_TT_L2_SIZE boundary */
999 tt_entry_t *tte2;
1000 tt_entry_t tmplate;
1001 pmap_paddr_t pa;
1002 pt_entry_t *ppte, ptmp;
1003 addr64_t ppte_phys;
1004 unsigned i;
1005
1006 va &= ~ARM_TT_L2_OFFMASK;
1007 pa = va - gVirtBase + gPhysBase - pa_offset;
1008
1009 if (pa >= real_avail_end) {
1010 return;
1011 }
1012
1013 tte2 = arm_kva_to_tte(va);
1014
1015 assert(_end >= va);
1016 tmplate = *tte2;
1017
1018 if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) {
1019 /* pick up the existing page table. */
1020 ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK));
1021 } else {
1022 // TTE must be reincarnated with page level mappings.
1023
1024 // ... but we don't want to break up blocks on live
1025 // translation tables.
1026 assert(!kva_active);
1027
1028 ppte = (pt_entry_t*)alloc_ptpage(pa_offset == 0);
1029 bzero(ppte, ARM_PGBYTES);
1030 ppte_phys = kvtophys((vm_offset_t)ppte);
1031
1032 *tte2 = pa_to_tte(ppte_phys) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
1033 }
1034
1035 vm_offset_t len = _end - va;
1036 if ((pa + len) > real_avail_end) {
1037 _end -= (pa + len - real_avail_end);
1038 }
1039 assert((start - gVirtBase + gPhysBase - pa_offset) >= gPhysBase);
1040
1041 /* Round up to the nearest PAGE_SIZE boundary when creating mappings:
1042 * PAGE_SIZE may be a multiple of ARM_PGBYTES, and we don't want to leave
1043 * a ragged non-PAGE_SIZE-aligned edge. */
1044 vm_offset_t rounded_end = round_page(_end);
1045 /* Apply the desired protections to the specified page range */
1046 for (i = 0; i <= (ARM_TT_L3_INDEX_MASK >> ARM_TT_L3_SHIFT); i++) {
1047 if ((start <= va) && (va < rounded_end)) {
1048 ptmp = pa | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_TYPE;
1049 ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
1050 ptmp = ptmp | ARM_PTE_AP(pte_prot_APX);
1051 ptmp = ptmp | ARM_PTE_NX;
1052 #if __ARM_KERNEL_PROTECT__
1053 ptmp = ptmp | ARM_PTE_NG;
1054 #endif /* __ARM_KERNEL_PROTECT__ */
1055
1056 if (pte_prot_XN) {
1057 ptmp = ptmp | ARM_PTE_PNX;
1058 }
1059
1060 /*
1061 * If we can, apply the contiguous hint to this range. The hint is
1062 * applicable if the current address falls within a hint-sized range that will
1063 * be fully covered by this mapping request.
1064 */
1065 if ((va >= round_up_pte_hint_address(start)) && (round_up_pte_hint_address(va + 1) <= _end) &&
1066 (granule & ARM64_GRANULE_ALLOW_HINT) && use_contiguous_hint) {
1067 assert((va & ((1 << ARM_PTE_HINT_ADDR_SHIFT) - 1)) == ((pa & ((1 << ARM_PTE_HINT_ADDR_SHIFT) - 1))));
1068 ptmp |= ARM_PTE_HINT;
1069 /* Do not attempt to reapply the hint bit to an already-active mapping.
1070 * This very likely means we're attempting to change attributes on an already-active mapping,
1071 * which violates the requirement of the hint bit.*/
1072 assert(!kva_active || (ppte[i] == ARM_PTE_TYPE_FAULT));
1073 }
1074
1075 #if BTI_ENFORCED
1076 /*
1077 * Set the 'guarded page' flag to enable ARM BTI enforcement.
1078 */
1079 if (guarded) {
1080 ptmp |= ARM_PTE_GP;
1081 }
1082 #endif /* BTI_ENFORCED */
1083 /*
1084 * Do not change the contiguous bit on an active mapping. Even in a single-threaded
1085 * environment, it's possible for prefetch to produce a TLB conflict by trying to pull in
1086 * a hint-sized entry on top of one or more existing page-sized entries. It's also useful
1087 * to make sure we're not trying to unhint a sub-range of a larger hinted range, which
1088 * could produce a later TLB conflict.
1089 */
1090 assert(!kva_active || (ppte[i] == ARM_PTE_TYPE_FAULT) || ((ppte[i] & ARM_PTE_HINT) == (ptmp & ARM_PTE_HINT)));
1091
1092 update_or_defer_tte(&ppte[i], ptmp, pa, (vm_map_address_t)ppte, granule, deferred_ttep_pair, deferred_tte_pair);
1093 }
1094
1095 va += ARM_PGBYTES;
1096 pa += ARM_PGBYTES;
1097 }
1098 }
1099 }
1100
1101 /*
1102 * arm_vm_page_granular_prot updates protections by iterating over the L2 entries and
1103 * changing them. If a particular chunk necessitates L3 entries (for reasons of
1104 * alignment or length, or an explicit request that the entry be fully expanded), we
1105 * hand off to arm_vm_page_granular_helper to deal with the L3 chunk of the logic.
1106 */
1107 static void
arm_vm_page_granular_prot(vm_offset_t start,unsigned long size,pmap_paddr_t pa_offset,int tte_prot_XN,int pte_prot_APX,int pte_prot_XN,unsigned granule,unsigned int guarded)1108 arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, pmap_paddr_t pa_offset,
1109 int tte_prot_XN, int pte_prot_APX, int pte_prot_XN,
1110 unsigned granule, unsigned int guarded)
1111 {
1112 tt_entry_t *deferred_ttep_pair[2] = {NULL};
1113 tt_entry_t deferred_tte_pair[2] = {0};
1114 vm_offset_t _end = start + size;
1115 vm_offset_t align_start = (start + ARM_TT_L2_OFFMASK) & ~ARM_TT_L2_OFFMASK;
1116
1117 if (size == 0x0UL) {
1118 return;
1119 }
1120
1121 if (align_start > _end) {
1122 align_start = _end;
1123 }
1124
1125 arm_vm_page_granular_helper(start, align_start, start, pa_offset, pte_prot_APX, pte_prot_XN, granule, guarded, deferred_ttep_pair, deferred_tte_pair);
1126
1127 while ((_end - align_start) >= ARM_TT_L2_SIZE) {
1128 if (!(granule & ARM64_GRANULE_ALLOW_BLOCK)) {
1129 arm_vm_page_granular_helper(align_start, align_start + ARM_TT_L2_SIZE, align_start + 1, pa_offset,
1130 pte_prot_APX, pte_prot_XN, granule, guarded, deferred_ttep_pair, deferred_tte_pair);
1131 } else {
1132 pmap_paddr_t pa = align_start - gVirtBase + gPhysBase - pa_offset;
1133 assert((pa & ARM_TT_L2_OFFMASK) == 0);
1134 tt_entry_t *tte2;
1135 tt_entry_t tmplate;
1136
1137 tte2 = arm_kva_to_tte(align_start);
1138
1139 if ((pa >= gPhysBase) && (pa < real_avail_end)) {
1140 tmplate = (pa & ARM_TTE_BLOCK_L2_MASK) | ARM_TTE_TYPE_BLOCK
1141 | ARM_TTE_VALID | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NX
1142 | ARM_TTE_BLOCK_AP(pte_prot_APX) | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)
1143 | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
1144
1145 #if __ARM_KERNEL_PROTECT__
1146 tmplate = tmplate | ARM_TTE_BLOCK_NG;
1147 #endif /* __ARM_KERNEL_PROTECT__ */
1148 if (tte_prot_XN) {
1149 tmplate = tmplate | ARM_TTE_BLOCK_PNX;
1150 }
1151
1152 update_or_defer_tte(tte2, tmplate, pa, (vm_map_address_t)tte2 & ~ARM_TT_L2_OFFMASK,
1153 granule, deferred_ttep_pair, deferred_tte_pair);
1154 }
1155 }
1156 align_start += ARM_TT_L2_SIZE;
1157 }
1158
1159 if (align_start < _end) {
1160 arm_vm_page_granular_helper(align_start, _end, _end, pa_offset, pte_prot_APX, pte_prot_XN, granule, guarded, deferred_ttep_pair, deferred_tte_pair);
1161 }
1162
1163 if (deferred_ttep_pair[0] != NULL) {
1164 #if DEBUG || DEVELOPMENT
1165 /*
1166 * Flush the TLB to catch bugs that might cause us to prematurely revoke write access from the pagetable page.
1167 * These bugs may otherwise be hidden by TLB entries in most cases, resulting in very rare panics.
1168 * Note that we always flush the TLB at the end of arm_vm_prot_finalize().
1169 */
1170 flush_mmu_tlb();
1171 #endif
1172 /*
1173 * The first TTE in the pair is a recursive mapping of the pagetable region, so we must update it last
1174 * to avoid potentially marking deferred_pte_pair[1] read-only.
1175 */
1176 if (deferred_tte_pair[1] != 0) {
1177 os_atomic_store(deferred_ttep_pair[1], deferred_tte_pair[1], release);
1178 }
1179 os_atomic_store(deferred_ttep_pair[0], deferred_tte_pair[0], release);
1180 }
1181 }
1182
1183 static inline void
arm_vm_page_granular_RNX(vm_offset_t start,unsigned long size,unsigned granule)1184 arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, unsigned granule)
1185 {
1186 arm_vm_page_granular_prot(start, size, 0, 1, AP_RONA, 1, granule, ARM64_PAGE_UNGUARDED);
1187 }
1188
1189 static inline void
arm_vm_page_granular_ROX(vm_offset_t start,unsigned long size,unsigned granule,unsigned int guarded)1190 arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, unsigned granule, unsigned int guarded)
1191 {
1192 arm_vm_page_granular_prot(start, size, 0, 0, AP_RONA, 0, granule, guarded);
1193 }
1194
1195 static inline void
arm_vm_page_granular_RWNX(vm_offset_t start,unsigned long size,unsigned granule)1196 arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, unsigned granule)
1197 {
1198 arm_vm_page_granular_prot(start, size, 0, 1, AP_RWNA, 1, granule, ARM64_PAGE_UNGUARDED);
1199 }
1200
1201 // Populate seg...AuxKC and fixup AuxKC permissions
1202 static bool
arm_vm_auxkc_init(void)1203 arm_vm_auxkc_init(void)
1204 {
1205 if (auxkc_mh == 0 || auxkc_base == 0) {
1206 return false; // no auxKC.
1207 }
1208
1209 /* Fixup AuxKC and populate seg*AuxKC globals used below */
1210 arm_auxkc_init((void*)auxkc_mh, (void*)auxkc_base);
1211
1212 if (segLOWESTAuxKC != segLOWEST) {
1213 panic("segLOWESTAuxKC (%p) not equal to segLOWEST (%p). auxkc_mh: %p, auxkc_base: %p",
1214 (void*)segLOWESTAuxKC, (void*)segLOWEST,
1215 (void*)auxkc_mh, (void*)auxkc_base);
1216 }
1217
1218 /*
1219 * The AuxKC LINKEDIT segment needs to be covered by the RO region but is excluded
1220 * from the RO address range returned by kernel_collection_adjust_mh_addrs().
1221 * Ensure the highest non-LINKEDIT address in the AuxKC is the current end of
1222 * its RO region before extending it.
1223 */
1224 assert(segHIGHESTROAuxKC == segHIGHESTNLEAuxKC);
1225 assert(segHIGHESTAuxKC >= segHIGHESTROAuxKC);
1226 if (segHIGHESTAuxKC > segHIGHESTROAuxKC) {
1227 segHIGHESTROAuxKC = segHIGHESTAuxKC;
1228 }
1229
1230 /*
1231 * The AuxKC RO region must be right below the device tree/trustcache so that it can be covered
1232 * by CTRR, and the AuxKC RX region must be within the RO region.
1233 */
1234 assert(segHIGHESTROAuxKC == auxkc_right_above);
1235 assert(segHIGHESTRXAuxKC <= segHIGHESTROAuxKC);
1236 assert(segLOWESTRXAuxKC <= segHIGHESTRXAuxKC);
1237 assert(segLOWESTROAuxKC <= segLOWESTRXAuxKC);
1238 assert(segLOWESTAuxKC <= segLOWESTROAuxKC);
1239
1240 if (segHIGHESTRXAuxKC < segLOWEST) {
1241 arm_vm_page_granular_RNX(segHIGHESTRXAuxKC, segLOWEST - segHIGHESTRXAuxKC, 0);
1242 }
1243 if (segLOWESTRXAuxKC < segHIGHESTRXAuxKC) {
1244 /*
1245 * We cannot mark auxKC text as guarded because doing so would enforce
1246 * BTI on oblivious third-party kexts and break ABI compatibility.
1247 * Doing this defeats the purpose of BTI (branches to these pages are
1248 * unchecked!) but given both the relative rarity and the diversity of
1249 * third-party kexts, we expect that this is likely impractical to
1250 * exploit in practice.
1251 */
1252 arm_vm_page_granular_ROX(segLOWESTRXAuxKC, segHIGHESTRXAuxKC - segLOWESTRXAuxKC, 0, ARM64_PAGE_UNGUARDED); // Refined in OSKext::readPrelinkedExtensions
1253 }
1254 if (segLOWESTROAuxKC < segLOWESTRXAuxKC) {
1255 arm_vm_page_granular_RNX(segLOWESTROAuxKC, segLOWESTRXAuxKC - segLOWESTROAuxKC, 0);
1256 }
1257 if (segLOWESTAuxKC < segLOWESTROAuxKC) {
1258 arm_vm_page_granular_RWNX(segLOWESTAuxKC, segLOWESTROAuxKC - segLOWESTAuxKC, 0);
1259 }
1260
1261 return true;
1262 }
1263
1264 void
arm_vm_prot_init(__unused boot_args * args)1265 arm_vm_prot_init(__unused boot_args * args)
1266 {
1267 segLOWESTTEXT = UINT64_MAX;
1268 if (segSizePRELINKTEXT && (segPRELINKTEXTB < segLOWESTTEXT)) {
1269 segLOWESTTEXT = segPRELINKTEXTB;
1270 }
1271 assert(segSizeTEXT);
1272 if (segTEXTB < segLOWESTTEXT) {
1273 segLOWESTTEXT = segTEXTB;
1274 }
1275 assert(segLOWESTTEXT < UINT64_MAX);
1276
1277 segEXTRADATA = 0;
1278 segSizeEXTRADATA = 0;
1279 segTRUSTCACHE = 0;
1280 segSizeTRUSTCACHE = 0;
1281
1282 segLOWEST = segLOWESTTEXT;
1283 segLOWESTRO = segLOWESTTEXT;
1284
1285 if (segLOWESTKC && segLOWESTKC < segLOWEST) {
1286 /*
1287 * kernel collections have segments below the kernel. In particular the collection mach header
1288 * is below PRELINK_TEXT and is not covered by any other segments already tracked.
1289 */
1290 arm_vm_page_granular_RNX(segLOWESTKC, segLOWEST - segLOWESTKC, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1291 segLOWEST = segLOWESTKC;
1292 if (segLOWESTROKC && segLOWESTROKC < segLOWESTRO) {
1293 segLOWESTRO = segLOWESTROKC;
1294 }
1295 if (segHIGHESTROKC && segHIGHESTROKC > segHIGHESTRO) {
1296 segHIGHESTRO = segHIGHESTROKC;
1297 }
1298 }
1299
1300 DTEntry memory_map;
1301 int err;
1302
1303 // Device Tree portion of EXTRADATA
1304 if (SecureDTIsLockedDown()) {
1305 segEXTRADATA = (vm_offset_t)PE_state.deviceTreeHead;
1306 segSizeEXTRADATA = PE_state.deviceTreeSize;
1307 }
1308
1309 // Trust Caches portion of EXTRADATA
1310 {
1311 DTMemoryMapRange const *trustCacheRange;
1312 unsigned int trustCacheRangeSize;
1313
1314 err = SecureDTLookupEntry(NULL, "chosen/memory-map", &memory_map);
1315 assert(err == kSuccess);
1316
1317 err = SecureDTGetProperty(memory_map, "TrustCache", (void const **)&trustCacheRange, &trustCacheRangeSize);
1318 if (err == kSuccess) {
1319 if (trustCacheRangeSize != sizeof(DTMemoryMapRange)) {
1320 panic("Unexpected /chosen/memory-map/TrustCache property size %u != %zu", trustCacheRangeSize, sizeof(DTMemoryMapRange));
1321 }
1322
1323 vm_offset_t const trustCacheRegion = phystokv(trustCacheRange->paddr);
1324 if (trustCacheRegion < segLOWEST) {
1325 if (segEXTRADATA != 0) {
1326 if (trustCacheRegion != segEXTRADATA + segSizeEXTRADATA) {
1327 panic("Unexpected location of TrustCache region: %#lx != %#lx",
1328 trustCacheRegion, segEXTRADATA + segSizeEXTRADATA);
1329 }
1330 segSizeEXTRADATA += trustCacheRange->length;
1331 } else {
1332 // Not all devices support CTRR device trees.
1333 segEXTRADATA = trustCacheRegion;
1334 segSizeEXTRADATA = trustCacheRange->length;
1335 }
1336 }
1337 #if !(DEVELOPMENT || DEBUG)
1338 else {
1339 panic("TrustCache region is in an unexpected place: %#lx > %#lx", trustCacheRegion, segLOWEST);
1340 }
1341 #endif
1342 segTRUSTCACHE = trustCacheRegion;
1343 segSizeTRUSTCACHE = trustCacheRange->length;
1344 }
1345 }
1346
1347 if (segSizeEXTRADATA != 0) {
1348 if (segEXTRADATA <= segLOWEST) {
1349 segLOWEST = segEXTRADATA;
1350 if (segEXTRADATA <= segLOWESTRO) {
1351 segLOWESTRO = segEXTRADATA;
1352 }
1353 } else {
1354 panic("EXTRADATA is in an unexpected place: %#lx > %#lx", segEXTRADATA, segLOWEST);
1355 }
1356
1357 arm_vm_page_granular_RNX(segEXTRADATA, segSizeEXTRADATA, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1358 }
1359
1360 const DTMemoryMapRange *auxKC_range, *auxKC_header_range;
1361 unsigned int auxKC_range_size, auxKC_header_range_size;
1362
1363 err = SecureDTGetProperty(memory_map, "AuxKC", (const void**)&auxKC_range,
1364 &auxKC_range_size);
1365 if (err != kSuccess) {
1366 goto noAuxKC;
1367 }
1368 assert(auxKC_range_size == sizeof(DTMemoryMapRange));
1369 err = SecureDTGetProperty(memory_map, "AuxKC-mach_header",
1370 (const void**)&auxKC_header_range, &auxKC_header_range_size);
1371 if (err != kSuccess) {
1372 goto noAuxKC;
1373 }
1374 assert(auxKC_header_range_size == sizeof(DTMemoryMapRange));
1375
1376 if (auxKC_header_range->paddr == 0 || auxKC_range->paddr == 0) {
1377 goto noAuxKC;
1378 }
1379
1380 auxkc_mh = phystokv(auxKC_header_range->paddr);
1381 auxkc_base = phystokv(auxKC_range->paddr);
1382
1383 if (auxkc_base < segLOWEST) {
1384 auxkc_right_above = segLOWEST;
1385 segLOWEST = auxkc_base;
1386 } else {
1387 panic("auxkc_base (%p) not below segLOWEST (%p)", (void*)auxkc_base, (void*)segLOWEST);
1388 }
1389
1390 /* Map AuxKC RWNX initially so that arm_vm_auxkc_init can traverse
1391 * it and apply fixups (after we're off the bootstrap translation
1392 * tables).
1393 */
1394 arm_vm_page_granular_RWNX(auxkc_base, auxKC_range->length, 0);
1395
1396 noAuxKC:
1397 /* Map coalesced kext TEXT segment RWNX for now */
1398 arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, ARM64_GRANULE_ALLOW_BLOCK); // Refined in OSKext::readPrelinkedExtensions
1399
1400 /* Map coalesced kext DATA_CONST segment RWNX (could be empty) */
1401 arm_vm_page_granular_RWNX(segPLKDATACONSTB, segSizePLKDATACONST, ARM64_GRANULE_ALLOW_BLOCK); // Refined in OSKext::readPrelinkedExtensions
1402
1403 /* Map coalesced kext TEXT_EXEC segment RX (could be empty) */
1404 arm_vm_page_granular_ROX(segPLKTEXTEXECB, segSizePLKTEXTEXEC, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT, ARM64_PAGE_GUARDED); // Refined in OSKext::readPrelinkedExtensions
1405
1406 /* if new segments not present, set space between PRELINK_TEXT and xnu TEXT to RWNX
1407 * otherwise we no longer expect any space between the coalesced kext read only segments and xnu rosegments
1408 */
1409 if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC) {
1410 if (segSizePRELINKTEXT) {
1411 arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT),
1412 ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1413 }
1414 } else {
1415 /*
1416 * If we have the new segments, we should still protect the gap between kext
1417 * read-only pages and kernel read-only pages, in the event that this gap
1418 * exists.
1419 */
1420 if ((segPLKDATACONSTB + segSizePLKDATACONST) < segTEXTB) {
1421 arm_vm_page_granular_RWNX(segPLKDATACONSTB + segSizePLKDATACONST, segTEXTB - (segPLKDATACONSTB + segSizePLKDATACONST),
1422 ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1423 }
1424 }
1425
1426 /*
1427 * Protection on kernel text is loose here to allow shenanigans early on. These
1428 * protections are tightened in arm_vm_prot_finalize(). This is necessary because
1429 * we currently patch LowResetVectorBase in cpu.c.
1430 *
1431 * TEXT segment contains mach headers and other non-executable data. This will become RONX later.
1432 */
1433 arm_vm_page_granular_RNX(segTEXTB, segSizeTEXT, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1434
1435 /* Can DATACONST start out and stay RNX?
1436 * NO, stuff in this segment gets modified during startup (viz. mac_policy_init()/mac_policy_list)
1437 * Make RNX in prot_finalize
1438 */
1439 arm_vm_page_granular_RWNX(segDATACONSTB, segSizeDATACONST, ARM64_GRANULE_ALLOW_BLOCK);
1440
1441 arm_vm_page_granular_ROX(segTEXTEXECB, segSizeTEXTEXEC, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT, ARM64_PAGE_GUARDED);
1442
1443 #if XNU_MONITOR
1444 arm_vm_page_granular_ROX(segPPLTEXTB, segSizePPLTEXT, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT, ARM64_PAGE_UNGUARDED);
1445 arm_vm_page_granular_ROX(segPPLTRAMPB, segSizePPLTRAMP, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT, ARM64_PAGE_UNGUARDED);
1446 arm_vm_page_granular_RNX(segPPLDATACONSTB, segSizePPLDATACONST, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1447 #endif
1448
1449 /* DATA segment will remain RWNX */
1450 arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1451 #if XNU_MONITOR
1452 arm_vm_page_granular_RWNX(segPPLDATAB, segSizePPLDATA, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1453 #endif
1454
1455 arm_vm_page_granular_RWNX(segHIBDATAB, segSizeHIBDATA, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1456
1457 arm_vm_page_granular_RWNX(segBOOTDATAB, segSizeBOOTDATA, 0);
1458 arm_vm_page_granular_RNX((vm_offset_t)&intstack_low_guard, PAGE_MAX_SIZE, 0);
1459 arm_vm_page_granular_RNX((vm_offset_t)&intstack_high_guard, PAGE_MAX_SIZE, 0);
1460 arm_vm_page_granular_RNX((vm_offset_t)&excepstack_high_guard, PAGE_MAX_SIZE, 0);
1461
1462 arm_vm_page_granular_ROX(segKLDB, segSizeKLD, 0, ARM64_PAGE_GUARDED);
1463 arm_vm_page_granular_RNX(segKLDDATAB, segSizeKLDDATA, 0);
1464 arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, 0);
1465 arm_vm_page_granular_RWNX(segPLKLINKEDITB, segSizePLKLINKEDIT, 0); // Coalesced kext LINKEDIT segment
1466 arm_vm_page_granular_ROX(segLASTB, segSizeLAST, ARM64_GRANULE_ALLOW_BLOCK, ARM64_PAGE_GUARDED); // __LAST may be empty, but we cannot assume this
1467 if (segLASTDATACONSTB) {
1468 arm_vm_page_granular_RWNX(segLASTDATACONSTB, segSizeLASTDATACONST, ARM64_GRANULE_ALLOW_BLOCK); // __LASTDATA_CONST may be empty, but we cannot assume this
1469 }
1470 arm_vm_page_granular_RWNX(segPRELINKDATAB, segSizePRELINKDATA, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT); // Prelink __DATA for kexts (RW data)
1471
1472 if (segSizePLKLLVMCOV > 0) {
1473 arm_vm_page_granular_RWNX(segPLKLLVMCOVB, segSizePLKLLVMCOV, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT); // LLVM code coverage data
1474 }
1475 arm_vm_page_granular_RWNX(segPRELINKINFOB, segSizePRELINKINFO, 0); /* PreLinkInfoDictionary */
1476
1477 /* Record the bounds of the kernelcache. */
1478 vm_kernelcache_base = segLOWEST;
1479 vm_kernelcache_top = end_kern;
1480 }
1481
1482 /*
1483 * return < 0 for a < b
1484 * 0 for a == b
1485 * > 0 for a > b
1486 */
1487 typedef int (*cmpfunc_t)(const void *a, const void *b);
1488
1489 extern void
1490 qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
1491
1492 static int
cmp_ptov_entries(const void * a,const void * b)1493 cmp_ptov_entries(const void *a, const void *b)
1494 {
1495 const ptov_table_entry *entry_a = a;
1496 const ptov_table_entry *entry_b = b;
1497 // Sort in descending order of segment length
1498 if (entry_a->len < entry_b->len) {
1499 return 1;
1500 } else if (entry_a->len > entry_b->len) {
1501 return -1;
1502 } else {
1503 return 0;
1504 }
1505 }
1506
1507 SECURITY_READ_ONLY_LATE(static unsigned int) ptov_index = 0;
1508
1509 #define ROUND_L1(addr) (((addr) + ARM_TT_L1_OFFMASK) & ~(ARM_TT_L1_OFFMASK))
1510 #define ROUND_TWIG(addr) (((addr) + ARM_TT_TWIG_OFFMASK) & ~(ARM_TT_TWIG_OFFMASK))
1511
1512 static void
arm_vm_physmap_slide(ptov_table_entry * temp_ptov_table,vm_map_address_t orig_va,vm_size_t len,int pte_prot_APX,unsigned granule)1513 arm_vm_physmap_slide(ptov_table_entry *temp_ptov_table, vm_map_address_t orig_va, vm_size_t len, int pte_prot_APX, unsigned granule)
1514 {
1515 pmap_paddr_t pa_offset;
1516
1517 if (__improbable(ptov_index >= PTOV_TABLE_SIZE)) {
1518 panic("%s: PTOV table limit exceeded; segment va = 0x%llx, size = 0x%llx", __func__,
1519 (unsigned long long)orig_va, (unsigned long long)len);
1520 }
1521 assert((orig_va & ARM_PGMASK) == 0);
1522 temp_ptov_table[ptov_index].pa = orig_va - gVirtBase + gPhysBase;
1523 if (ptov_index == 0) {
1524 temp_ptov_table[ptov_index].va = physmap_base;
1525 } else {
1526 temp_ptov_table[ptov_index].va = temp_ptov_table[ptov_index - 1].va + temp_ptov_table[ptov_index - 1].len;
1527 }
1528 if (granule & ARM64_GRANULE_ALLOW_BLOCK) {
1529 vm_map_address_t orig_offset = temp_ptov_table[ptov_index].pa & ARM_TT_TWIG_OFFMASK;
1530 vm_map_address_t new_offset = temp_ptov_table[ptov_index].va & ARM_TT_TWIG_OFFMASK;
1531 if (new_offset < orig_offset) {
1532 temp_ptov_table[ptov_index].va += (orig_offset - new_offset);
1533 } else if (new_offset > orig_offset) {
1534 temp_ptov_table[ptov_index].va = ROUND_TWIG(temp_ptov_table[ptov_index].va) + orig_offset;
1535 }
1536 }
1537 assert((temp_ptov_table[ptov_index].va & ARM_PGMASK) == 0);
1538 temp_ptov_table[ptov_index].len = round_page(len);
1539 pa_offset = temp_ptov_table[ptov_index].va - orig_va;
1540 arm_vm_page_granular_prot(temp_ptov_table[ptov_index].va, temp_ptov_table[ptov_index].len, pa_offset, 1, pte_prot_APX, 1, granule, ARM64_PAGE_UNGUARDED);
1541 ++ptov_index;
1542 }
1543
1544 #if XNU_MONITOR
1545
1546
1547
1548 SECURITY_READ_ONLY_LATE(static boolean_t) keep_linkedit = FALSE;
1549
1550 static void
arm_vm_physmap_init(boot_args * args)1551 arm_vm_physmap_init(boot_args *args)
1552 {
1553 ptov_table_entry temp_ptov_table[PTOV_TABLE_SIZE];
1554 bzero(temp_ptov_table, sizeof(temp_ptov_table));
1555
1556 // This is memory that will either be handed back to the VM layer via ml_static_mfree(),
1557 // or will be available for general-purpose use. Physical aperture mappings for this memory
1558 // must be at page granularity, so that PPL ownership or cache attribute changes can be reflected
1559 // in the physical aperture mappings.
1560
1561 // Slid region between gPhysBase and beginning of protected text
1562 arm_vm_physmap_slide(temp_ptov_table, gVirtBase, segLOWEST - gVirtBase, AP_RWNA, 0);
1563
1564 // kext bootstrap segments
1565 #if !defined(KERNEL_INTEGRITY_KTRR) && !defined(KERNEL_INTEGRITY_CTRR)
1566 /* __KLD,__text is covered by the rorgn */
1567 arm_vm_physmap_slide(temp_ptov_table, segKLDB, segSizeKLD, AP_RONA, 0);
1568 #endif
1569 arm_vm_physmap_slide(temp_ptov_table, segKLDDATAB, segSizeKLDDATA, AP_RONA, 0);
1570
1571 // Early-boot data
1572 arm_vm_physmap_slide(temp_ptov_table, segBOOTDATAB, segSizeBOOTDATA, AP_RONA, 0);
1573
1574 PE_parse_boot_argn("keepsyms", &keep_linkedit, sizeof(keep_linkedit));
1575 #if CONFIG_DTRACE
1576 if (dtrace_keep_kernel_symbols()) {
1577 keep_linkedit = TRUE;
1578 }
1579 #endif /* CONFIG_DTRACE */
1580 #if KASAN_DYNAMIC_BLACKLIST
1581 /* KASAN's dynamic blacklist needs to query the LINKEDIT segment at runtime. As such, the
1582 * kext bootstrap code will not jettison LINKEDIT on kasan kernels, so don't bother to relocate it. */
1583 keep_linkedit = TRUE;
1584 #endif
1585 if (!keep_linkedit) {
1586 // Kernel LINKEDIT
1587 arm_vm_physmap_slide(temp_ptov_table, segLINKB, segSizeLINK, AP_RWNA, 0);
1588
1589 if (segSizePLKLINKEDIT) {
1590 // Prelinked kernel LINKEDIT
1591 arm_vm_physmap_slide(temp_ptov_table, segPLKLINKEDITB, segSizePLKLINKEDIT, AP_RWNA, 0);
1592 }
1593 }
1594
1595 // Prelinked kernel plists
1596 arm_vm_physmap_slide(temp_ptov_table, segPRELINKINFOB, segSizePRELINKINFO, AP_RWNA, 0);
1597
1598 // Device tree (if not locked down), ramdisk, boot args
1599 arm_vm_physmap_slide(temp_ptov_table, end_kern, (args->topOfKernelData - gPhysBase + gVirtBase) - end_kern, AP_RWNA, 0);
1600 if (!SecureDTIsLockedDown()) {
1601 PE_slide_devicetree(temp_ptov_table[ptov_index - 1].va - end_kern);
1602 }
1603
1604 // Remainder of physical memory
1605 arm_vm_physmap_slide(temp_ptov_table, (args->topOfKernelData - gPhysBase + gVirtBase),
1606 real_avail_end - args->topOfKernelData, AP_RWNA, 0);
1607
1608
1609
1610 assert((temp_ptov_table[ptov_index - 1].va + temp_ptov_table[ptov_index - 1].len) <= physmap_end);
1611
1612 // Sort in descending order of segment length. LUT traversal is linear, so largest (most likely used)
1613 // segments should be placed earliest in the table to optimize lookup performance.
1614 qsort(temp_ptov_table, PTOV_TABLE_SIZE, sizeof(temp_ptov_table[0]), cmp_ptov_entries);
1615
1616 memcpy(ptov_table, temp_ptov_table, sizeof(ptov_table));
1617 }
1618
1619 #else
1620
1621 static void
arm_vm_physmap_init(boot_args * args)1622 arm_vm_physmap_init(boot_args *args)
1623 {
1624 ptov_table_entry temp_ptov_table[PTOV_TABLE_SIZE];
1625 bzero(temp_ptov_table, sizeof(temp_ptov_table));
1626
1627 // Will be handed back to VM layer through ml_static_mfree() in arm_vm_prot_finalize()
1628 arm_vm_physmap_slide(temp_ptov_table, gVirtBase, segLOWEST - gVirtBase, AP_RWNA,
1629 ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT);
1630
1631 arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern,
1632 ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT); /* Device Tree (if not locked down), RAM Disk (if present), bootArgs */
1633
1634 arm_vm_physmap_slide(temp_ptov_table, (args->topOfKernelData - gPhysBase + gVirtBase),
1635 real_avail_end - args->topOfKernelData, AP_RWNA, ARM64_GRANULE_ALLOW_BLOCK | ARM64_GRANULE_ALLOW_HINT); // rest of physmem
1636
1637 assert((temp_ptov_table[ptov_index - 1].va + temp_ptov_table[ptov_index - 1].len) <= physmap_end);
1638
1639 // Sort in descending order of segment length. LUT traversal is linear, so largest (most likely used)
1640 // segments should be placed earliest in the table to optimize lookup performance.
1641 qsort(temp_ptov_table, PTOV_TABLE_SIZE, sizeof(temp_ptov_table[0]), cmp_ptov_entries);
1642
1643 memcpy(ptov_table, temp_ptov_table, sizeof(ptov_table));
1644 }
1645
1646 #endif // XNU_MONITOR
1647
1648 void
arm_vm_prot_finalize(boot_args * args __unused)1649 arm_vm_prot_finalize(boot_args * args __unused)
1650 {
1651 /*
1652 * At this point, we are far enough along in the boot process that it will be
1653 * safe to free up all of the memory preceeding the kernel. It may in fact
1654 * be safe to do this earlier.
1655 *
1656 * This keeps the memory in the V-to-P mapping, but advertises it to the VM
1657 * as usable.
1658 */
1659
1660 /*
1661 * if old style PRELINK segment exists, free memory before it, and after it before XNU text
1662 * otherwise we're dealing with a new style kernel cache, so we should just free the
1663 * memory before PRELINK_TEXT segment, since the rest of the KEXT read only data segments
1664 * should be immediately followed by XNU's TEXT segment
1665 */
1666
1667 ml_static_mfree(phystokv(gPhysBase), segLOWEST - gVirtBase);
1668
1669 /*
1670 * KTRR support means we will be mucking with these pages and trying to
1671 * protect them; we cannot free the pages to the VM if we do this.
1672 */
1673 if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC && segSizePRELINKTEXT) {
1674 /* If new segments not present, PRELINK_TEXT is not dynamically sized, free DRAM between it and xnu TEXT */
1675 ml_static_mfree(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT));
1676 }
1677
1678 /* tighten permissions on kext read only data and code */
1679 arm_vm_page_granular_RNX(segPRELINKTEXTB, segSizePRELINKTEXT, ARM64_GRANULE_ALLOW_BLOCK);
1680 arm_vm_page_granular_RNX(segPLKDATACONSTB, segSizePLKDATACONST, ARM64_GRANULE_ALLOW_BLOCK);
1681
1682 cpu_stack_alloc(&BootCpuData);
1683 arm64_replace_bootstack(&BootCpuData);
1684 ml_static_mfree(phystokv(segBOOTDATAB - gVirtBase + gPhysBase), segSizeBOOTDATA);
1685
1686 #if __ARM_KERNEL_PROTECT__
1687 arm_vm_populate_kernel_el0_mappings();
1688 #endif /* __ARM_KERNEL_PROTECT__ */
1689
1690 #if XNU_MONITOR
1691 #if !defined(KERNEL_INTEGRITY_KTRR) && !defined(KERNEL_INTEGRITY_CTRR)
1692 /* __KLD,__text is covered by the rorgn */
1693 for (vm_offset_t va = segKLDB; va < (segKLDB + segSizeKLD); va += ARM_PGBYTES) {
1694 pt_entry_t *pte = arm_kva_to_pte(va);
1695 *pte = ARM_PTE_EMPTY;
1696 }
1697 #endif
1698 for (vm_offset_t va = segKLDDATAB; va < (segKLDDATAB + segSizeKLDDATA); va += ARM_PGBYTES) {
1699 pt_entry_t *pte = arm_kva_to_pte(va);
1700 *pte = ARM_PTE_EMPTY;
1701 }
1702 /* Clear the original stack mappings; these pages should be mapped through ptov_table. */
1703 for (vm_offset_t va = segBOOTDATAB; va < (segBOOTDATAB + segSizeBOOTDATA); va += ARM_PGBYTES) {
1704 pt_entry_t *pte = arm_kva_to_pte(va);
1705 *pte = ARM_PTE_EMPTY;
1706 }
1707 /* Clear the original PRELINKINFO mapping. This segment should be jettisoned during I/O Kit
1708 * initialization before we reach this point. */
1709 for (vm_offset_t va = segPRELINKINFOB; va < (segPRELINKINFOB + segSizePRELINKINFO); va += ARM_PGBYTES) {
1710 pt_entry_t *pte = arm_kva_to_pte(va);
1711 *pte = ARM_PTE_EMPTY;
1712 }
1713 if (!keep_linkedit) {
1714 for (vm_offset_t va = segLINKB; va < (segLINKB + segSizeLINK); va += ARM_PGBYTES) {
1715 pt_entry_t *pte = arm_kva_to_pte(va);
1716 *pte = ARM_PTE_EMPTY;
1717 }
1718 if (segSizePLKLINKEDIT) {
1719 for (vm_offset_t va = segPLKLINKEDITB; va < (segPLKLINKEDITB + segSizePLKLINKEDIT); va += ARM_PGBYTES) {
1720 pt_entry_t *pte = arm_kva_to_pte(va);
1721 *pte = ARM_PTE_EMPTY;
1722 }
1723 }
1724 }
1725 #endif /* XNU_MONITOR */
1726
1727 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
1728 /*
1729 * __LAST,__pinst should no longer be executable.
1730 */
1731 arm_vm_page_granular_RNX(segLASTB, segSizeLAST, ARM64_GRANULE_ALLOW_BLOCK);
1732
1733 /* __LASTDATA_CONST should no longer be writable. */
1734 if (segLASTDATACONSTB) {
1735 arm_vm_page_granular_RNX(segLASTDATACONSTB, segSizeLASTDATACONST, ARM64_GRANULE_ALLOW_BLOCK);
1736 }
1737
1738 /*
1739 * __KLD,__text should no longer be executable.
1740 */
1741 arm_vm_page_granular_RNX(segKLDB, segSizeKLD, ARM64_GRANULE_ALLOW_BLOCK);
1742
1743 /*
1744 * Must wait until all other region permissions are set before locking down DATA_CONST
1745 * as the kernel static page tables live in DATA_CONST on KTRR enabled systems
1746 * and will become immutable.
1747 */
1748 #endif
1749
1750 arm_vm_page_granular_RNX(segDATACONSTB, segSizeDATACONST, ARM64_GRANULE_ALLOW_BLOCK);
1751
1752 __builtin_arm_dsb(DSB_ISH);
1753 flush_mmu_tlb();
1754 }
1755
1756 /*
1757 * TBI (top-byte ignore) is an ARMv8 feature for ignoring the top 8 bits of
1758 * address accesses. It can be enabled separately for TTBR0 (user) and
1759 * TTBR1 (kernel).
1760 */
1761 void
arm_set_kernel_tbi(void)1762 arm_set_kernel_tbi(void)
1763 {
1764 #if !__ARM_KERNEL_PROTECT__ && CONFIG_KERNEL_TBI
1765 uint64_t old_tcr, new_tcr;
1766
1767 old_tcr = new_tcr = get_tcr();
1768 /*
1769 * For kernel configurations that require TBI support on
1770 * PAC systems, we enable DATA TBI only.
1771 */
1772 new_tcr |= TCR_TBI1_TOPBYTE_IGNORED;
1773 new_tcr |= TCR_TBID1_ENABLE;
1774
1775 if (old_tcr != new_tcr) {
1776 set_tcr(new_tcr);
1777 sysreg_restore.tcr_el1 = new_tcr;
1778 }
1779 #endif /* !__ARM_KERNEL_PROTECT__ && CONFIG_KERNEL_TBI */
1780 }
1781
1782 static void
arm_set_user_tbi(void)1783 arm_set_user_tbi(void)
1784 {
1785 #if !__ARM_KERNEL_PROTECT__
1786 uint64_t old_tcr, new_tcr;
1787
1788 old_tcr = new_tcr = get_tcr();
1789 new_tcr |= TCR_TBI0_TOPBYTE_IGNORED;
1790
1791 if (old_tcr != new_tcr) {
1792 set_tcr(new_tcr);
1793 sysreg_restore.tcr_el1 = new_tcr;
1794 }
1795 #endif /* !__ARM_KERNEL_PROTECT__ */
1796 }
1797
1798 /*
1799 * Initialize and enter blank (invalid) page tables in a L1 translation table for a given VA range.
1800 *
1801 * This is a helper function used to build up the initial page tables for the kernel translation table.
1802 * With KERNEL_INTEGRITY we keep at least the root level of the kernel page table immutable, thus the need
1803 * to preallocate before machine_lockdown any L1 entries necessary during the entire kernel runtime.
1804 *
1805 * For a given VA range, if necessary, allocate new L2 translation tables and install the table entries in
1806 * the appropriate L1 table indexes. called before the translation table is active
1807 *
1808 * parameters:
1809 *
1810 * tt: virtual address of L1 translation table to modify
1811 * start: beginning of VA range
1812 * end: end of VA range
1813 * static_map: whether to allocate the new translation table page from read only memory
1814 * table_attrs: attributes of new table entry in addition to VALID and TYPE_TABLE attributes
1815 *
1816 */
1817
1818 static void
init_ptpages(tt_entry_t * tt,vm_map_address_t start,vm_map_address_t end,bool static_map,uint64_t table_attrs)1819 init_ptpages(tt_entry_t *tt, vm_map_address_t start, vm_map_address_t end, bool static_map, uint64_t table_attrs)
1820 {
1821 tt_entry_t *l1_tte;
1822 vm_offset_t ptpage_vaddr;
1823
1824 l1_tte = tt + ((start & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
1825
1826 while (start < end) {
1827 if (*l1_tte == ARM_TTE_EMPTY) {
1828 /* Allocate a page and setup L1 Table TTE in L1 */
1829 ptpage_vaddr = alloc_ptpage(static_map);
1830 *l1_tte = (kvtophys(ptpage_vaddr) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | table_attrs;
1831 bzero((void *)ptpage_vaddr, ARM_PGBYTES);
1832 }
1833
1834 if ((start + ARM_TT_L1_SIZE) < start) {
1835 /* If this is the last L1 entry, it must cover the last mapping. */
1836 break;
1837 }
1838
1839 start += ARM_TT_L1_SIZE;
1840 l1_tte++;
1841 }
1842 }
1843
1844 #define ARM64_PHYSMAP_SLIDE_RANGE (1ULL << 30) // 1 GB
1845 #define ARM64_PHYSMAP_SLIDE_MASK (ARM64_PHYSMAP_SLIDE_RANGE - 1)
1846
1847 void
arm_vm_init(uint64_t memory_size,boot_args * args)1848 arm_vm_init(uint64_t memory_size, boot_args * args)
1849 {
1850 vm_map_address_t va_l1, va_l1_end;
1851 tt_entry_t *cpu_l1_tte;
1852 vm_map_address_t va_l2, va_l2_end;
1853 tt_entry_t *cpu_l2_tte;
1854 pmap_paddr_t boot_ttep;
1855 tt_entry_t *boot_tte;
1856 uint64_t mem_segments;
1857 vm_offset_t ptpage_vaddr;
1858 vm_map_address_t dynamic_memory_begin;
1859
1860 /*
1861 * Get the virtual and physical kernel-managed memory base from boot_args.
1862 */
1863 gVirtBase = args->virtBase;
1864 gPhysBase = args->physBase;
1865 #if KASAN
1866 real_phys_size = args->memSize + (shadow_ptop - shadow_pbase);
1867 #else
1868 real_phys_size = args->memSize;
1869 #endif
1870 /*
1871 * Ensure the physical region we specify for the VM to manage ends on a
1872 * software page boundary. Note that the software page size (PAGE_SIZE)
1873 * may be a multiple of the hardware page size specified in ARM_PGBYTES.
1874 * We must round the reported memory size down to the nearest PAGE_SIZE
1875 * boundary to ensure the VM does not try to manage a page it does not
1876 * completely own. The KASAN shadow region, if present, is managed entirely
1877 * in units of the hardware page size and should not need similar treatment.
1878 */
1879 gPhysSize = mem_size = ((gPhysBase + args->memSize) & ~PAGE_MASK) - gPhysBase;
1880
1881 mem_actual = args->memSizeActual ? args->memSizeActual : mem_size;
1882
1883 if ((memory_size != 0) && (mem_size > memory_size)) {
1884 mem_size = memory_size;
1885 max_mem_actual = memory_size;
1886 } else {
1887 max_mem_actual = mem_actual;
1888 }
1889 #if !defined(ARM_LARGE_MEMORY)
1890 if (mem_size >= ((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 2)) {
1891 panic("Unsupported memory configuration %lx", mem_size);
1892 }
1893 #endif
1894
1895 #if defined(ARM_LARGE_MEMORY)
1896 unsigned long physmap_l1_entries = ((real_phys_size + ARM64_PHYSMAP_SLIDE_RANGE) >> ARM_TT_L1_SHIFT) + 1;
1897 physmap_base = VM_MIN_KERNEL_ADDRESS - (physmap_l1_entries << ARM_TT_L1_SHIFT);
1898 #else
1899 physmap_base = phystokv(args->topOfKernelData);
1900 #endif
1901
1902 // Slide the physical aperture to a random page-aligned location within the slide range
1903 uint64_t physmap_slide = early_random() & ARM64_PHYSMAP_SLIDE_MASK & ~((uint64_t)PAGE_MASK);
1904 assert(physmap_slide < ARM64_PHYSMAP_SLIDE_RANGE);
1905
1906 physmap_base += physmap_slide;
1907
1908 #if XNU_MONITOR
1909 physmap_base = ROUND_TWIG(physmap_base);
1910 #if defined(ARM_LARGE_MEMORY)
1911 static_memory_end = phystokv(args->topOfKernelData);
1912 #else
1913 static_memory_end = physmap_base + mem_size;
1914 #endif // ARM_LARGE_MEMORY
1915 physmap_end = physmap_base + real_phys_size;
1916
1917
1918 #else
1919 #if defined(ARM_LARGE_MEMORY)
1920 /* For large memory systems with no PPL such as virtual machines */
1921 static_memory_end = phystokv(args->topOfKernelData);
1922 physmap_end = physmap_base + real_phys_size;
1923 #else
1924 static_memory_end = physmap_base + mem_size + (PTOV_TABLE_SIZE * ARM_TT_TWIG_SIZE); // worst possible case for block alignment
1925 physmap_end = physmap_base + real_phys_size + (PTOV_TABLE_SIZE * ARM_TT_TWIG_SIZE);
1926 #endif // ARM_LARGE_MEMORY
1927 #endif
1928
1929 #if KASAN && !defined(ARM_LARGE_MEMORY)
1930 /* add the KASAN stolen memory to the physmap */
1931 dynamic_memory_begin = static_memory_end + (shadow_ptop - shadow_pbase);
1932 #else
1933 dynamic_memory_begin = static_memory_end;
1934 #endif
1935 #if XNU_MONITOR
1936 pmap_stacks_start = (void*)dynamic_memory_begin;
1937 dynamic_memory_begin += PPL_STACK_REGION_SIZE;
1938 pmap_stacks_end = (void*)dynamic_memory_begin;
1939
1940 #if HAS_GUARDED_IO_FILTER
1941 iofilter_stacks_start = (void*)dynamic_memory_begin;
1942 dynamic_memory_begin += IOFILTER_STACK_REGION_SIZE;
1943 iofilter_stacks_end = (void*)dynamic_memory_begin;
1944 #endif
1945 #endif
1946 if (dynamic_memory_begin > VM_MAX_KERNEL_ADDRESS) {
1947 panic("Unsupported memory configuration %lx", mem_size);
1948 }
1949
1950 boot_tte = (tt_entry_t *)&bootstrap_pagetables;
1951 boot_ttep = kvtophys((vm_offset_t)boot_tte);
1952
1953 #if DEVELOPMENT || DEBUG
1954 /* Sanity check - assert that BOOTSTRAP_TABLE_SIZE is sufficiently-large to
1955 * hold our bootstrap mappings for any possible slide */
1956 size_t bytes_mapped = dynamic_memory_begin - gVirtBase;
1957 size_t l1_entries = 1 + ((bytes_mapped + ARM_TT_L1_SIZE - 1) / ARM_TT_L1_SIZE);
1958 /* 1 L1 each for V=P and KVA, plus 1 page for each L2 */
1959 size_t pages_used = 2 * (l1_entries + 1);
1960 if (pages_used > BOOTSTRAP_TABLE_SIZE) {
1961 panic("BOOTSTRAP_TABLE_SIZE too small for memory config");
1962 }
1963 #endif
1964
1965 /*
1966 * TTBR0 L1, TTBR0 L2 - 1:1 bootstrap mapping.
1967 * TTBR1 L1, TTBR1 L2 - kernel mapping
1968 */
1969
1970 /*
1971 * TODO: free bootstrap table memory back to allocator.
1972 * on large memory systems bootstrap tables could be quite large.
1973 * after bootstrap complete, xnu can warm start with a single 16KB page mapping
1974 * to trampoline to KVA. this requires only 3 pages to stay resident.
1975 */
1976 first_avail_phys = avail_start = args->topOfKernelData;
1977
1978 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
1979 arm_replace_identity_map();
1980 #endif
1981
1982 /* Initialize invalid tte page */
1983 invalid_tte = (tt_entry_t *)alloc_ptpage(TRUE);
1984 invalid_ttep = kvtophys((vm_offset_t)invalid_tte);
1985 bzero(invalid_tte, ARM_PGBYTES);
1986
1987 /*
1988 * Initialize l1 page table page
1989 */
1990 cpu_tte = (tt_entry_t *)alloc_ptpage(TRUE);
1991 cpu_ttep = kvtophys((vm_offset_t)cpu_tte);
1992 bzero(cpu_tte, ARM_PGBYTES);
1993 avail_end = gPhysBase + mem_size;
1994 assert(!(avail_end & PAGE_MASK));
1995
1996 #if KASAN
1997 real_avail_end = gPhysBase + real_phys_size;
1998 #else
1999 real_avail_end = avail_end;
2000 #endif
2001
2002 /*
2003 * Initialize l1 and l2 page table pages :
2004 * map physical memory at the kernel base virtual address
2005 * cover the kernel dynamic address range section
2006 *
2007 * the so called physical aperture should be statically mapped
2008 */
2009 init_ptpages(cpu_tte, gVirtBase, dynamic_memory_begin, TRUE, ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2010
2011 #if defined(ARM_LARGE_MEMORY)
2012 /*
2013 * Initialize l1 page table pages :
2014 * on large memory systems the physical aperture exists separately below
2015 * the rest of the kernel virtual address space
2016 */
2017 init_ptpages(cpu_tte, physmap_base, ROUND_L1(physmap_end), TRUE, ARM_DYNAMIC_TABLE_XN | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2018 #endif
2019
2020
2021 #if __ARM_KERNEL_PROTECT__
2022 /* Expand the page tables to prepare for the EL0 mappings. */
2023 arm_vm_expand_kernel_el0_mappings();
2024 #endif /* __ARM_KERNEL_PROTECT__ */
2025
2026 /*
2027 * Now retrieve addresses for various segments from kernel mach-o header
2028 */
2029 segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT);
2030 segPLKDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_DATA_CONST", &segSizePLKDATACONST);
2031 segPLKTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_TEXT_EXEC", &segSizePLKTEXTEXEC);
2032 segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT);
2033 segDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA_CONST", &segSizeDATACONST);
2034 segTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT_EXEC", &segSizeTEXTEXEC);
2035 #if XNU_MONITOR
2036 segPPLTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PPLTEXT", &segSizePPLTEXT);
2037 segPPLTRAMPB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PPLTRAMP", &segSizePPLTRAMP);
2038 segPPLDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PPLDATA_CONST", &segSizePPLDATACONST);
2039 #endif
2040 segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA);
2041 #if XNU_MONITOR
2042 segPPLDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PPLDATA", &segSizePPLDATA);
2043 #endif
2044
2045 segBOOTDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__BOOTDATA", &segSizeBOOTDATA);
2046 segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK);
2047 segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD);
2048 segKLDDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLDDATA", &segSizeKLDDATA);
2049 segPRELINKDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_DATA", &segSizePRELINKDATA);
2050 segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO);
2051 segPLKLLVMCOVB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LLVM_COV", &segSizePLKLLVMCOV);
2052 segPLKLINKEDITB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LINKEDIT", &segSizePLKLINKEDIT);
2053 segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST);
2054 segLASTDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LASTDATA_CONST", &segSizeLASTDATACONST);
2055
2056 sectHIBTEXTB = (vm_offset_t) getsectdatafromheader(&_mh_execute_header, "__TEXT_EXEC", "__hib_text", §SizeHIBTEXT);
2057 sectHIBDATACONSTB = (vm_offset_t) getsectdatafromheader(&_mh_execute_header, "__DATA_CONST", "__hib_const", §SizeHIBDATACONST);
2058 segHIBDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__HIBDATA", &segSizeHIBDATA);
2059
2060 if (kernel_mach_header_is_in_fileset(&_mh_execute_header)) {
2061 kernel_mach_header_t *kc_mh = PE_get_kc_header(KCKindPrimary);
2062
2063 // fileset has kext PLK_TEXT_EXEC under kernel collection TEXT_EXEC following kernel's LAST
2064 segKCTEXTEXECB = (vm_offset_t) getsegdatafromheader(kc_mh, "__TEXT_EXEC", &segSizeKCTEXTEXEC);
2065 assert(segPLKTEXTEXECB && !segSizePLKTEXTEXEC); // kernel PLK_TEXT_EXEC must be empty
2066
2067 assert(segLASTB); // kernel LAST can be empty, but it must have
2068 // a valid address for computations below.
2069
2070 assert(segKCTEXTEXECB <= segLASTB); // KC TEXT_EXEC must contain kernel LAST
2071 assert(segKCTEXTEXECB + segSizeKCTEXTEXEC >= segLASTB + segSizeLAST);
2072 segPLKTEXTEXECB = segLASTB + segSizeLAST;
2073 segSizePLKTEXTEXEC = segSizeKCTEXTEXEC - (segPLKTEXTEXECB - segKCTEXTEXECB);
2074
2075 // fileset has kext PLK_DATA_CONST under kernel collection DATA_CONST following kernel's LASTDATA_CONST
2076 segKCDATACONSTB = (vm_offset_t) getsegdatafromheader(kc_mh, "__DATA_CONST", &segSizeKCDATACONST);
2077 assert(segPLKDATACONSTB && !segSizePLKDATACONST); // kernel PLK_DATA_CONST must be empty
2078 assert(segLASTDATACONSTB && segSizeLASTDATACONST); // kernel LASTDATA_CONST must be non-empty
2079 assert(segKCDATACONSTB <= segLASTDATACONSTB); // KC DATA_CONST must contain kernel LASTDATA_CONST
2080 assert(segKCDATACONSTB + segSizeKCDATACONST >= segLASTDATACONSTB + segSizeLASTDATACONST);
2081 segPLKDATACONSTB = segLASTDATACONSTB + segSizeLASTDATACONST;
2082 segSizePLKDATACONST = segSizeKCDATACONST - (segPLKDATACONSTB - segKCDATACONSTB);
2083
2084 // fileset has kext PRELINK_DATA under kernel collection DATA following kernel's empty PRELINK_DATA
2085 segKCDATAB = (vm_offset_t) getsegdatafromheader(kc_mh, "__DATA", &segSizeKCDATA);
2086 assert(segPRELINKDATAB && !segSizePRELINKDATA); // kernel PRELINK_DATA must be empty
2087 assert(segKCDATAB <= segPRELINKDATAB); // KC DATA must contain kernel PRELINK_DATA
2088 assert(segKCDATAB + segSizeKCDATA >= segPRELINKDATAB + segSizePRELINKDATA);
2089 segSizePRELINKDATA = segSizeKCDATA - (segPRELINKDATAB - segKCDATAB);
2090
2091 // fileset has consolidated PRELINK_TEXT, PRELINK_INFO and LINKEDIT at the kernel collection level
2092 assert(segPRELINKTEXTB && !segSizePRELINKTEXT); // kernel PRELINK_TEXT must be empty
2093 segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(kc_mh, "__PRELINK_TEXT", &segSizePRELINKTEXT);
2094 assert(segPRELINKINFOB && !segSizePRELINKINFO); // kernel PRELINK_INFO must be empty
2095 segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(kc_mh, "__PRELINK_INFO", &segSizePRELINKINFO);
2096 segLINKB = (vm_offset_t) getsegdatafromheader(kc_mh, "__LINKEDIT", &segSizeLINK);
2097 }
2098
2099 (void) PE_parse_boot_argn("use_contiguous_hint", &use_contiguous_hint, sizeof(use_contiguous_hint));
2100 assert(segSizePRELINKTEXT < 0x03000000); /* 23355738 */
2101
2102 /* if one of the new segments is present, the other one better be as well */
2103 if (segSizePLKDATACONST || segSizePLKTEXTEXEC) {
2104 assert(segSizePLKDATACONST && segSizePLKTEXTEXEC);
2105 }
2106
2107 etext = (vm_offset_t) segTEXTB + segSizeTEXT;
2108 sdata = (vm_offset_t) segDATAB;
2109 edata = (vm_offset_t) segDATAB + segSizeDATA;
2110 end_kern = round_page(segHIGHESTKC ? segHIGHESTKC : getlastkerneladdr()); /* Force end to next page */
2111
2112 vm_set_page_size();
2113
2114 vm_kernel_base = segTEXTB;
2115 vm_kernel_top = (vm_offset_t) &last_kernel_symbol;
2116 vm_kext_base = segPRELINKTEXTB;
2117 vm_kext_top = vm_kext_base + segSizePRELINKTEXT;
2118
2119 vm_prelink_stext = segPRELINKTEXTB;
2120 if (!segSizePLKTEXTEXEC && !segSizePLKDATACONST) {
2121 vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT;
2122 } else {
2123 vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT + segSizePLKDATACONST + segSizePLKTEXTEXEC;
2124 }
2125 vm_prelink_sinfo = segPRELINKINFOB;
2126 vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO;
2127 vm_slinkedit = segLINKB;
2128 vm_elinkedit = segLINKB + segSizeLINK;
2129
2130 vm_prelink_sdata = segPRELINKDATAB;
2131 vm_prelink_edata = segPRELINKDATAB + segSizePRELINKDATA;
2132
2133 arm_vm_prot_init(args);
2134
2135 /*
2136 * Initialize the page tables for the low globals:
2137 * cover this address range:
2138 * LOW_GLOBAL_BASE_ADDRESS + 2MB
2139 */
2140 va_l1 = va_l2 = LOW_GLOBAL_BASE_ADDRESS;
2141 cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
2142 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
2143 ptpage_vaddr = alloc_ptpage(TRUE);
2144 *cpu_l2_tte = (kvtophys(ptpage_vaddr) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN;
2145 bzero((void *)ptpage_vaddr, ARM_PGBYTES);
2146
2147 /*
2148 * Initialize l2 page table pages :
2149 * cover this address range:
2150 * KERNEL_DYNAMIC_ADDR - VM_MAX_KERNEL_ADDRESS
2151 */
2152 #if defined(ARM_LARGE_MEMORY)
2153 /*
2154 * dynamic mapped memory outside the VM allocator VA range required to bootstrap VM system
2155 * don't expect to exceed 64GB, no sense mapping any more space between here and the VM heap range
2156 */
2157 init_ptpages(cpu_tte, dynamic_memory_begin, ROUND_L1(dynamic_memory_begin), FALSE, ARM_DYNAMIC_TABLE_XN | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2158 #else
2159 /*
2160 * TODO: do these pages really need to come from RO memory?
2161 * With legacy 3 level table systems we never mapped more than a single L1 entry so this may be dead code
2162 */
2163 init_ptpages(cpu_tte, dynamic_memory_begin, VM_MAX_KERNEL_ADDRESS, TRUE, ARM_DYNAMIC_TABLE_XN | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2164 #endif
2165
2166 #if KASAN
2167 /* record the extent of the physmap */
2168 physmap_vbase = physmap_base;
2169 physmap_vtop = physmap_end;
2170 kasan_init();
2171 #endif /* KASAN */
2172
2173 #if CONFIG_CPU_COUNTERS
2174 mt_early_init();
2175 #endif /* CONFIG_CPU_COUNTERS */
2176
2177 arm_set_user_tbi();
2178
2179 arm_vm_physmap_init(args);
2180 set_mmu_ttb_alternate(cpu_ttep & TTBR_BADDR_MASK);
2181
2182 ml_enable_monitor();
2183
2184 set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK);
2185
2186 flush_mmu_tlb();
2187 kva_active = TRUE;
2188 // global table pointers may need to be different due to physical aperture remapping
2189 cpu_tte = (tt_entry_t*)(phystokv(cpu_ttep));
2190 invalid_tte = (tt_entry_t*)(phystokv(invalid_ttep));
2191
2192 // From here on out, we're off the bootstrap translation tables.
2193
2194
2195 /* AuxKC initialization has to be deferred until this point, since
2196 * the AuxKC may not have been fully mapped in the bootstrap
2197 * tables, if it spilled downwards into the prior L2 block.
2198 *
2199 * Now that its mapping set up by arm_vm_prot_init() is active,
2200 * we can traverse and fix it up.
2201 */
2202
2203 /* Calculate the physical bounds of the kernelcache; using
2204 * gVirtBase/gPhysBase math to do this directly is generally a bad idea
2205 * as the physmap is no longer physically contiguous. However, this is
2206 * done here as segLOWEST and end_kern are both virtual addresses the
2207 * bootstrap physmap, and because kvtophys references the page tables
2208 * (at least at the time this comment was written), meaning that at
2209 * least end_kern may not point to a valid mapping on some kernelcache
2210 * configurations, so kvtophys would report a physical address of 0.
2211 *
2212 * Long term, the kernelcache should probably be described in terms of
2213 * multiple physical ranges, as there is no strong guarantee or
2214 * requirement that the kernelcache will always be physically
2215 * contiguous.
2216 */
2217 arm_vm_kernelcache_phys_start = segLOWEST - gVirtBase + gPhysBase;
2218 arm_vm_kernelcache_phys_end = end_kern - gVirtBase + gPhysBase;;
2219
2220 /* Calculate the number of pages that belong to the kernelcache. */
2221 vm_page_kernelcache_count = (unsigned int) (atop_64(arm_vm_kernelcache_phys_end - arm_vm_kernelcache_phys_start));
2222
2223 if (arm_vm_auxkc_init()) {
2224 if (segLOWESTROAuxKC < segLOWESTRO) {
2225 segLOWESTRO = segLOWESTROAuxKC;
2226 }
2227 if (segHIGHESTROAuxKC > segHIGHESTRO) {
2228 segHIGHESTRO = segHIGHESTROAuxKC;
2229 }
2230 if (segLOWESTRXAuxKC < segLOWESTTEXT) {
2231 segLOWESTTEXT = segLOWESTRXAuxKC;
2232 }
2233 assert(segLOWEST == segLOWESTAuxKC);
2234
2235 // The preliminary auxKC mapping has been broken up.
2236 flush_mmu_tlb();
2237 }
2238
2239 sane_size = mem_size - (avail_start - gPhysBase);
2240 max_mem = mem_size;
2241 vm_kernel_slid_base = segLOWESTTEXT;
2242 // vm_kernel_slide is set by arm_init()->arm_slide_rebase_and_sign_image()
2243 vm_kernel_stext = segTEXTB;
2244
2245 if (kernel_mach_header_is_in_fileset(&_mh_execute_header)) {
2246 vm_kernel_etext = segTEXTEXECB + segSizeTEXTEXEC;
2247 vm_kernel_slid_top = vm_slinkedit;
2248 } else {
2249 assert(segDATACONSTB == segTEXTB + segSizeTEXT);
2250 assert(segTEXTEXECB == segDATACONSTB + segSizeDATACONST);
2251 vm_kernel_etext = segTEXTB + segSizeTEXT + segSizeDATACONST + segSizeTEXTEXEC;
2252 vm_kernel_slid_top = vm_prelink_einfo;
2253 }
2254
2255 dynamic_memory_begin = ROUND_TWIG(dynamic_memory_begin);
2256 #if defined(KERNEL_INTEGRITY_CTRR) && defined(CONFIG_XNUPOST)
2257 // reserve a 32MB region without permission overrides to use later for a CTRR unit test
2258 {
2259 extern vm_offset_t ctrr_test_page;
2260 tt_entry_t *new_tte;
2261
2262 ctrr_test_page = dynamic_memory_begin;
2263 dynamic_memory_begin += ARM_TT_L2_SIZE;
2264 cpu_l1_tte = cpu_tte + ((ctrr_test_page & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
2265 assert((*cpu_l1_tte) & ARM_TTE_VALID);
2266 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((ctrr_test_page & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
2267 assert((*cpu_l2_tte) == ARM_TTE_EMPTY);
2268 new_tte = (tt_entry_t *)alloc_ptpage(FALSE);
2269 bzero(new_tte, ARM_PGBYTES);
2270 *cpu_l2_tte = (kvtophys((vm_offset_t)new_tte) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
2271 }
2272 #endif /* defined(KERNEL_INTEGRITY_CTRR) && defined(CONFIG_XNUPOST) */
2273 #if XNU_MONITOR
2274 for (vm_offset_t cur = (vm_offset_t)pmap_stacks_start; cur < (vm_offset_t)pmap_stacks_end; cur += ARM_PGBYTES) {
2275 arm_vm_map(cpu_tte, cur, ARM_PTE_EMPTY);
2276 }
2277 #if HAS_GUARDED_IO_FILTER
2278 for (vm_offset_t cur = (vm_offset_t)iofilter_stacks_start; cur < (vm_offset_t)iofilter_stacks_end; cur += ARM_PGBYTES) {
2279 arm_vm_map(cpu_tte, cur, ARM_PTE_EMPTY);
2280 }
2281 #endif
2282 #endif
2283 pmap_bootstrap(dynamic_memory_begin);
2284
2285 disable_preemption();
2286
2287 /*
2288 * Initialize l3 page table pages :
2289 * cover this address range:
2290 * 2MB + FrameBuffer size + 10MB for each 256MB segment
2291 */
2292
2293 mem_segments = (mem_size + 0x0FFFFFFF) >> 28;
2294
2295 va_l1 = dynamic_memory_begin;
2296 va_l1_end = va_l1 + ((2 + (mem_segments * 10)) << 20);
2297 va_l1_end += round_page(args->Video.v_height * args->Video.v_rowBytes);
2298 va_l1_end = (va_l1_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL;
2299
2300 cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
2301
2302 while (va_l1 < va_l1_end) {
2303 va_l2 = va_l1;
2304
2305 if (((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE) < va_l1) {
2306 /* If this is the last L1 entry, it must cover the last mapping. */
2307 va_l2_end = va_l1_end;
2308 } else {
2309 va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE, va_l1_end);
2310 }
2311
2312 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
2313
2314 while (va_l2 < va_l2_end) {
2315 pt_entry_t * ptp;
2316 pmap_paddr_t ptp_phys;
2317
2318 /* Allocate a page and setup L3 Table TTE in L2 */
2319 ptp = (pt_entry_t *) alloc_ptpage(FALSE);
2320 ptp_phys = (pmap_paddr_t)kvtophys((vm_offset_t)ptp);
2321
2322 bzero(ptp, ARM_PGBYTES);
2323 pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE);
2324
2325 *cpu_l2_tte = (pa_to_tte(ptp_phys)) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_DYNAMIC_TABLE_XN;
2326
2327 va_l2 += ARM_TT_L2_SIZE;
2328 cpu_l2_tte++;
2329 }
2330
2331 va_l1 = va_l2_end;
2332 cpu_l1_tte++;
2333 }
2334
2335 #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
2336 /*
2337 * In this configuration, the bootstrap mappings (arm_vm_init) and
2338 * the heap mappings occupy separate L1 regions. Explicitly set up
2339 * the heap L1 allocations here.
2340 */
2341 #if defined(ARM_LARGE_MEMORY)
2342 init_ptpages(cpu_tte, KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_L1_OFFMASK, VM_MAX_KERNEL_ADDRESS, FALSE, ARM_DYNAMIC_TABLE_XN | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2343 #else // defined(ARM_LARGE_MEMORY)
2344 va_l1 = VM_MIN_KERNEL_ADDRESS & ~ARM_TT_L1_OFFMASK;
2345 init_ptpages(cpu_tte, VM_MIN_KERNEL_ADDRESS & ~ARM_TT_L1_OFFMASK, VM_MAX_KERNEL_ADDRESS, FALSE, ARM_DYNAMIC_TABLE_XN | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2346 #endif // defined(ARM_LARGE_MEMORY)
2347 #else
2348 #if defined(ARM_LARGE_MEMORY)
2349 /* For large memory systems with no KTRR/CTRR such as virtual machines */
2350 init_ptpages(cpu_tte, KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_L1_OFFMASK, VM_MAX_KERNEL_ADDRESS, FALSE, ARM_DYNAMIC_TABLE_XN | ARM_TTE_TABLE_AP(ARM_TTE_TABLE_AP_USER_NA));
2351 #endif
2352 #endif // defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
2353
2354 /*
2355 * Initialize l3 page table pages :
2356 * cover this address range:
2357 * ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - PE_EARLY_BOOT_VA) to VM_MAX_KERNEL_ADDRESS
2358 */
2359 va_l1 = (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - PE_EARLY_BOOT_VA;
2360 va_l1_end = VM_MAX_KERNEL_ADDRESS;
2361
2362 cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
2363
2364 while (va_l1 < va_l1_end) {
2365 va_l2 = va_l1;
2366
2367 if (((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE) < va_l1) {
2368 /* If this is the last L1 entry, it must cover the last mapping. */
2369 va_l2_end = va_l1_end;
2370 } else {
2371 va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE, va_l1_end);
2372 }
2373
2374 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
2375
2376 while (va_l2 < va_l2_end) {
2377 pt_entry_t * ptp;
2378 pmap_paddr_t ptp_phys;
2379
2380 /* Allocate a page and setup L3 Table TTE in L2 */
2381 ptp = (pt_entry_t *) alloc_ptpage(FALSE);
2382 ptp_phys = (pmap_paddr_t)kvtophys((vm_offset_t)ptp);
2383
2384 bzero(ptp, ARM_PGBYTES);
2385 pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE);
2386
2387 *cpu_l2_tte = (pa_to_tte(ptp_phys)) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_DYNAMIC_TABLE_XN;
2388
2389 va_l2 += ARM_TT_L2_SIZE;
2390 cpu_l2_tte++;
2391 }
2392
2393 va_l1 = va_l2_end;
2394 cpu_l1_tte++;
2395 }
2396
2397
2398 /*
2399 * Adjust avail_start so that the range that the VM owns
2400 * starts on a PAGE_SIZE aligned boundary.
2401 */
2402 avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK;
2403
2404 #if XNU_MONITOR
2405 pmap_static_allocations_done();
2406 #endif
2407 first_avail = avail_start;
2408 patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData);
2409 enable_preemption();
2410 }
2411
2412 /*
2413 * Returns true if the address is within __TEXT, __TEXT_EXEC or __DATA_CONST
2414 * segment range. This is what [vm_kernel_stext, vm_kernel_etext) range used to
2415 * cover. The segments together may not be continuous anymore and so individual
2416 * intervals are inspected.
2417 */
2418 bool
kernel_text_contains(vm_offset_t addr)2419 kernel_text_contains(vm_offset_t addr)
2420 {
2421 if (segTEXTB <= addr && addr < (segTEXTB + segSizeTEXT)) {
2422 return true;
2423 }
2424 if (segTEXTEXECB <= addr && addr < (segTEXTEXECB + segSizeTEXTEXEC)) {
2425 return true;
2426 }
2427 return segDATACONSTB <= addr && addr < (segDATACONSTB + segSizeDATACONST);
2428 }
2429