1 /**
2 * Copyright (c) 2022-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm64/lowglobals.h>
30 #include <kern/ecc.h>
31 #include <kern/timer_queue.h>
32 #include <kern/monotonic.h>
33 #include <machine/commpage.h>
34 #include <pexpert/device_tree.h>
35 #include <arm/cpu_internal.h>
36 #include <arm/misc_protos.h>
37 #include <arm/machine_cpu.h>
38 #include <arm/rtclock.h>
39 #include <vm/vm_map.h>
40 #include <mach/exclaves.h>
41 #include <mach/vm_param.h>
42 #include <libkern/stack_protector.h>
43 #include <console/serial_protos.h>
44 #include <arm64/sptm/pmap/pmap_pt_geometry.h>
45 #include <arm64/sptm/sptm.h>
46 #include <sptm/sptm_common.h>
47 #include <vm/vm_page_internal.h>
48
49 #if CONFIG_TELEMETRY
50 #include <kern/telemetry.h>
51 #endif /* CONFIG_TELEMETRY */
52
53 #if KPERF
54 #include <kperf/kptimer.h>
55 #endif /* KPERF */
56
57 #if HIBERNATION
58 #include <IOKit/IOPlatformExpert.h>
59 #include <machine/pal_hibernate.h>
60 #endif /* HIBERNATION */
61
62 #if __arm64__
63 #include <pexpert/arm64/apt_msg.h>
64 #endif
65
66
67 /**
68 * Functions defined elsewhere that are required by this source file.
69 */
70 extern void patch_low_glo(void);
71 extern int serial_init(void);
72 extern void sleep_token_buffer_init(void);
73
74 /**
75 * Bootstrap stacks. Used on the cold boot path to set up the boot CPU's
76 * per-CPU data structure.
77 */
78 extern vm_offset_t intstack_top;
79 extern vm_offset_t excepstack_top;
80
81 /* First (inclusive) and last (exclusive) physical addresses */
82 extern pmap_paddr_t vm_first_phys;
83 extern pmap_paddr_t vm_last_phys;
84
85 /* UART hibernation flag - import so we can set it ASAP on resume. */
86 extern MARK_AS_HIBERNATE_DATA bool uart_hibernation;
87
88 /* Used to cache memSize, as passed by iBoot */
89 SECURITY_READ_ONLY_LATE(uint64_t) memSize = 0;
90
91 int debug_task;
92
93 /**
94 * Set according to what serial-related boot-args have been passed to XUN.
95 */
96 extern int disableConsoleOutput;
97
98 #if XNU_TARGET_OS_OSX
99 /**
100 * Extern the PMAP boot-arg to enable/disable XNU_KERNEL_RESTRICTED.
101 * We need it here because if we detect an auxKC, we disable the mitigation.
102 */
103 extern bool use_xnu_restricted;
104 #endif /* XNU_TARGET_OS_OSX */
105
106 /**
107 * SPTM devices do not support static kernelcaches, but the rest of XNU
108 * expects this variable to be defined. Set it to false at build time.
109 */
110 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
111
112 TUNABLE(bool, restore_boot, "-restore", false);
113
114 /**
115 * First physical address freely available to xnu.
116 */
117 SECURITY_READ_ONLY_LATE(addr64_t) first_avail_phys = 0;
118
119 #if HAS_BP_RET
120 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
121 uint32_t bp_ret = 3;
122 extern void set_bp_ret(void);
123 #endif
124
125 #if SCHED_HYGIENE_DEBUG
126
127 #if XNU_PLATFORM_iPhoneOS
128 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 48000 /* 2ms */
129 #elif XNU_PLATFORM_XROS
130 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
131 #else
132 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 0xd0000 /* 35.499ms */
133 #endif /* XNU_PLATFORM_iPhoneOS */
134
135 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, interrupt_masked_debug_mode,
136 "machine-timeouts", "interrupt-masked-debug-mode",
137 "interrupt-masked-debug-mode",
138 SCHED_HYGIENE_MODE_PANIC,
139 TUNABLE_DT_CHECK_CHOSEN);
140
141 MACHINE_TIMEOUT_DEV_WRITEABLE(interrupt_masked_timeout, "interrupt-masked",
142 DEFAULT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
143 NULL);
144 #if __arm64__
145 #define SSHOT_INTERRUPT_MASKED_TIMEOUT 0xf9999 /* 64-bit: 42.599ms */
146 #endif
147 MACHINE_TIMEOUT_DEV_WRITEABLE(stackshot_interrupt_masked_timeout, "sshot-interrupt-masked",
148 SSHOT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
149 NULL);
150 #undef SSHOT_INTERRUPT_MASKED_TIMEOUT
151 #endif
152
153 /*
154 * A 6-second timeout will give the watchdog code a chance to run
155 * before a panic is triggered by the xcall routine.
156 */
157 #define XCALL_ACK_TIMEOUT_NS ((uint64_t) 6000000000)
158 uint64_t xcall_ack_timeout_abstime;
159
160 #ifndef __BUILDING_XNU_LIBRARY__
161 #define BOOTARGS_SECTION_ATTR __attribute__((section("__DATA, __const")))
162 #else /* __BUILDING_XNU_LIBRARY__ */
163 /* Special segments are not used when building for user-mode */
164 #define BOOTARGS_SECTION_ATTR
165 #endif /* __BUILDING_XNU_LIBRARY__ */
166
167 boot_args const_boot_args BOOTARGS_SECTION_ATTR;
168 boot_args *BootArgs BOOTARGS_SECTION_ATTR;
169
170 /**
171 * The SPTM provides a second set of boot arguments, on top of those
172 * provided by iBoot.
173 */
174 SECURITY_READ_ONLY_LATE(sptm_bootstrap_args_xnu_t) const_sptm_args;
175 SECURITY_READ_ONLY_LATE(const sptm_bootstrap_args_xnu_t *) SPTMArgs;
176 SECURITY_READ_ONLY_LATE(const bool *) sptm_xnu_triggered_panic_ptr;
177
178 extern char osbuild_config[];
179
180 TUNABLE(uint32_t, arm_diag, "diag", 0);
181 #ifdef APPLETYPHOON
182 static unsigned cpus_defeatures = 0x0;
183 extern void cpu_defeatures_set(unsigned int);
184 #endif
185
186 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
187 extern volatile boolean_t arm64_stall_sleep;
188 #endif
189
190 extern boolean_t force_immediate_debug_halt;
191
192 #if HAS_APPLE_PAC
193 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
194 #endif
195
196
197
198 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
199 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
200 SECURITY_READ_ONLY_LATE(ppnum_t) pmap_first_pnum;
201
202 SECURITY_READ_ONLY_LATE(bool) serial_console_enabled = false;
203
204 /**
205 * SPTM TODO: The following flag is set up based on the presence and
206 * configuration of the 'sptm_stability_hacks' boot-arg; this
207 * is used in certain codepaths that do not properly function
208 * today in SPTM systems to make the system more stable and fully
209 * able to boot to user space.
210 */
211 SECURITY_READ_ONLY_LATE(bool) sptm_stability_hacks = false;
212
213 #if APPLEVIRTUALPLATFORM
214 SECURITY_READ_ONLY_LATE(vm_offset_t) reset_vector_vaddr = 0;
215 #endif /* APPLEVIRTUALPLATFORM */
216
217 /*
218 * Forward definition
219 */
220 void arm_init(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_args);
221 #if KASAN
222 void arm_init_kasan(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_args);
223 #endif /* KASAN */
224
225 #if __arm64__
226 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
227
228 extern void configure_misc_apple_boot_args(void);
229 extern void configure_misc_apple_regs(bool is_boot_cpu);
230 extern void configure_timer_apple_regs(void);
231 #endif /* __arm64__ */
232
233
234 /*
235 * JOP rebasing
236 */
237
238 #define dyldLogFunc(msg, ...)
239 #include <mach/dyld_kernel_fixups.h>
240
241 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
242 extern uint32_t __thread_starts_sect_end[] __asm("section$end$__TEXT$__thread_starts");
243 #if defined(HAS_APPLE_PAC)
244 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
245 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
246 #endif /* defined(HAS_APPLE_PAC) */
247
248 extern vm_offset_t vm_kernel_slide;
249 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
250 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
251 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
252
253
254 void arm_slide_rebase_and_sign_image(void);
255 MARK_AS_FIXUP_TEXT void
arm_slide_rebase_and_sign_image(void)256 arm_slide_rebase_and_sign_image(void)
257 {
258 kernel_mach_header_t *k_mh, *kc_mh = NULL;
259 kernel_segment_command_t *seg;
260 uintptr_t slide;
261
262 /*
263 * The kernel is part of a MH_FILESET kernel collection, determine slide
264 * based on first segment's mach-o vmaddr (requires first kernel load
265 * command to be LC_SEGMENT_64 of the __TEXT segment)
266 */
267 k_mh = &_mh_execute_header;
268 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
269 assert(seg->cmd == LC_SEGMENT_KERNEL);
270 slide = (uintptr_t)k_mh - seg->vmaddr;
271
272 /*
273 * The kernel collection linker guarantees that the boot collection mach
274 * header vmaddr is the hardcoded kernel link address (as specified to
275 * ld64 when linking the kernel).
276 */
277 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
278 assert(kc_mh->filetype == MH_FILESET);
279
280 /*
281 * rebase and sign jops
282 * Note that we can't call any functions before this point, so
283 * we have to hard-code the knowledge that the base of the KC
284 * is the KC's mach-o header. This would change if any
285 * segment's VA started *before* the text segment
286 * (as the HIB segment does on x86).
287 */
288 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
289 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
290 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
291
292 /*
293 * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
294 * collection, so adjust them now, and determine the vmaddr range
295 * covered by read-only segments for the CTRR rorgn.
296 */
297 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
298 (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
299 (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
300 NULL, NULL, NULL);
301
302 /*
303 * Initialize slide global here to avoid duplicating this logic in
304 * arm_vm_init()
305 */
306 vm_kernel_slide = slide;
307 }
308
309 void arm_static_if_init(boot_args *args);
310 MARK_AS_FIXUP_TEXT void
arm_static_if_init(boot_args * args)311 arm_static_if_init(boot_args *args)
312 {
313 static_if_init(args->CommandLine);
314 }
315
316 void
arm_auxkc_init(void * mh,void * base)317 arm_auxkc_init(void *mh, void *base)
318 {
319 /*
320 * The kernel collection linker guarantees that the lowest vmaddr in an
321 * AuxKC collection is 0 (but note that the mach header is higher up since
322 * RW segments precede RO segments in the AuxKC).
323 */
324 uintptr_t slide = (uintptr_t)base;
325 kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
326
327 assert(akc_mh->filetype == MH_FILESET);
328 PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
329
330 /* rebase and sign jops */
331 const void *collection_base_pointers[KCNumKinds];
332 memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
333 kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
334
335 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
336 (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
337 (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
338 (uintptr_t *)&segHIGHESTNLEAuxKC);
339 #if defined(HAS_APPLE_PAC)
340 OSRuntimeSignStructorsInFileset(akc_mh);
341 #endif /* defined(HAS_APPLE_PAC) */
342 }
343
344 /*
345 * boot kernelcache ranges; used for accounting.
346 */
347 SECURITY_READ_ONLY_LATE(const arm_physrange_t *) arm_vm_kernelcache_ranges;
348 SECURITY_READ_ONLY_LATE(int) arm_vm_kernelcache_numranges;
349
350 #if __ARM_KERNEL_PROTECT__
351 /*
352 * If we want to support __ARM_KERNEL_PROTECT__, we need a sufficient amount of
353 * mappable space preceeding the kernel (as we unmap the kernel by cutting the
354 * range covered by TTBR1 in half). This must also cover the exception vectors.
355 */
356 static_assert(KERNEL_PMAP_HEAP_RANGE_START > ARM_KERNEL_PROTECT_EXCEPTION_START);
357
358 /* The exception vectors and the kernel cannot share root TTEs. */
359 static_assert((KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_ROOT_OFFMASK) > ARM_KERNEL_PROTECT_EXCEPTION_START);
360
361 /*
362 * We must have enough space in the TTBR1_EL1 range to create the EL0 mapping of
363 * the exception vectors.
364 */
365 static_assert((((~ARM_KERNEL_PROTECT_EXCEPTION_START) + 1) * 2ULL) <= (ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK));
366 #endif /* __ARM_KERNEL_PROTECT__ */
367
368 #define ARM_DYNAMIC_TABLE_XN (ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN)
369
370 #if KASAN
371 extern vm_offset_t shadow_pbase;
372 extern vm_offset_t shadow_ptop;
373 extern vm_offset_t physmap_vbase;
374 extern vm_offset_t physmap_vtop;
375 #endif
376
377 /*
378 * We explicitly place this in const, as it is not const from a language
379 * perspective, but it is only modified before we actually switch away from
380 * the bootstrap page tables.
381 */
382 SECURITY_READ_ONLY_LATE(uint8_t) bootstrap_pagetables[BOOTSTRAP_TABLE_SIZE] __attribute__((aligned(ARM_PGBYTES)));
383
384 /*
385 * Denotes the end of xnu.
386 */
387 extern void *last_kernel_symbol;
388
389 extern void arm64_replace_bootstack(cpu_data_t*);
390 extern void PE_slide_devicetree(vm_offset_t);
391
392 /*
393 * KASLR parameters
394 */
395 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_base;
396 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_top;
397 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_base;
398 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_top;
399 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_stext;
400 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_etext;
401 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slide;
402 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_base;
403 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_top;
404
405 SECURITY_READ_ONLY_LATE(vm_image_offsets) vm_sptm_offsets;
406 SECURITY_READ_ONLY_LATE(vm_image_offsets) vm_txm_offsets;
407
408 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_stext;
409 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_etext;
410 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sdata;
411 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_edata;
412 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sinfo;
413 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_einfo;
414 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_slinkedit;
415 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_elinkedit;
416
417 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_builtinkmod_text;
418 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_builtinkmod_text_end;
419
420 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernelcache_base;
421 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernelcache_top;
422
423 /* Used by <mach/arm/vm_param.h> */
424 SECURITY_READ_ONLY_LATE(unsigned long) gVirtBase;
425 SECURITY_READ_ONLY_LATE(unsigned long) gPhysBase;
426 SECURITY_READ_ONLY_LATE(unsigned long) gPhysSize;
427 SECURITY_READ_ONLY_LATE(unsigned long) gT0Sz = T0SZ_BOOT;
428 SECURITY_READ_ONLY_LATE(unsigned long) gT1Sz = T1SZ_BOOT;
429
430 /* 23543331 - step 1 of kext / kernel __TEXT and __DATA colocation is to move
431 * all kexts before the kernel. This is only for arm64 devices and looks
432 * something like the following:
433 * -- vmaddr order --
434 * 0xffffff8004004000 __PRELINK_TEXT
435 * 0xffffff8007004000 __TEXT (xnu)
436 * 0xffffff80075ec000 __DATA (xnu)
437 * 0xffffff80076dc000 __KLD (xnu)
438 * 0xffffff80076e0000 __LAST (xnu)
439 * 0xffffff80076e4000 __LINKEDIT (xnu)
440 * 0xffffff80076e4000 __PRELINK_DATA (not used yet)
441 * 0xffffff800782c000 __PRELINK_INFO
442 * 0xffffff80078e4000 -- End of kernelcache
443 */
444
445 /* 24921709 - make XNU ready for KTRR
446 *
447 * Two possible kernel cache layouts, depending on which kcgen is being used.
448 * VAs increasing downwards.
449 * Old KCGEN:
450 *
451 * __PRELINK_TEXT
452 * __TEXT
453 * __DATA_CONST
454 * __TEXT_EXEC
455 * __KLD
456 * __LAST
457 * __DATA
458 * __PRELINK_DATA (expected empty)
459 * __LINKEDIT
460 * __PRELINK_INFO
461 *
462 * New kcgen:
463 *
464 * __PRELINK_TEXT <--- First KTRR (ReadOnly) segment
465 * __PLK_DATA_CONST
466 * __PLK_TEXT_EXEC
467 * __TEXT
468 * __DATA_CONST
469 * __TEXT_EXEC
470 * __KLD
471 * __LAST <--- Last KTRR (ReadOnly) segment
472 * __DATA
473 * __BOOTDATA (if present)
474 * __LINKEDIT
475 * __PRELINK_DATA (expected populated now)
476 * __PLK_LINKEDIT
477 * __PRELINK_INFO
478 *
479 */
480
481 vm_offset_t mem_size; /* Size of actual physical memory present
482 * minus any performance buffer and possibly
483 * limited by mem_limit in bytes */
484 uint64_t mem_actual; /* The "One True" physical memory size
485 * actually, it's the highest physical
486 * address + 1 */
487 uint64_t max_mem; /* Size of physical memory (bytes), adjusted
488 * by maxmem */
489 uint64_t max_mem_actual; /* Actual size of physical memory (bytes),
490 * adjusted by the maxmem boot-arg */
491 uint64_t sane_size; /* Memory size to use for defaults
492 * calculations */
493 /* This no longer appears to be used; kill it? */
494 addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel
495 * virtual address known
496 * to the VM system */
497
498 SECURITY_READ_ONLY_LATE(vm_offset_t) segEXTRADATA;
499 SECURITY_READ_ONLY_LATE(unsigned long) segSizeEXTRADATA;
500
501 /* Trust cache portion of EXTRADATA (if within it) */
502 SECURITY_READ_ONLY_LATE(vm_offset_t) segTRUSTCACHE;
503 SECURITY_READ_ONLY_LATE(unsigned long) segSizeTRUSTCACHE;
504
505 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTTEXT;
506 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWEST;
507 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTRO;
508 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTRO;
509
510 /* Only set when booted from MH_FILESET kernel collections */
511 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTKC;
512 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTKC;
513 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTROKC;
514 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTROKC;
515 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTAuxKC;
516 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTAuxKC;
517 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTROAuxKC;
518 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTROAuxKC;
519 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTRXAuxKC;
520 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTRXAuxKC;
521 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTNLEAuxKC;
522
523 SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTB;
524 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXT;
525
526 SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATACONSTB;
527 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATACONST;
528
529 SECURITY_READ_ONLY_LATE(vm_offset_t) segTEXTEXECB;
530 SECURITY_READ_ONLY_LATE(unsigned long) segSizeTEXTEXEC;
531
532 SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATAB;
533 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATA;
534
535 SECURITY_READ_ONLY_LATE(vm_offset_t) segBOOTDATAB;
536 SECURITY_READ_ONLY_LATE(unsigned long) segSizeBOOTDATA;
537 extern vm_offset_t intstack_low_guard;
538 extern vm_offset_t intstack_high_guard;
539 extern vm_offset_t excepstack_high_guard;
540
541 SECURITY_READ_ONLY_LATE(vm_offset_t) segLINKB;
542 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLINK;
543
544 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDB;
545 SECURITY_READ_ONLY_LATE(unsigned long) segSizeKLD;
546 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDDATAB;
547 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKLDDATA;
548 SECURITY_READ_ONLY_LATE(vm_offset_t) segLASTB;
549 SECURITY_READ_ONLY_LATE(unsigned long) segSizeLAST;
550 SECURITY_READ_ONLY_LATE(vm_offset_t) segLASTDATACONSTB;
551 SECURITY_READ_ONLY_LATE(unsigned long) segSizeLASTDATACONST;
552
553 SECURITY_READ_ONLY_LATE(vm_offset_t) sectHIBTEXTB;
554 SECURITY_READ_ONLY_LATE(unsigned long) sectSizeHIBTEXT;
555 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIBDATAB;
556 SECURITY_READ_ONLY_LATE(unsigned long) segSizeHIBDATA;
557 SECURITY_READ_ONLY_LATE(vm_offset_t) sectHIBDATACONSTB;
558 SECURITY_READ_ONLY_LATE(unsigned long) sectSizeHIBDATACONST;
559
560 SECURITY_READ_ONLY_LATE(vm_offset_t) segPRELINKTEXTB;
561 SECURITY_READ_ONLY_LATE(unsigned long) segSizePRELINKTEXT;
562
563 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKTEXTEXECB;
564 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKTEXTEXEC;
565
566 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKDATACONSTB;
567 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKDATACONST;
568
569 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKDATAB;
570 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKDATA;
571
572 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLLVMCOVB = 0;
573 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLLVMCOV = 0;
574
575 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLINKEDITB;
576 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLINKEDIT;
577
578 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKINFOB;
579 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKINFO;
580
581 /* Only set when booted from MH_FILESET primary kernel collection */
582 SECURITY_READ_ONLY_LATE(vm_offset_t) segKCTEXTEXECB;
583 SECURITY_READ_ONLY_LATE(unsigned long) segSizeKCTEXTEXEC;
584 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKCDATACONSTB;
585 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKCDATACONST;
586 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKCDATAB;
587 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKCDATA;
588
589 SECURITY_READ_ONLY_LATE(static boolean_t) use_contiguous_hint = TRUE;
590
591 SECURITY_READ_ONLY_LATE(int) PAGE_SHIFT_CONST;
592
593 SECURITY_READ_ONLY_LATE(vm_offset_t) end_kern;
594 SECURITY_READ_ONLY_LATE(vm_offset_t) etext;
595 SECURITY_READ_ONLY_LATE(vm_offset_t) sdata;
596 SECURITY_READ_ONLY_LATE(vm_offset_t) edata;
597
598 SECURITY_READ_ONLY_LATE(static vm_offset_t) auxkc_mh, auxkc_base;
599
600 pmap_paddr_t alloc_ptpage(sptm_pt_level_t level, bool map_static);
601 SECURITY_READ_ONLY_LATE(vm_offset_t) ropage_next;
602 extern int dtrace_keep_kernel_symbols(void);
603
604 /*
605 * Bootstrap the system enough to run with virtual memory.
606 * Map the kernel's code and data, and allocate the system page table.
607 * Page_size must already be set.
608 *
609 * Parameters:
610 * first_avail: first available physical page -
611 * after kernel page tables
612 * avail_start: PA of first physical page
613 * avail_end: PA of last physical page
614 */
615 SECURITY_READ_ONLY_LATE(vm_offset_t) first_avail;
616 SECURITY_READ_ONLY_LATE(vm_offset_t) static_memory_end;
617 SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_start;
618 SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_end;
619 SECURITY_READ_ONLY_LATE(pmap_paddr_t) real_avail_end;
620 SECURITY_READ_ONLY_LATE(unsigned long) real_phys_size;
621 SECURITY_READ_ONLY_LATE(vm_map_address_t) physmap_base = (vm_map_address_t)0;
622 SECURITY_READ_ONLY_LATE(vm_map_address_t) physmap_end = (vm_map_address_t)0;
623
624 typedef struct {
625 pmap_paddr_t pa;
626 vm_map_address_t va;
627 vm_size_t len;
628 } ptov_table_entry;
629
630 SECURITY_READ_ONLY_LATE(static boolean_t) kva_active = FALSE;
631
632 #if HAS_ARM_FEAT_SME
633 static SECURITY_READ_ONLY_LATE(bool) enable_sme = true;
634 #endif
635
636 /**
637 * sptm_supports_local_coredump is set in start_sptm.s when SPTM dispatch logic
638 * calls into XNU to handle a panic from SPTM/TXM/cL4. If this variable is set
639 * to false then osfmk/kern/debug.c:debugger_collect_diagnostic() will skip
640 * taking a local core dump. This defaults to true since as long as the panic
641 * doesn't occur within the SPTM, then the SPTM will support making calls during
642 * the panic path to save the coredump. Only when the panic occurs from within
643 * guarded mode do we let SPTM decide whether it supports local coredumps.
644 */
645 bool sptm_supports_local_coredump = true;
646
647 #if KASAN
648 /* Prototypes for KASAN functions */
649 void kasan_bootstrap(boot_args *, vm_offset_t pgtable, sptm_bootstrap_args_xnu_t *sptm_boot_args);
650
651 /**
652 * Entry point for systems that support an SPTM and are booting a KASAN kernel.
653 * This is required because KASAN kernels need to set up the shadow map before
654 * arm_init() can even run.
655 */
656 void
arm_init_kasan(boot_args * args,sptm_bootstrap_args_xnu_t * sptm_boot_args)657 arm_init_kasan(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_boot_args)
658 {
659 /* Initialize SPTM helper library. */
660 uint8_t ret = libsptm_init(&sptm_boot_args->libsptm_state);
661 if (ret != LIBSPTM_SUCCESS) {
662 panic("%s: libsptm_init failed: %u", __func__, ret);
663 }
664
665 memSize = args->memSize;
666 kasan_bootstrap(args, phystokv(sptm_boot_args->libsptm_state.root_table_paddr), sptm_boot_args);
667
668 arm_init(args, sptm_boot_args);
669 }
670 #endif /* KASAN */
671
672 /**
673 * Entry point for systems that support an SPTM - except on KASAN kernels,
674 * see above. Bootstrap stacks have been set up by the SPTM by this point,
675 * and XNU is responsible for rebasing and signing absolute addresses.
676 */
677 void
arm_init(boot_args * args,sptm_bootstrap_args_xnu_t * sptm_boot_args)678 arm_init(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_boot_args)
679 {
680 unsigned int maxmem;
681 uint32_t memsize;
682 uint64_t xmaxmem;
683 thread_t thread;
684 DTEntry chosen;
685 unsigned int dt_entry_size;
686
687 extern void xnu_return_to_gl2(void);
688 const sptm_vaddr_t handler_addr = (sptm_vaddr_t) ptrauth_strip((void *)xnu_return_to_gl2, ptrauth_key_function_pointer);
689 sptm_register_xnu_exc_return(handler_addr);
690
691 #if defined(HAS_APPLE_PAC)
692 kernel_mach_header_t *kc_mh = PE_get_kc_header(KCKindPrimary);
693 OSRuntimeSignStructorsInFileset(kc_mh);
694 #endif /* defined(HAS_APPLE_PAC) */
695
696 /* If kernel integrity is supported, use a constant copy of the boot args. */
697 const_boot_args = *args;
698 BootArgs = args = &const_boot_args;
699 const_sptm_args = *sptm_boot_args;
700 SPTMArgs = sptm_boot_args = &const_sptm_args;
701 sptm_xnu_triggered_panic_ptr = sptm_boot_args->xnu_triggered_panic;
702 /*
703 * Initialize first_avail_phys from what the SPTM tells us.
704 * We're not using iBoot's topOfKernelData, as SPTM and other
705 * components have consumed pages themselves.
706 */
707 first_avail_phys = sptm_boot_args->first_avail_phys;
708
709 #if APPLEVIRTUALPLATFORM
710 reset_vector_vaddr = (vm_offset_t) sptm_boot_args->sptm_reset_vector_vaddr;
711 #endif /* APPLEVIRTUALPLATFORM */
712
713 cpu_data_init(&BootCpuData);
714 #if defined(HAS_APPLE_PAC)
715 /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
716 BootCpuData.rop_key = ml_default_rop_pid();
717 BootCpuData.jop_key = ml_default_jop_pid();
718 #endif /* defined(HAS_APPLE_PAC) */
719
720 PE_init_platform(FALSE, args); /* Get platform expert set up */
721
722 #if !KASAN
723 memSize = args->memSize;
724
725 /* Initialize SPTM helper library. */
726 uint8_t ret = libsptm_init(&const_sptm_args.libsptm_state);
727 if (ret != LIBSPTM_SUCCESS) {
728 panic("%s: libsptm_init failed: %u", __func__, ret);
729 }
730 #endif
731
732 #if __arm64__
733 configure_timer_apple_regs();
734 wfe_timeout_configure();
735 wfe_timeout_init();
736
737 configure_misc_apple_boot_args();
738 configure_misc_apple_regs(true);
739
740 #if HAS_UPSI_FAILURE_INJECTION
741 /* UPSI (Universal Panic and Stall Injection) Logic
742 * iBoot/XNU are both configured for failure injection at specific stages
743 * The injected failure and stage is populated through EDT properties by iBoot
744 *
745 * iBoot populates the EDT properties for XNU based upon PMU scratch bits
746 * This is done because the EDT is available sooner in XNU than the PMU Kext
747 */
748 uint64_t const *upsi_info = NULL;
749
750 /* Not usable TUNABLE here because TUNABLEs are parsed at a later point. */
751 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
752 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
753 }
754
755 /* Check if there is a requested injection stage */
756 if (SecureDTGetProperty(chosen, "injection_stage", (void const **)&upsi_info,
757 &dt_entry_size) == kSuccess) {
758 assert3u(dt_entry_size, ==, 8);
759 xnu_upsi_injection_stage = *upsi_info;
760 }
761
762 /* Check if there is a requested injection action */
763 if (SecureDTGetProperty(chosen, "injection_action", (void const **)&upsi_info,
764 &dt_entry_size) == kSuccess) {
765 assert3u(dt_entry_size, ==, 8);
766 xnu_upsi_injection_action = *upsi_info;
767 }
768
769 check_for_failure_injection(XNU_STAGE_ARM_INIT);
770
771 chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
772 dt_entry_size = 0;
773 #endif // HAS_UPSI_FAILURE_INJECTION
774
775 #if HAS_ARM_FEAT_SME
776 (void)PE_parse_boot_argn("enable_sme", &enable_sme, sizeof(enable_sme));
777 if (enable_sme) {
778 arm_sme_init(true);
779 }
780 #endif
781
782
783 {
784 /*
785 * Select the advertised kernel page size.
786 */
787 if (memSize > 1ULL * 1024 * 1024 * 1024) {
788 /*
789 * arm64 device with > 1GB of RAM:
790 * kernel uses 16KB pages.
791 */
792 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
793 } else {
794 /*
795 * arm64 device with <= 1GB of RAM:
796 * kernel uses hardware page size
797 * (4KB for H6/H7, 16KB for H8+).
798 */
799 PAGE_SHIFT_CONST = ARM_PGSHIFT;
800 }
801
802 /* 32-bit apps always see 16KB page size */
803 page_shift_user32 = PAGE_MAX_SHIFT;
804 #ifdef APPLETYPHOON
805 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
806 if ((cpus_defeatures & 0xF) != 0) {
807 cpu_defeatures_set(cpus_defeatures & 0xF);
808 }
809 }
810 #endif
811 }
812 #endif
813
814 /* Enable SPTM stability hacks if requested */
815 PE_parse_boot_argn("sptm_stability_hacks", &sptm_stability_hacks, sizeof(sptm_stability_hacks));
816
817 ml_parse_cpu_topology();
818
819 siq_init();
820
821 master_cpu = ml_get_boot_cpu_number();
822 assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
823
824 BootCpuData.cpu_number = (unsigned short)master_cpu;
825 BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
826 BootCpuData.istackptr = &intstack_top;
827 BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
828 BootCpuData.excepstackptr = &excepstack_top;
829 CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
830 CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
831 + ((uintptr_t)&BootCpuData
832 - (uintptr_t)(args->virtBase)));
833
834 thread = thread_bootstrap();
835 thread->machine.CpuDatap = &BootCpuData;
836 thread->machine.pcpu_data_base_and_cpu_number =
837 ml_make_pcpu_base_and_cpu_number(0, BootCpuData.cpu_number);
838 machine_set_current_thread(thread);
839
840 /*
841 * Preemption is enabled for this thread so that it can lock mutexes without
842 * tripping the preemption check. In reality scheduling is not enabled until
843 * this thread completes, and there are no other threads to switch to, so
844 * preemption level is not really meaningful for the bootstrap thread.
845 */
846 thread->machine.preemption_count = 0;
847 cpu_bootstrap();
848
849 rtclock_early_init();
850
851 kernel_debug_string_early("kernel_startup_bootstrap");
852 kernel_startup_bootstrap();
853
854 /*
855 * Initialize the timer callout world
856 */
857 timer_call_init();
858
859 cpu_init();
860
861 processor_bootstrap();
862
863 if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
864 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
865 } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
866 xmaxmem = (uint64_t) memsize;
867 } else {
868 xmaxmem = 0;
869 }
870
871 #if SCHED_HYGIENE_DEBUG
872 {
873 int wdt_boot_arg = 0;
874 bool const wdt_disabled = (PE_parse_boot_argn("wdt", &wdt_boot_arg, sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1));
875
876 /* Disable if WDT is disabled */
877 if (wdt_disabled || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
878 interrupt_masked_debug_mode = SCHED_HYGIENE_MODE_OFF;
879 }
880 if (wdt_disabled || kern_feature_override(KF_PREEMPTION_DISABLED_DEBUG_OVRD)) {
881 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
882 }
883 }
884 #endif /* SCHED_HYGIENE_DEBUG */
885
886 nanoseconds_to_absolutetime(XCALL_ACK_TIMEOUT_NS, &xcall_ack_timeout_abstime);
887
888 #if HAS_BP_RET
889 PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
890 set_bp_ret(); // Apply branch predictor retention settings to boot CPU
891 #endif
892
893 PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
894
895 #if __ARM_PAN_AVAILABLE__
896 __builtin_arm_wsr("pan", 1);
897 #endif /* __ARM_PAN_AVAILABLE__ */
898
899
900 /**
901 * Check SPTM feature flag for ARM_LARGE_MEMORY irrespective of XNU
902 * definition to detect mismatch in cases where ARM_LARGE_MEMORY is
903 * defined in SPTM but not in XNU and vice versa.
904 */
905 const uint64_t sptm_is_large_memory = SPTMArgs->feature_flags & SPTM_FEATURE_LARGE_MEMORY;
906 const uint64_t sptm_is_large_memory_kernonly = SPTMArgs->feature_flags & SPTM_FEATURE_LARGE_MEMORY_KERNONLY;
907 #if ARM_LARGE_MEMORY
908 const uint64_t xnu_is_large_memory = SPTM_FEATURE_LARGE_MEMORY;
909 #if ARM_LARGE_MEMORY_KERNONLY
910 const uint64_t xnu_is_large_memory_kernonly = SPTM_FEATURE_LARGE_MEMORY_KERNONLY;
911 #else /* ARM_LARGE_MEMORY_KERNONLY */
912 const uint64_t xnu_is_large_memory_kernonly = 0;
913 #endif /* ARM_LARGE_MEMORY_KERNONLY */
914 #else /* ARM_LARGE_MEMORY */
915 const uint64_t xnu_is_large_memory = 0;
916 const uint64_t xnu_is_large_memory_kernonly = 0;
917 #endif /* ARM_LARGE_MEMORY */
918
919 if (sptm_is_large_memory != xnu_is_large_memory) {
920 panic("Mismatch of ARM_LARGE_MEMORY in SPTM (%#llx)/XNU (%#llx)", sptm_is_large_memory, xnu_is_large_memory);
921 }
922
923 if (sptm_is_large_memory_kernonly != xnu_is_large_memory_kernonly) {
924 panic("Mismatch of ARM_LARGE_MEMORY_KERNONLY in SPTM (%#llx)/XNU (%#llx)",
925 sptm_is_large_memory_kernonly, xnu_is_large_memory_kernonly);
926 }
927
928 /*
929 * gPhysBase/Size only represent kernel-managed memory. These globals represent
930 * the actual DRAM base address and size as reported by iBoot through the
931 * device tree.
932 */
933 unsigned long const *dram_base;
934 unsigned long const *dram_size;
935 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
936 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
937 }
938
939 if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
940 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
941 }
942
943 if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
944 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
945 }
946
947 gDramBase = *dram_base;
948 gDramSize = *dram_size;
949 pmap_first_pnum = (ppnum_t)atop(gDramBase);
950
951 arm_vm_init(xmaxmem, args);
952
953 if (debug_boot_arg) {
954 patch_low_glo();
955 }
956
957 #if __arm64__ && WITH_CLASSIC_S2R
958 sleep_token_buffer_init();
959 #endif
960
961 PE_consistent_debug_inherit();
962
963 /* Setup debugging output. */
964 const unsigned int serial_exists = serial_init();
965 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
966 kprintf("kprintf initialized\n");
967
968 serialmode = 0;
969 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
970 /* Do we want a serial keyboard and/or console? */
971 kprintf("Serial mode specified: %08X\n", serialmode);
972 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
973 enable_dklog_serial_output = restore_boot || (serialmode & SERIALMODE_DKLOG) != 0;
974 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
975 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
976 if (force_sync) {
977 serialmode |= SERIALMODE_SYNCDRAIN;
978 kprintf(
979 "WARNING: Forcing uart driver to output synchronously."
980 "printf()s/IOLogs will impact kernel performance.\n"
981 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
982 }
983 }
984 }
985 if (kern_feature_override(KF_SERIAL_OVRD)) {
986 serialmode = 0;
987 }
988
989 /* Start serial if requested and a serial device was enumerated in serial_init(). */
990 if ((serialmode & SERIALMODE_OUTPUT) && serial_exists) {
991 serial_console_enabled = true;
992 (void)switch_to_serial_console(); /* Switch into serial mode from video console */
993 disableConsoleOutput = FALSE; /* Allow printfs to happen */
994 }
995 PE_create_console();
996
997 /* setup console output */
998 PE_init_printf(FALSE);
999
1000 #if __arm64__
1001 #if DEBUG
1002 dump_kva_space();
1003 #endif
1004 #endif
1005
1006 cpu_machine_idle_init(TRUE);
1007
1008 PE_init_platform(TRUE, &BootCpuData);
1009
1010 /* Initialize the debug infrastructure system-wide and on the local core. */
1011 pe_arm_debug_init_early(&BootCpuData);
1012
1013 #if RELEASE
1014 /* Validate SPTM variant. */
1015 if (const_sptm_args.sptm_variant != SPTM_VARIANT_RELEASE) {
1016 panic("arm_init: Development SPTM / Release XNU is not a supported configuration.");
1017 }
1018 #endif /* RELEASE */
1019
1020 #if __arm64__
1021 extern bool cpu_config_correct;
1022 if (!cpu_config_correct) {
1023 panic("The cpumask=N boot arg cannot be used together with cpus=N, and the boot CPU must be enabled");
1024 }
1025
1026 ml_map_cpu_pio();
1027 #endif
1028
1029 cpu_timebase_init(TRUE);
1030
1031 #if KPERF
1032 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
1033 kptimer_curcpu_up();
1034 #endif /* KPERF */
1035
1036 PE_init_cpu();
1037 #if __arm64__
1038 apt_msg_init();
1039 apt_msg_init_cpu();
1040 #endif
1041 fiq_context_init(TRUE);
1042
1043
1044 #if HIBERNATION
1045 pal_hib_init();
1046 #endif /* HIBERNATION */
1047
1048 /*
1049 * Initialize the stack protector for all future calls
1050 * to C code. Since kernel_bootstrap() eventually
1051 * switches stack context without returning through this
1052 * function, we do not risk failing the check even though
1053 * we mutate the guard word during execution.
1054 */
1055 __stack_chk_guard = (unsigned long)early_random();
1056 /* Zero a byte of the protector to guard
1057 * against string vulnerabilities
1058 */
1059 __stack_chk_guard &= ~(0xFFULL << 8);
1060 machine_startup(args);
1061 }
1062
1063 /*
1064 * Routine: arm_init_cpu
1065 * Function:
1066 * Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
1067 */
1068
1069 void
arm_init_cpu(cpu_data_t * cpu_data_ptr,__unused uint64_t hibernation_args)1070 arm_init_cpu(
1071 cpu_data_t *cpu_data_ptr,
1072 __unused uint64_t hibernation_args)
1073 {
1074 #if HIBERNATION
1075 sptm_hibernation_args_xnu_t *hibargs = (sptm_hibernation_args_xnu_t *)hibernation_args;
1076
1077 if ((hibargs != 0) && (hibargs->hib_header_phys != 0) && (hibargs->handoff_page_count > 0)) {
1078 /*
1079 * We must copy the handoff region before anything else because the physical pages
1080 * holding the handoff region are not tracked by xnu as in-use.
1081 */
1082 HibernationCopyHandoffRegionFromPageArray(&hibargs->handoff_pages[0], hibargs->handoff_page_count);
1083 }
1084 #endif /* HIBERNATION */
1085
1086 #if __ARM_PAN_AVAILABLE__
1087 __builtin_arm_wsr("pan", 1);
1088 #endif
1089
1090 #ifdef __arm64__
1091 configure_timer_apple_regs();
1092 configure_misc_apple_regs(false);
1093 #endif
1094 #if HAS_ARM_FEAT_SME
1095 if (enable_sme) {
1096 arm_sme_init(false);
1097 }
1098 #endif
1099
1100 os_atomic_andnot(&cpu_data_ptr->cpu_flags, SleepState, relaxed);
1101
1102 siq_cpu_init();
1103
1104 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
1105
1106
1107 #if HIBERNATION
1108 if (hibargs != 0 && hibargs->hib_header_phys != 0) {
1109 gIOHibernateState = kIOHibernateStateWakingFromHibernate;
1110 uart_hibernation = true;
1111
1112
1113 __nosan_memcpy(gIOHibernateCurrentHeader, (void*)phystokv(hibargs->hib_header_phys), sizeof(IOHibernateImageHeader));
1114
1115 }
1116 if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate) && ml_is_quiescing()) {
1117 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
1118 extern uint64_t wake_abstime;
1119 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
1120
1121 // since the hw clock stops ticking across hibernation, we need to apply an offset;
1122 // iBoot computes this offset for us and passes it via the hibernation header
1123 extern uint64_t hwclock_conttime_offset;
1124 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
1125
1126 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
1127 // fix up its preemption count
1128 unsigned int expected_preemption_count = (gEnforcePlatformActionSafety ? 2 : 1);
1129 if (get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread) !=
1130 expected_preemption_count) {
1131 panic("unexpected preemption count %u on boot cpu thread (should be %u)",
1132 get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread),
1133 expected_preemption_count);
1134 }
1135 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
1136 }
1137 #endif /* HIBERNATION */
1138
1139 #if __arm64__
1140 wfe_timeout_init();
1141 flush_mmu_tlb();
1142 #endif
1143
1144 cpu_machine_idle_init(FALSE);
1145
1146 cpu_init();
1147
1148 #ifdef APPLETYPHOON
1149 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
1150 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
1151 }
1152 #endif
1153 /* Initialize the timebase before serial_init, as some serial
1154 * drivers use mach_absolute_time() to implement rate control
1155 */
1156 cpu_timebase_init(FALSE);
1157
1158 #if KPERF
1159 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
1160 kptimer_curcpu_up();
1161 #endif /* KPERF */
1162
1163 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
1164 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
1165 /*
1166 * Prevent CPUs from going into deep sleep until all
1167 * CPUs are ready to do so.
1168 */
1169 arm64_stall_sleep = TRUE;
1170 #endif
1171 serial_init();
1172 PE_init_platform(TRUE, NULL);
1173 commpage_update_timebase();
1174
1175 exclaves_update_timebase(EXCLAVES_CLOCK_ABSOLUTE,
1176 rtclock_base_abstime);
1177 #if HIBERNATION
1178 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
1179 exclaves_update_timebase(EXCLAVES_CLOCK_CONTINUOUS,
1180 hwclock_conttime_offset);
1181 }
1182 #endif /* HIBERNATION */
1183 }
1184 PE_init_cpu();
1185 #if __arm64__
1186 apt_msg_init_cpu();
1187 #endif
1188
1189 fiq_context_init(TRUE);
1190 cpu_data_ptr->rtcPop = EndOfAllTime;
1191 timer_resync_deadlines();
1192
1193 /* Start tracing (secondary CPU). */
1194 #if DEVELOPMENT || DEBUG
1195 PE_arm_debug_enable_trace(true);
1196 #endif /* DEVELOPMENT || DEBUG */
1197
1198 kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
1199
1200 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
1201 if (kdebug_enable == 0) {
1202 __kdebug_only uint64_t elapsed = kdebug_wake();
1203 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
1204 }
1205
1206 #if CONFIG_TELEMETRY
1207 bootprofile_wake_from_sleep();
1208 #endif /* CONFIG_TELEMETRY */
1209 }
1210 #if CONFIG_CPU_COUNTERS
1211 mt_wake_per_core();
1212 #endif /* CONFIG_CPU_COUNTERS */
1213
1214 #if defined(KERNEL_INTEGRITY_CTRR)
1215 if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
1216 lck_spin_lock(&ctrr_cpu_start_lck);
1217 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
1218 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
1219 lck_spin_unlock(&ctrr_cpu_start_lck);
1220 }
1221 #endif
1222
1223
1224 secondary_cpu_main(NULL);
1225 }
1226
1227 /*
1228 * Routine: arm_init_idle_cpu
1229 * Function: Resume from non-retention WFI. Called from the reset vector.
1230 */
1231 void __attribute__((noreturn))
arm_init_idle_cpu(cpu_data_t * cpu_data_ptr)1232 arm_init_idle_cpu(
1233 cpu_data_t *cpu_data_ptr)
1234 {
1235 #if __ARM_PAN_AVAILABLE__
1236 __builtin_arm_wsr("pan", 1);
1237 #endif
1238
1239 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
1240
1241 #if __arm64__
1242 wfe_timeout_init();
1243 #endif
1244
1245 #ifdef APPLETYPHOON
1246 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
1247 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
1248 }
1249 #endif
1250
1251 /*
1252 * Update the active debug object to reflect that debug registers have been reset.
1253 * This will force any thread with active debug state to resync the debug registers
1254 * if it returns to userspace on this CPU.
1255 */
1256 if (cpu_data_ptr->cpu_user_debug != NULL) {
1257 arm_debug_set(NULL);
1258 }
1259
1260 fiq_context_init(FALSE);
1261
1262 cpu_idle_exit(TRUE);
1263 }
1264
1265 vm_map_address_t
phystokv(pmap_paddr_t pa)1266 phystokv(pmap_paddr_t pa)
1267 {
1268 sptm_papt_t va;
1269 if (sptm_phystokv(pa, &va) != LIBSPTM_SUCCESS) {
1270 return 0;
1271 }
1272 return (vm_map_address_t)va;
1273 }
1274
1275 vm_map_address_t
phystokv_range(pmap_paddr_t pa,vm_size_t * max_len)1276 phystokv_range(pmap_paddr_t pa, vm_size_t *max_len)
1277 {
1278
1279 vm_size_t len;
1280
1281 len = PAGE_SIZE - (pa & PAGE_MASK);
1282 if (*max_len > len) {
1283 *max_len = len;
1284 }
1285
1286 return phystokv((sptm_paddr_t)pa);
1287 }
1288
1289 vm_offset_t
ml_static_vtop(vm_offset_t va)1290 ml_static_vtop(vm_offset_t va)
1291 {
1292 return (vm_offset_t)kvtophys_nofail((sptm_papt_t)va);
1293 }
1294
1295 #define ARM64_GRANULE_ALLOW_BLOCK (1 << 0)
1296 #define ARM64_GRANULE_ALLOW_HINT (1 << 1)
1297
1298 // Populate seg...AuxKC and fixup AuxKC permissions
1299 static bool
arm_vm_auxkc_init(void)1300 arm_vm_auxkc_init(void)
1301 {
1302 if (auxkc_mh == 0 || auxkc_base == 0) {
1303 return false; // no auxKC.
1304 }
1305
1306 /* Fixup AuxKC and populate seg*AuxKC globals used below */
1307 arm_auxkc_init((void*)auxkc_mh, (void*)auxkc_base);
1308
1309 /*
1310 * The AuxKC LINKEDIT segment needs to be covered by the RO region but is excluded
1311 * from the RO address range returned by kernel_collection_adjust_mh_addrs().
1312 * Ensure the highest non-LINKEDIT address in the AuxKC is the current end of
1313 * its RO region before extending it.
1314 */
1315 assert(segHIGHESTROAuxKC == segHIGHESTNLEAuxKC);
1316 assert(segHIGHESTAuxKC >= segHIGHESTROAuxKC);
1317 if (segHIGHESTAuxKC > segHIGHESTROAuxKC) {
1318 segHIGHESTROAuxKC = segHIGHESTAuxKC;
1319 }
1320
1321 /*
1322 * The AuxKC RO region must be right below the device tree/trustcache so that it can be covered
1323 * by CTRR, and the AuxKC RX region must be within the RO region.
1324 */
1325 assert(segHIGHESTRXAuxKC <= segHIGHESTROAuxKC);
1326 assert(segLOWESTRXAuxKC <= segHIGHESTRXAuxKC);
1327 assert(segLOWESTROAuxKC <= segLOWESTRXAuxKC);
1328 assert(segLOWESTAuxKC <= segLOWESTROAuxKC);
1329
1330 return true;
1331 }
1332
1333 /*
1334 * Looks up the set of properties that describe the physical load addresses and sizes of the boot
1335 * kernelcache's loaded segments in the device tree and returns (1) the number of segments found
1336 * in *arm_vm_kernelcache_numrangesp and (2) their starting/ending addresses as an array of type
1337 * arm_physrange_t in *arm_vm_kernelcache_rangesp.
1338 * The function returns the total number of pages across all loaded boot kernelcache segments.
1339 * If there is a problem looking up the /chosen/memory-map node in the DT, all arguments are
1340 * zeroed and the function returns 0.
1341 */
1342 static unsigned int
arm_get_bootkc_ranges_from_DT(const arm_physrange_t ** arm_vm_kernelcache_rangesp,int * arm_vm_kernelcache_numrangesp)1343 arm_get_bootkc_ranges_from_DT(const arm_physrange_t **arm_vm_kernelcache_rangesp, int *arm_vm_kernelcache_numrangesp)
1344 {
1345 DTEntry memory_map;
1346 int err;
1347 DTMemoryMapRange const *range;
1348 unsigned int rangeSize;
1349 #define NUM_BOOTKC_RANGES 5
1350 static arm_physrange_t bootkc_physranges[NUM_BOOTKC_RANGES] = { {0, } };
1351 static int bootkc_numranges = 0;
1352 static unsigned int bootkc_total_pages = 0;
1353
1354 assert(arm_vm_kernelcache_rangesp != NULL);
1355 assert(arm_vm_kernelcache_numrangesp != NULL);
1356
1357 /* return cached values if previously computed */
1358 if (bootkc_numranges == 0) {
1359 err = SecureDTLookupEntry(NULL, "chosen/memory-map", &memory_map);
1360 if (err != kSuccess) {
1361 *arm_vm_kernelcache_numrangesp = 0;
1362 *arm_vm_kernelcache_rangesp = NULL;
1363 return 0;
1364 }
1365
1366 /* We're looking for 5 ranges: BootKC-ro, BootKC-rx, BootKC-bx, BootKC-rw, and BootKC-le */
1367 const char *BootKC_Properties[NUM_BOOTKC_RANGES] = {
1368 "BootKC-ro", "BootKC-rx", "BootKC-bx", "BootKC-rw", "BootKC-le"
1369 };
1370
1371 for (int i = 0; i < NUM_BOOTKC_RANGES; i++) {
1372 err = SecureDTGetProperty(memory_map, BootKC_Properties[i], (void const **)&range, &rangeSize);
1373 if (err == kSuccess && rangeSize == sizeof(DTMemoryMapRange)) {
1374 bootkc_physranges[i].start_phys = range->paddr;
1375 bootkc_physranges[i].end_phys = range->paddr + range->length;
1376 assert((bootkc_physranges[i].end_phys & PAGE_MASK) == 0);
1377 bootkc_numranges++;
1378 bootkc_total_pages += (unsigned int) atop_64(bootkc_physranges[i].end_phys - bootkc_physranges[i].start_phys);
1379 }
1380 }
1381 }
1382
1383 *arm_vm_kernelcache_numrangesp = bootkc_numranges;
1384 *arm_vm_kernelcache_rangesp = &bootkc_physranges[0];
1385 return bootkc_total_pages;
1386 }
1387
1388 void
arm_vm_prot_init(__unused boot_args * args)1389 arm_vm_prot_init(__unused boot_args * args)
1390 {
1391 segLOWESTTEXT = UINT64_MAX;
1392 if (segSizePRELINKTEXT && (segPRELINKTEXTB < segLOWESTTEXT)) {
1393 segLOWESTTEXT = segPRELINKTEXTB;
1394 }
1395 assert(segSizeTEXT);
1396 if (segTEXTB < segLOWESTTEXT) {
1397 segLOWESTTEXT = segTEXTB;
1398 }
1399 assert(segLOWESTTEXT < UINT64_MAX);
1400
1401 segEXTRADATA = 0;
1402 segSizeEXTRADATA = 0;
1403 segTRUSTCACHE = 0;
1404 segSizeTRUSTCACHE = 0;
1405
1406 segLOWEST = segLOWESTTEXT;
1407 segLOWESTRO = segLOWESTTEXT;
1408
1409 if (segLOWESTKC && segLOWESTKC < segLOWEST) {
1410 /*
1411 * kernel collections have segments below the kernel. In particular the collection mach header
1412 * is below PRELINK_TEXT and is not covered by any other segments already tracked.
1413 */
1414 segLOWEST = segLOWESTKC;
1415 if (segLOWESTROKC && segLOWESTROKC < segLOWESTRO) {
1416 segLOWESTRO = segLOWESTROKC;
1417 }
1418 if (segHIGHESTROKC && segHIGHESTROKC > segHIGHESTRO) {
1419 segHIGHESTRO = segHIGHESTROKC;
1420 }
1421 }
1422
1423 DTEntry memory_map;
1424 int err;
1425
1426 // Device Tree portion of EXTRADATA
1427 if (SecureDTIsLockedDown()) {
1428 segEXTRADATA = (vm_offset_t)PE_state.deviceTreeHead;
1429 segSizeEXTRADATA = PE_state.deviceTreeSize;
1430 }
1431
1432 // Trust Caches portion of EXTRADATA
1433 {
1434 DTMemoryMapRange const *trustCacheRange;
1435 unsigned int trustCacheRangeSize;
1436
1437 err = SecureDTLookupEntry(NULL, "chosen/memory-map", &memory_map);
1438 assert(err == kSuccess);
1439
1440 err = SecureDTGetProperty(memory_map, "TrustCache", (void const **)&trustCacheRange, &trustCacheRangeSize);
1441 if (err == kSuccess) {
1442 if (trustCacheRangeSize != sizeof(DTMemoryMapRange)) {
1443 panic("Unexpected /chosen/memory-map/TrustCache property size %u != %zu", trustCacheRangeSize, sizeof(DTMemoryMapRange));
1444 }
1445
1446 vm_offset_t const trustCacheRegion = phystokv(trustCacheRange->paddr);
1447 if (trustCacheRegion < segLOWEST) {
1448 if (segEXTRADATA != 0) {
1449 if (trustCacheRegion != segEXTRADATA + segSizeEXTRADATA) {
1450 panic("Unexpected location of TrustCache region: %#lx != %#lx",
1451 trustCacheRegion, segEXTRADATA + segSizeEXTRADATA);
1452 }
1453 segSizeEXTRADATA += trustCacheRange->length;
1454 } else {
1455 // Not all devices support CTRR device trees.
1456 segEXTRADATA = trustCacheRegion;
1457 segSizeEXTRADATA = trustCacheRange->length;
1458 }
1459 }
1460 segTRUSTCACHE = trustCacheRegion;
1461 segSizeTRUSTCACHE = trustCacheRange->length;
1462 }
1463 }
1464
1465 if (segSizeEXTRADATA != 0) {
1466 if (segEXTRADATA <= segLOWEST) {
1467 segLOWEST = segEXTRADATA;
1468 if (segEXTRADATA <= segLOWESTRO) {
1469 segLOWESTRO = segEXTRADATA;
1470 }
1471 } else {
1472 panic("EXTRADATA is in an unexpected place: %#lx > %#lx", segEXTRADATA, segLOWEST);
1473 }
1474 }
1475
1476 /* Record the bounds of the kernelcache. */
1477 vm_kernelcache_base = segLOWEST;
1478
1479 auxkc_mh = SPTMArgs->auxkc_mh;
1480 auxkc_base = SPTMArgs->auxkc_base;
1481 end_kern = SPTMArgs->auxkc_end;
1482
1483 vm_kernelcache_top = end_kern;
1484 }
1485
1486 static void
arm_vm_slide_region(vm_offset_t phys_start,size_t size)1487 arm_vm_slide_region(vm_offset_t phys_start, size_t size)
1488 {
1489 sptm_slide_region(phys_start, (unsigned int)(size >> PAGE_SHIFT));
1490 }
1491
1492 /*
1493 * return < 0 for a < b
1494 * 0 for a == b
1495 * > 0 for a > b
1496 */
1497 typedef int (*cmpfunc_t)(const void *a, const void *b);
1498
1499 extern void
1500 qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
1501
1502 SECURITY_READ_ONLY_LATE(static unsigned int) ptov_index = 0;
1503
1504 #define ROUND_L1(addr) (((addr) + ARM_TT_L1_OFFMASK) & ~(ARM_TT_L1_OFFMASK))
1505 #define ROUND_TWIG(addr) (((addr) + ARM_TT_TWIG_OFFMASK) & ~(ARM_TT_TWIG_OFFMASK))
1506
1507 void
arm_vm_prot_finalize(boot_args * args __unused)1508 arm_vm_prot_finalize(boot_args * args __unused)
1509 {
1510 /*
1511 * At this point, we are far enough along in the boot process that it will be
1512 * safe to free up all of the memory preceeding the kernel. It may in fact
1513 * be safe to do this earlier.
1514 *
1515 * This keeps the memory in the V-to-P mapping, but advertises it to the VM
1516 * as usable.
1517 */
1518
1519 /* Slide KLDDATA */
1520 arm_vm_slide_region(segKLDDATAB, segSizeKLDDATA);
1521
1522 /*
1523 * Replace the boot CPU's stacks with properly-guarded dynamically allocated stacks.
1524 * This must happen prior to sliding segBOOTDATAB, which will effectively remove
1525 * the existing boot stacks.
1526 */
1527 cpu_stack_alloc(&BootCpuData);
1528 arm64_replace_bootstack(&BootCpuData);
1529
1530 /* Slide early-boot data */
1531 arm_vm_slide_region(segBOOTDATAB, segSizeBOOTDATA);
1532
1533 /* Slide linkedit, unless otherwise requested */
1534 bool keep_linkedit = false;
1535 PE_parse_boot_argn("keepsyms", &keep_linkedit, sizeof(keep_linkedit));
1536 #if CONFIG_DTRACE
1537 if (dtrace_keep_kernel_symbols()) {
1538 keep_linkedit = true;
1539 }
1540 #endif /* CONFIG_DTRACE */
1541 #if KASAN_DYNAMIC_DENYLIST
1542 /* KASAN's dynamic denylist needs to query the LINKEDIT segment at runtime. As such, the
1543 * kext bootstrap code will not jettison LINKEDIT on kasan kernels, so don't bother to relocate it. */
1544 keep_linkedit = true;
1545 #endif /* KASAN_DYNAMIC_DENYLIST */
1546
1547 if (!keep_linkedit) {
1548 arm_vm_slide_region(segLINKB, segSizeLINK);
1549 if (segSizePLKLINKEDIT) {
1550 /* Prelinked kernel LINKEDIT */
1551 arm_vm_slide_region(segPLKLINKEDITB, segSizePLKLINKEDIT);
1552 }
1553 }
1554
1555 /* Slide prelinked kernel plists */
1556 arm_vm_slide_region(segPRELINKINFOB, segSizePRELINKINFO);
1557
1558 /*
1559 * Free the portion of memory that precedes the first usable region, known
1560 * as the physical slide.
1561 */
1562 ml_static_mfree(SPTMArgs->phys_slide_papt, SPTMArgs->phys_slide_size);
1563
1564 /*
1565 * KTRR support means we will be mucking with these pages and trying to
1566 * protect them; we cannot free the pages to the VM if we do this.
1567 */
1568 if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC && segSizePRELINKTEXT) {
1569 /* If new segments not present, PRELINK_TEXT is not dynamically sized, free DRAM between it and xnu TEXT */
1570 ml_static_mfree(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT));
1571 }
1572
1573 ml_static_mfree(segBOOTDATAB, segSizeBOOTDATA);
1574
1575 #if __ARM_KERNEL_PROTECT__
1576 arm_vm_populate_kernel_el0_mappings();
1577 #endif /* __ARM_KERNEL_PROTECT__ */
1578 }
1579
1580 /* allocate a page for a page table: we support static and dynamic mappings.
1581 *
1582 * returns a physical address for the allocated page
1583 *
1584 * for static mappings, we allocate from the region ropagetable_begin to ro_pagetable_end-1,
1585 * which is defined in the DATA_CONST segment and will be protected RNX when vm_prot_finalize runs.
1586 *
1587 * for dynamic mappings, we allocate from avail_start, which should remain RWNX.
1588 */
1589 pmap_paddr_t
alloc_ptpage(sptm_pt_level_t level,bool map_static)1590 alloc_ptpage(sptm_pt_level_t level, bool map_static)
1591 {
1592 pmap_paddr_t paddr = 0;
1593
1594 #if !(defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR))
1595 map_static = FALSE;
1596 #endif
1597
1598 /* Set the next free ropage if this is the first call to this function */
1599 if (!ropage_next) {
1600 ropage_next = (vm_offset_t)&ropagetable_begin;
1601 }
1602
1603 if (map_static) {
1604 /* This is a RO allocation. Make sure we have room in the ropagetable area */
1605 assert(ropage_next < (vm_offset_t)&ropagetable_end);
1606
1607 /* Obtain physical address and increment the index into the ropagetable area */
1608 paddr = (pmap_paddr_t)kvtophys((sptm_papt_t)ropage_next);
1609 ropage_next += ARM_PGBYTES;
1610 } else {
1611 /* This is a RW allocation. Simply grab a page from [avail_start] */
1612 paddr = avail_start;
1613 avail_start += ARM_PGBYTES;
1614 }
1615
1616 /* Retype the page to XNU_PAGE_TABLE, with the desired level */
1617 sptm_retype_params_t retype_params;
1618 retype_params.level = level;
1619 sptm_retype(paddr, XNU_DEFAULT, XNU_PAGE_TABLE, retype_params);
1620
1621 return paddr;
1622 }
1623
1624 /**
1625 * Initialize a vm_image_offsets structure with information obtained from a
1626 * Mach-O header for the wanted image.
1627 *
1628 * @param debug_header_entry The entry in the debug header images list to obtain
1629 * a pointer to the Mach-O header from. This must be
1630 * either the SPTM or TXM debug header entry.
1631 * @param offsets Output pointer of the vm_image_offsets structure to fill in.
1632 */
1633 static void
init_image_offsets(size_t debug_header_entry,vm_image_offsets * offsets)1634 init_image_offsets(size_t debug_header_entry, vm_image_offsets *offsets)
1635 {
1636 assert(offsets != NULL);
1637 assert((debug_header_entry == DEBUG_HEADER_ENTRY_SPTM) ||
1638 (debug_header_entry == DEBUG_HEADER_ENTRY_TXM));
1639
1640 offsets->slid_base = (vm_offset_t)SPTMArgs->debug_header->image[debug_header_entry];
1641 kernel_mach_header_t *macho = (kernel_mach_header_t*)offsets->slid_base;
1642 offsets->unslid_base = (vm_offset_t)getsegbynamefromheader(macho, "__TEXT")->vmaddr;
1643 assert((offsets->slid_base != 0) && (offsets->unslid_base != 0));
1644 offsets->slide = offsets->slid_base - offsets->unslid_base;
1645 offsets->unslid_top = getlastaddr(macho);
1646 offsets->slid_top = offsets->unslid_top + offsets->slide;
1647 }
1648
1649 #define ARM64_PHYSMAP_SLIDE_RANGE (1ULL << 30) // 1 GB
1650 #define ARM64_PHYSMAP_SLIDE_MASK (ARM64_PHYSMAP_SLIDE_RANGE - 1)
1651
1652 void
arm_vm_init(uint64_t memory_size_override,boot_args * args)1653 arm_vm_init(uint64_t memory_size_override, boot_args * args)
1654 {
1655 vm_map_address_t va_l1, va_l1_end;
1656 tt_entry_t *cpu_l1_tte;
1657 tt_entry_t *cpu_l2_tte;
1658 vm_map_address_t va_l2, va_l2_end;
1659 vm_map_address_t dynamic_memory_begin;
1660 uint64_t mem_segments;
1661
1662 /* Get the virtual and physical kernel-managed memory base from boot_args */
1663 gVirtBase = args->virtBase;
1664 gPhysBase = args->physBase;
1665
1666 /* Get the memory size */
1667 #if KASAN
1668 real_phys_size = memSize + (shadow_ptop - shadow_pbase);
1669 #else
1670 real_phys_size = memSize;
1671 #endif
1672
1673 /**
1674 * Ensure the physical region we specify for the VM to manage ends on a
1675 * software page boundary. Note that the software page size (PAGE_SIZE)
1676 * may be a multiple of the hardware page size specified in ARM_PGBYTES.
1677 * We must round the reported memory size down to the nearest PAGE_SIZE
1678 * boundary to ensure the VM does not try to manage a page it does not
1679 * completely own. The KASAN shadow region, if present, is managed entirely
1680 * in units of the hardware page size and should not need similar treatment.
1681 */
1682 gPhysSize = mem_size = ((gPhysBase + memSize) & ~PAGE_MASK) - gPhysBase;
1683
1684
1685
1686 /* Obtain total memory size, including non-managed memory */
1687 mem_actual = args->memSizeActual ? args->memSizeActual : mem_size;
1688 if ((memory_size_override != 0) && (mem_size > memory_size_override)) {
1689 {
1690 mem_size = memory_size_override;
1691 }
1692 max_mem_actual = memory_size_override;
1693 } else {
1694 max_mem_actual = mem_actual;
1695 }
1696
1697 #if !defined(ARM_LARGE_MEMORY)
1698 /* Make sure the system does not have more physical memory than what can be mapped */
1699 if (mem_size >= ((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 2)) {
1700 panic("Unsupported memory configuration %lx", mem_size);
1701 }
1702 #endif /* !defined(ARM_LARGE_MEMORY) */
1703
1704 physmap_base = SPTMArgs->physmap_base;
1705 physmap_end = static_memory_end = SPTMArgs->physmap_end;
1706
1707 #if KASAN && !defined(ARM_LARGE_MEMORY) && !defined(CONFIG_SPTM)
1708 /* add the KASAN stolen memory to the physmap */
1709 dynamic_memory_begin = static_memory_end + (shadow_ptop - shadow_pbase);
1710 #else
1711 dynamic_memory_begin = static_memory_end;
1712 #endif
1713
1714 if (dynamic_memory_begin > VM_MAX_KERNEL_ADDRESS) {
1715 panic("Unsupported memory configuration %lx", mem_size);
1716 }
1717
1718 /*
1719 * TODO: free bootstrap table memory back to allocator.
1720 * on large memory systems bootstrap tables could be quite large.
1721 * after bootstrap complete, xnu can warm start with a single 16KB page mapping
1722 * to trampoline to KVA. this requires only 3 pages to stay resident.
1723 */
1724 avail_start = first_avail_phys;
1725
1726 /*
1727 * Initialize l1 page table page.
1728 *
1729 * SPTM TODO: Have a separate root_table_paddr field in the sptm_args
1730 * instead of snooping the libsptm_state (XNU should not be
1731 * snooping the libsptm_state directly in general).
1732 */
1733 cpu_ttep = (pmap_paddr_t)const_sptm_args.libsptm_state.root_table_paddr;
1734 cpu_tte = (tt_entry_t *)phystokv(cpu_ttep);
1735 avail_end = gPhysBase + mem_size;
1736 assert(!(avail_end & PAGE_MASK));
1737
1738 /* These need to be set early so pa_valid() works */
1739 vm_first_phys = gPhysBase;
1740 vm_last_phys = trunc_page(avail_end);
1741
1742 #if KASAN
1743 real_avail_end = gPhysBase + real_phys_size;
1744 #else
1745 real_avail_end = avail_end;
1746 #endif
1747
1748 /*
1749 * Now retrieve addresses for various segments from kernel mach-o header
1750 */
1751 segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT);
1752 segPLKDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_DATA_CONST", &segSizePLKDATACONST);
1753 segPLKTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_TEXT_EXEC", &segSizePLKTEXTEXEC);
1754 segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT);
1755 segDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA_CONST", &segSizeDATACONST);
1756 segTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT_EXEC", &segSizeTEXTEXEC);
1757 segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA);
1758
1759 segBOOTDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__BOOTDATA", &segSizeBOOTDATA);
1760 segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK);
1761 segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD);
1762 segKLDDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLDDATA", &segSizeKLDDATA);
1763 segPRELINKDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_DATA", &segSizePRELINKDATA);
1764 segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO);
1765 segPLKLLVMCOVB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LLVM_COV", &segSizePLKLLVMCOV);
1766 segPLKLINKEDITB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LINKEDIT", &segSizePLKLINKEDIT);
1767 segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST);
1768 segLASTDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LASTDATA_CONST", &segSizeLASTDATACONST);
1769
1770 sectHIBTEXTB = (vm_offset_t) getsectdatafromheader(&_mh_execute_header, "__TEXT_EXEC", "__hib_text", §SizeHIBTEXT);
1771 sectHIBDATACONSTB = (vm_offset_t) getsectdatafromheader(&_mh_execute_header, "__DATA_CONST", "__hib_const", §SizeHIBDATACONST);
1772 segHIBDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__HIBDATA", &segSizeHIBDATA);
1773
1774 if (kernel_mach_header_is_in_fileset(&_mh_execute_header)) {
1775 kernel_mach_header_t *kc_mh = PE_get_kc_header(KCKindPrimary);
1776
1777 // fileset has kext PLK_TEXT_EXEC under kernel collection TEXT_EXEC following kernel's LAST
1778 segKCTEXTEXECB = (vm_offset_t) getsegdatafromheader(kc_mh, "__TEXT_EXEC", &segSizeKCTEXTEXEC);
1779 assert(segPLKTEXTEXECB && !segSizePLKTEXTEXEC); // kernel PLK_TEXT_EXEC must be empty
1780
1781 assert(segLASTB); // kernel LAST can be empty, but it must have
1782 // a valid address for computations below.
1783
1784 assert(segKCTEXTEXECB <= segLASTB); // KC TEXT_EXEC must contain kernel LAST
1785 assert(segKCTEXTEXECB + segSizeKCTEXTEXEC >= segLASTB + segSizeLAST);
1786 segPLKTEXTEXECB = segLASTB + segSizeLAST;
1787 segSizePLKTEXTEXEC = segSizeKCTEXTEXEC - (segPLKTEXTEXECB - segKCTEXTEXECB);
1788
1789 // fileset has kext PLK_DATA_CONST under kernel collection DATA_CONST following kernel's LASTDATA_CONST
1790 segKCDATACONSTB = (vm_offset_t) getsegdatafromheader(kc_mh, "__DATA_CONST", &segSizeKCDATACONST);
1791 assert(segPLKDATACONSTB && !segSizePLKDATACONST); // kernel PLK_DATA_CONST must be empty
1792 assert(segLASTDATACONSTB && segSizeLASTDATACONST); // kernel LASTDATA_CONST must be non-empty
1793 assert(segKCDATACONSTB <= segLASTDATACONSTB); // KC DATA_CONST must contain kernel LASTDATA_CONST
1794 assert(segKCDATACONSTB + segSizeKCDATACONST >= segLASTDATACONSTB + segSizeLASTDATACONST);
1795 segPLKDATACONSTB = segLASTDATACONSTB + segSizeLASTDATACONST;
1796 segSizePLKDATACONST = segSizeKCDATACONST - (segPLKDATACONSTB - segKCDATACONSTB);
1797
1798 // fileset has kext PRELINK_DATA under kernel collection DATA following kernel's empty PRELINK_DATA
1799 segKCDATAB = (vm_offset_t) getsegdatafromheader(kc_mh, "__DATA", &segSizeKCDATA);
1800 assert(segPRELINKDATAB && !segSizePRELINKDATA); // kernel PRELINK_DATA must be empty
1801 assert(segKCDATAB <= segPRELINKDATAB); // KC DATA must contain kernel PRELINK_DATA
1802 assert(segKCDATAB + segSizeKCDATA >= segPRELINKDATAB + segSizePRELINKDATA);
1803 segSizePRELINKDATA = segSizeKCDATA - (segPRELINKDATAB - segKCDATAB);
1804
1805 // fileset has consolidated PRELINK_TEXT, PRELINK_INFO and LINKEDIT at the kernel collection level
1806 assert(segPRELINKTEXTB && !segSizePRELINKTEXT); // kernel PRELINK_TEXT must be empty
1807 segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(kc_mh, "__PRELINK_TEXT", &segSizePRELINKTEXT);
1808 assert(segPRELINKINFOB && !segSizePRELINKINFO); // kernel PRELINK_INFO must be empty
1809 segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(kc_mh, "__PRELINK_INFO", &segSizePRELINKINFO);
1810 segLINKB = (vm_offset_t) getsegdatafromheader(kc_mh, "__LINKEDIT", &segSizeLINK);
1811 }
1812
1813 /* if one of the new segments is present, the other one better be as well */
1814 if (segSizePLKDATACONST || segSizePLKTEXTEXEC) {
1815 assert(segSizePLKDATACONST && segSizePLKTEXTEXEC);
1816 }
1817
1818 etext = (vm_offset_t) segTEXTB + segSizeTEXT;
1819 sdata = (vm_offset_t) segDATAB;
1820 edata = (vm_offset_t) segDATAB + segSizeDATA;
1821 end_kern = round_page(segHIGHESTKC ? segHIGHESTKC : getlastkerneladdr()); /* Force end to next page */
1822
1823 vm_set_page_size();
1824
1825 vm_kernel_base = segTEXTB;
1826 vm_kernel_top = (vm_offset_t) &last_kernel_symbol;
1827 vm_kext_base = segPRELINKTEXTB;
1828 vm_kext_top = vm_kext_base + segSizePRELINKTEXT;
1829
1830 vm_prelink_stext = segPRELINKTEXTB;
1831 if (!segSizePLKTEXTEXEC && !segSizePLKDATACONST) {
1832 vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT;
1833 } else {
1834 vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT + segSizePLKDATACONST + segSizePLKTEXTEXEC;
1835 }
1836 vm_prelink_sinfo = segPRELINKINFOB;
1837 vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO;
1838 vm_slinkedit = segLINKB;
1839 vm_elinkedit = segLINKB + segSizeLINK;
1840
1841 vm_prelink_sdata = segPRELINKDATAB;
1842 vm_prelink_edata = segPRELINKDATAB + segSizePRELINKDATA;
1843
1844 arm_vm_prot_init(args);
1845
1846 /**
1847 * Count the number of pages the boot kernelcache occupies. Additionally,
1848 * ml_static_mfree() uses the BootKC ranges from the DT to account for freed kernelcache pages.
1849 */
1850 vm_page_kernelcache_count = arm_get_bootkc_ranges_from_DT(&arm_vm_kernelcache_ranges, &arm_vm_kernelcache_numranges);
1851
1852 assert(vm_page_kernelcache_count > 0);
1853
1854 #if KASAN
1855 /* record the extent of the physmap */
1856 physmap_vbase = physmap_base;
1857 physmap_vtop = physmap_end;
1858 kasan_init();
1859 #endif /* KASAN */
1860
1861 #if CONFIG_CPU_COUNTERS
1862 mt_early_init();
1863 #endif /* CONFIG_CPU_COUNTERS */
1864
1865 kva_active = TRUE;
1866
1867 if (arm_vm_auxkc_init()) {
1868 if (segLOWESTROAuxKC < segLOWESTRO) {
1869 segLOWESTRO = segLOWESTROAuxKC;
1870 }
1871 if (segHIGHESTROAuxKC > segHIGHESTRO) {
1872 segHIGHESTRO = segHIGHESTROAuxKC;
1873 }
1874 if (segLOWESTRXAuxKC < segLOWESTTEXT) {
1875 segLOWESTTEXT = segLOWESTRXAuxKC;
1876 }
1877
1878 #if XNU_TARGET_OS_OSX
1879 /**
1880 * If we are on macOS with 3P kexts, we disable
1881 * XNU_KERNEL_RESTRICTED for now.
1882 */
1883 use_xnu_restricted = false;
1884
1885
1886
1887 #endif /* XNU_TARGET_OS_OSX */
1888 }
1889
1890
1891 if (memory_size_override && memory_size_override < mem_size) {
1892 max_mem = memory_size_override;
1893 sane_size = memory_size_override - (avail_start - gPhysBase);
1894 } else {
1895 max_mem = mem_size;
1896 sane_size = mem_size - (avail_start - gPhysBase);
1897 }
1898 // vm_kernel_slide is set by arm_init()->arm_slide_rebase_and_sign_image()
1899 vm_kernel_slid_base = segLOWESTTEXT;
1900 vm_kernel_stext = segTEXTB;
1901
1902 if (kernel_mach_header_is_in_fileset(&_mh_execute_header)) {
1903 vm_kernel_etext = segTEXTEXECB + segSizeTEXTEXEC;
1904 vm_kernel_slid_top = vm_slinkedit;
1905 } else {
1906 assert(segDATACONSTB == segTEXTB + segSizeTEXT);
1907 assert(segTEXTEXECB == segDATACONSTB + segSizeDATACONST);
1908 vm_kernel_etext = segTEXTB + segSizeTEXT + segSizeDATACONST + segSizeTEXTEXEC;
1909 vm_kernel_slid_top = vm_prelink_einfo;
1910 }
1911
1912 /**
1913 * Calculate the address ranges used to determine whether an address is an
1914 * SPTM or TXM address, as well as the slides used to slide/unslide those
1915 * addresses.
1916 *
1917 * The debug header contains pointers to the beginning of the images loaded
1918 * up by iBoot (which always start with the Mach-O header). The __TEXT
1919 * segment should be the first (and lowest) segment in both of these
1920 * binaries (the addresses in the Mach-O header are all unslid).
1921 */
1922 init_image_offsets(DEBUG_HEADER_ENTRY_SPTM, &vm_sptm_offsets);
1923 init_image_offsets(DEBUG_HEADER_ENTRY_TXM, &vm_txm_offsets);
1924
1925 dynamic_memory_begin = ROUND_TWIG(dynamic_memory_begin);
1926
1927 /* TODO: CONFIG_XNUPOST CTRR test */
1928
1929 pmap_bootstrap(dynamic_memory_begin);
1930
1931 disable_preemption();
1932
1933 /*
1934 * Initialize l3 page table pages :
1935 * cover this address range:
1936 * 2MB + FrameBuffer size + 10MB for each 256MB segment
1937 *
1938 * Note: This does not allocate L3 page tables, since page tables for all static
1939 * memory is allocated and inserted into the hierarchy by the SPTM beforehand.
1940 * Instead, this code simply walks the page tables to find those pre-allocated
1941 * tables and allocates PTD objects for them.
1942 */
1943
1944 mem_segments = (mem_size + 0x0FFFFFFF) >> 28;
1945
1946 va_l1 = dynamic_memory_begin;
1947 va_l1_end = va_l1 + ((2 + (mem_segments * 10)) << 20);
1948 va_l1_end += round_page(args->Video.v_height * args->Video.v_rowBytes);
1949 va_l1_end = (va_l1_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL;
1950
1951 cpu_l1_tte = cpu_tte + L1_TABLE_T1_INDEX(va_l1, TCR_EL1_BOOT);
1952
1953 while (va_l1 < va_l1_end) {
1954 va_l2 = va_l1;
1955
1956 if (((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE) < va_l1) {
1957 /* If this is the last L1 entry, it must cover the last mapping. */
1958 va_l2_end = va_l1_end;
1959 } else {
1960 va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE, va_l1_end);
1961 }
1962
1963 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
1964
1965 while (va_l2 < va_l2_end) {
1966 /* Obtain pre-allocated page and setup L3 Table TTE in L2 */
1967 tt_entry_t *ttp = pmap_tt2e(kernel_pmap, va_l2);
1968 pt_entry_t *ptp = (pt_entry_t *)phystokv(tte_to_pa(*ttp));
1969 pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE);
1970
1971 va_l2 += ARM_TT_L2_SIZE;
1972 cpu_l2_tte++;
1973 }
1974
1975 va_l1 = va_l2_end;
1976 cpu_l1_tte++;
1977 }
1978
1979 /*
1980 * Initialize l3 page table pages :
1981 * cover this address range:
1982 * ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - PE_EARLY_BOOT_VA) to VM_MAX_KERNEL_ADDRESS
1983 *
1984 * Note: This does not allocate L3 page tables, since page tables for all static
1985 * memory is allocated and inserted into the hierarchy by the SPTM beforehand.
1986 * Instead, this code simply walks the page tables to find those pre-allocated
1987 * tables and allocates PTD objects for them.
1988 */
1989 va_l1 = (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - PE_EARLY_BOOT_VA;
1990 va_l1_end = VM_MAX_KERNEL_ADDRESS;
1991
1992 cpu_l1_tte = cpu_tte + L1_TABLE_T1_INDEX(va_l1, TCR_EL1_BOOT);
1993
1994 while (va_l1 < va_l1_end) {
1995 va_l2 = va_l1;
1996
1997 if (((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE) < va_l1) {
1998 /* If this is the last L1 entry, it must cover the last mapping. */
1999 va_l2_end = va_l1_end;
2000 } else {
2001 va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE, va_l1_end);
2002 }
2003
2004 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
2005
2006 while (va_l2 < va_l2_end) {
2007 /* Obtain pre-allocated page and setup L3 Table TTE in L2 */
2008 tt_entry_t *ttp = pmap_tt2e(kernel_pmap, va_l2);
2009 pt_entry_t *ptp = (pt_entry_t *)phystokv(tte_to_pa(*ttp));
2010 pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE);
2011
2012 va_l2 += ARM_TT_L2_SIZE;
2013 cpu_l2_tte++;
2014 }
2015
2016 va_l1 = va_l2_end;
2017 cpu_l1_tte++;
2018 }
2019
2020 /*
2021 * Adjust avail_start so that the range that the VM owns
2022 * starts on a PAGE_SIZE aligned boundary.
2023 */
2024 avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK;
2025
2026 /* TODO pmap_static_allocations_done() */
2027
2028 first_avail = avail_start;
2029 patch_low_glo_static_region(first_avail_phys, avail_start - first_avail_phys);
2030 enable_preemption();
2031 }
2032
2033 /*
2034 * Returns true if the address lies within __TEXT, __TEXT_EXEC or __DATA_CONST
2035 * segment range. This is what [vm_kernel_stext, vm_kernel_etext) used to cover.
2036 * The segments together may not make a continuous address space anymore and so
2037 * individual intervals are inspected.
2038 */
2039 bool
kernel_text_contains(vm_offset_t addr)2040 kernel_text_contains(vm_offset_t addr)
2041 {
2042 if (segTEXTB <= addr && addr < (segTEXTB + segSizeTEXT)) {
2043 return true;
2044 }
2045 if (segTEXTEXECB <= addr && addr < (segTEXTEXECB + segSizeTEXTEXEC)) {
2046 return true;
2047 }
2048 return segDATACONSTB <= addr && addr < (segDATACONSTB + segSizeDATACONST);
2049 }
2050