1 /**
2 * Copyright (c) 2022-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm64/lowglobals.h>
30 #include <kern/ecc.h>
31 #include <kern/timer_queue.h>
32 #include <kern/monotonic.h>
33 #include <machine/commpage.h>
34 #include <pexpert/device_tree.h>
35 #include <arm/cpu_internal.h>
36 #include <arm/misc_protos.h>
37 #include <arm/machine_cpu.h>
38 #include <arm/rtclock.h>
39 #include <vm/vm_map.h>
40 #include <mach/exclaves.h>
41 #include <mach/vm_param.h>
42 #include <libkern/stack_protector.h>
43 #include <console/serial_protos.h>
44 #include <arm64/sptm/pmap/pmap_pt_geometry.h>
45 #include <arm64/sptm/sptm.h>
46 #include <sptm/sptm_common.h>
47 #include <vm/vm_page_internal.h>
48
49 #if CONFIG_TELEMETRY
50 #include <kern/telemetry.h>
51 #endif /* CONFIG_TELEMETRY */
52
53 #if KPERF
54 #include <kperf/kptimer.h>
55 #endif /* KPERF */
56
57 #if HIBERNATION
58 #include <IOKit/IOPlatformExpert.h>
59 #include <machine/pal_hibernate.h>
60 #endif /* HIBERNATION */
61
62 #if __arm64__
63 #include <pexpert/arm64/apt_msg.h>
64 #endif
65
66
67 /**
68 * Functions defined elsewhere that are required by this source file.
69 */
70 extern void patch_low_glo(void);
71 extern int serial_init(void);
72 extern void sleep_token_buffer_init(void);
73
74 /**
75 * Bootstrap stacks. Used on the cold boot path to set up the boot CPU's
76 * per-CPU data structure.
77 */
78 extern vm_offset_t intstack_top;
79 extern vm_offset_t excepstack_top;
80
81 /* First (inclusive) and last (exclusive) physical addresses */
82 extern pmap_paddr_t vm_first_phys;
83 extern pmap_paddr_t vm_last_phys;
84
85 /* UART hibernation flag - import so we can set it ASAP on resume. */
86 extern MARK_AS_HIBERNATE_DATA bool uart_hibernation;
87
88 /* Used to cache memSize, as passed by iBoot */
89 SECURITY_READ_ONLY_LATE(uint64_t) memSize = 0;
90
91 int debug_task;
92
93 /**
94 * Set according to what serial-related boot-args have been passed to XUN.
95 */
96 extern int disableConsoleOutput;
97
98 #if XNU_TARGET_OS_OSX
99 /**
100 * Extern the PMAP boot-arg to enable/disable XNU_KERNEL_RESTRICTED.
101 * We need it here because if we detect an auxKC, we disable the mitigation.
102 */
103 extern bool use_xnu_restricted;
104 #endif /* XNU_TARGET_OS_OSX */
105
106 /**
107 * SPTM devices do not support static kernelcaches, but the rest of XNU
108 * expects this variable to be defined. Set it to false at build time.
109 */
110 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
111
112 TUNABLE(bool, restore_boot, "-restore", false);
113
114 /**
115 * First physical address freely available to xnu.
116 */
117 SECURITY_READ_ONLY_LATE(addr64_t) first_avail_phys = 0;
118
119 #if HAS_BP_RET
120 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
121 uint32_t bp_ret = 3;
122 extern void set_bp_ret(void);
123 #endif
124
125 #if SCHED_HYGIENE_DEBUG
126
127 #if XNU_PLATFORM_iPhoneOS
128 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 48000 /* 2ms */
129 #elif XNU_PLATFORM_XROS
130 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
131 #else
132 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 0xd0000 /* 35.499ms */
133 #endif /* XNU_PLATFORM_iPhoneOS */
134
135 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, interrupt_masked_debug_mode,
136 "machine-timeouts", "interrupt-masked-debug-mode",
137 "interrupt-masked-debug-mode",
138 SCHED_HYGIENE_MODE_PANIC,
139 TUNABLE_DT_CHECK_CHOSEN);
140
141 MACHINE_TIMEOUT_DEV_WRITEABLE(interrupt_masked_timeout, "interrupt-masked",
142 DEFAULT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
143 NULL);
144 #if __arm64__
145 #define SSHOT_INTERRUPT_MASKED_TIMEOUT 0xf9999 /* 64-bit: 42.599ms */
146 #endif
147 MACHINE_TIMEOUT_DEV_WRITEABLE(stackshot_interrupt_masked_timeout, "sshot-interrupt-masked",
148 SSHOT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
149 NULL);
150 #undef SSHOT_INTERRUPT_MASKED_TIMEOUT
151 #endif
152
153 /*
154 * A 6-second timeout will give the watchdog code a chance to run
155 * before a panic is triggered by the xcall routine.
156 */
157 #define XCALL_ACK_TIMEOUT_NS ((uint64_t) 6000000000)
158 uint64_t xcall_ack_timeout_abstime;
159
160 boot_args const_boot_args __attribute__((section("__DATA, __const")));
161 boot_args *BootArgs __attribute__((section("__DATA, __const")));
162
163 /**
164 * The SPTM provides a second set of boot arguments, on top of those
165 * provided by iBoot.
166 */
167 SECURITY_READ_ONLY_LATE(sptm_bootstrap_args_xnu_t) const_sptm_args;
168 SECURITY_READ_ONLY_LATE(const sptm_bootstrap_args_xnu_t *) SPTMArgs;
169 SECURITY_READ_ONLY_LATE(const bool *) sptm_xnu_triggered_panic_ptr;
170
171 extern char osbuild_config[];
172
173 TUNABLE(uint32_t, arm_diag, "diag", 0);
174 #ifdef APPLETYPHOON
175 static unsigned cpus_defeatures = 0x0;
176 extern void cpu_defeatures_set(unsigned int);
177 #endif
178
179 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
180 extern volatile boolean_t arm64_stall_sleep;
181 #endif
182
183 extern boolean_t force_immediate_debug_halt;
184
185 #if HAS_APPLE_PAC
186 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
187 #endif
188
189
190
191 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
192 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
193 SECURITY_READ_ONLY_LATE(ppnum_t) pmap_first_pnum;
194
195 SECURITY_READ_ONLY_LATE(bool) serial_console_enabled = false;
196
197 /**
198 * SPTM TODO: The following flag is set up based on the presence and
199 * configuration of the 'sptm_stability_hacks' boot-arg; this
200 * is used in certain codepaths that do not properly function
201 * today in SPTM systems to make the system more stable and fully
202 * able to boot to user space.
203 */
204 SECURITY_READ_ONLY_LATE(bool) sptm_stability_hacks = false;
205
206 #if APPLEVIRTUALPLATFORM
207 SECURITY_READ_ONLY_LATE(vm_offset_t) reset_vector_vaddr = 0;
208 #endif /* APPLEVIRTUALPLATFORM */
209
210 /*
211 * Forward definition
212 */
213 void arm_init(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_args);
214 #if KASAN
215 void arm_init_kasan(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_args);
216 #endif /* KASAN */
217
218 #if __arm64__
219 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
220
221 extern void configure_misc_apple_boot_args(void);
222 extern void configure_misc_apple_regs(bool is_boot_cpu);
223 extern void configure_timer_apple_regs(void);
224 #endif /* __arm64__ */
225
226
227 /*
228 * JOP rebasing
229 */
230
231 #define dyldLogFunc(msg, ...)
232 #include <mach/dyld_kernel_fixups.h>
233
234 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
235 extern uint32_t __thread_starts_sect_end[] __asm("section$end$__TEXT$__thread_starts");
236 #if defined(HAS_APPLE_PAC)
237 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
238 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
239 #endif /* defined(HAS_APPLE_PAC) */
240
241 extern vm_offset_t vm_kernel_slide;
242 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
243 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
244 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
245
246
247 void arm_slide_rebase_and_sign_image(void);
248 MARK_AS_FIXUP_TEXT void
arm_slide_rebase_and_sign_image(void)249 arm_slide_rebase_and_sign_image(void)
250 {
251 kernel_mach_header_t *k_mh, *kc_mh = NULL;
252 kernel_segment_command_t *seg;
253 uintptr_t slide;
254
255 /*
256 * The kernel is part of a MH_FILESET kernel collection, determine slide
257 * based on first segment's mach-o vmaddr (requires first kernel load
258 * command to be LC_SEGMENT_64 of the __TEXT segment)
259 */
260 k_mh = &_mh_execute_header;
261 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
262 assert(seg->cmd == LC_SEGMENT_KERNEL);
263 slide = (uintptr_t)k_mh - seg->vmaddr;
264
265 /*
266 * The kernel collection linker guarantees that the boot collection mach
267 * header vmaddr is the hardcoded kernel link address (as specified to
268 * ld64 when linking the kernel).
269 */
270 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
271 assert(kc_mh->filetype == MH_FILESET);
272
273 /*
274 * rebase and sign jops
275 * Note that we can't call any functions before this point, so
276 * we have to hard-code the knowledge that the base of the KC
277 * is the KC's mach-o header. This would change if any
278 * segment's VA started *before* the text segment
279 * (as the HIB segment does on x86).
280 */
281 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
282 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
283 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
284
285 /*
286 * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
287 * collection, so adjust them now, and determine the vmaddr range
288 * covered by read-only segments for the CTRR rorgn.
289 */
290 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
291 (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
292 (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
293 NULL, NULL, NULL);
294
295 /*
296 * Initialize slide global here to avoid duplicating this logic in
297 * arm_vm_init()
298 */
299 vm_kernel_slide = slide;
300 }
301
302 void arm_static_if_init(boot_args *args);
303 MARK_AS_FIXUP_TEXT void
arm_static_if_init(boot_args * args)304 arm_static_if_init(boot_args *args)
305 {
306 static_if_init(args->CommandLine);
307 }
308
309 void
arm_auxkc_init(void * mh,void * base)310 arm_auxkc_init(void *mh, void *base)
311 {
312 /*
313 * The kernel collection linker guarantees that the lowest vmaddr in an
314 * AuxKC collection is 0 (but note that the mach header is higher up since
315 * RW segments precede RO segments in the AuxKC).
316 */
317 uintptr_t slide = (uintptr_t)base;
318 kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
319
320 assert(akc_mh->filetype == MH_FILESET);
321 PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
322
323 /* rebase and sign jops */
324 const void *collection_base_pointers[KCNumKinds];
325 memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
326 kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
327
328 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
329 (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
330 (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
331 (uintptr_t *)&segHIGHESTNLEAuxKC);
332 #if defined(HAS_APPLE_PAC)
333 OSRuntimeSignStructorsInFileset(akc_mh);
334 #endif /* defined(HAS_APPLE_PAC) */
335 }
336
337 /*
338 * boot kernelcache ranges; used for accounting.
339 */
340 SECURITY_READ_ONLY_LATE(const arm_physrange_t *) arm_vm_kernelcache_ranges;
341 SECURITY_READ_ONLY_LATE(int) arm_vm_kernelcache_numranges;
342
343 #if __ARM_KERNEL_PROTECT__
344 /*
345 * If we want to support __ARM_KERNEL_PROTECT__, we need a sufficient amount of
346 * mappable space preceeding the kernel (as we unmap the kernel by cutting the
347 * range covered by TTBR1 in half). This must also cover the exception vectors.
348 */
349 static_assert(KERNEL_PMAP_HEAP_RANGE_START > ARM_KERNEL_PROTECT_EXCEPTION_START);
350
351 /* The exception vectors and the kernel cannot share root TTEs. */
352 static_assert((KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_ROOT_OFFMASK) > ARM_KERNEL_PROTECT_EXCEPTION_START);
353
354 /*
355 * We must have enough space in the TTBR1_EL1 range to create the EL0 mapping of
356 * the exception vectors.
357 */
358 static_assert((((~ARM_KERNEL_PROTECT_EXCEPTION_START) + 1) * 2ULL) <= (ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK));
359 #endif /* __ARM_KERNEL_PROTECT__ */
360
361 #define ARM_DYNAMIC_TABLE_XN (ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN)
362
363 #if KASAN
364 extern vm_offset_t shadow_pbase;
365 extern vm_offset_t shadow_ptop;
366 extern vm_offset_t physmap_vbase;
367 extern vm_offset_t physmap_vtop;
368 #endif
369
370 /*
371 * We explicitly place this in const, as it is not const from a language
372 * perspective, but it is only modified before we actually switch away from
373 * the bootstrap page tables.
374 */
375 SECURITY_READ_ONLY_LATE(uint8_t) bootstrap_pagetables[BOOTSTRAP_TABLE_SIZE] __attribute__((aligned(ARM_PGBYTES)));
376
377 /*
378 * Denotes the end of xnu.
379 */
380 extern void *last_kernel_symbol;
381
382 extern void arm64_replace_bootstack(cpu_data_t*);
383 extern void PE_slide_devicetree(vm_offset_t);
384
385 /*
386 * KASLR parameters
387 */
388 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_base;
389 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_top;
390 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_base;
391 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_top;
392 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_stext;
393 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_etext;
394 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slide;
395 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_base;
396 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_top;
397
398 SECURITY_READ_ONLY_LATE(vm_image_offsets) vm_sptm_offsets;
399 SECURITY_READ_ONLY_LATE(vm_image_offsets) vm_txm_offsets;
400
401 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_stext;
402 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_etext;
403 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sdata;
404 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_edata;
405 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sinfo;
406 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_einfo;
407 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_slinkedit;
408 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_elinkedit;
409
410 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_builtinkmod_text;
411 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_builtinkmod_text_end;
412
413 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernelcache_base;
414 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernelcache_top;
415
416 /* Used by <mach/arm/vm_param.h> */
417 SECURITY_READ_ONLY_LATE(unsigned long) gVirtBase;
418 SECURITY_READ_ONLY_LATE(unsigned long) gPhysBase;
419 SECURITY_READ_ONLY_LATE(unsigned long) gPhysSize;
420 SECURITY_READ_ONLY_LATE(unsigned long) gT0Sz = T0SZ_BOOT;
421 SECURITY_READ_ONLY_LATE(unsigned long) gT1Sz = T1SZ_BOOT;
422
423 /* 23543331 - step 1 of kext / kernel __TEXT and __DATA colocation is to move
424 * all kexts before the kernel. This is only for arm64 devices and looks
425 * something like the following:
426 * -- vmaddr order --
427 * 0xffffff8004004000 __PRELINK_TEXT
428 * 0xffffff8007004000 __TEXT (xnu)
429 * 0xffffff80075ec000 __DATA (xnu)
430 * 0xffffff80076dc000 __KLD (xnu)
431 * 0xffffff80076e0000 __LAST (xnu)
432 * 0xffffff80076e4000 __LINKEDIT (xnu)
433 * 0xffffff80076e4000 __PRELINK_DATA (not used yet)
434 * 0xffffff800782c000 __PRELINK_INFO
435 * 0xffffff80078e4000 -- End of kernelcache
436 */
437
438 /* 24921709 - make XNU ready for KTRR
439 *
440 * Two possible kernel cache layouts, depending on which kcgen is being used.
441 * VAs increasing downwards.
442 * Old KCGEN:
443 *
444 * __PRELINK_TEXT
445 * __TEXT
446 * __DATA_CONST
447 * __TEXT_EXEC
448 * __KLD
449 * __LAST
450 * __DATA
451 * __PRELINK_DATA (expected empty)
452 * __LINKEDIT
453 * __PRELINK_INFO
454 *
455 * New kcgen:
456 *
457 * __PRELINK_TEXT <--- First KTRR (ReadOnly) segment
458 * __PLK_DATA_CONST
459 * __PLK_TEXT_EXEC
460 * __TEXT
461 * __DATA_CONST
462 * __TEXT_EXEC
463 * __KLD
464 * __LAST <--- Last KTRR (ReadOnly) segment
465 * __DATA
466 * __BOOTDATA (if present)
467 * __LINKEDIT
468 * __PRELINK_DATA (expected populated now)
469 * __PLK_LINKEDIT
470 * __PRELINK_INFO
471 *
472 */
473
474 vm_offset_t mem_size; /* Size of actual physical memory present
475 * minus any performance buffer and possibly
476 * limited by mem_limit in bytes */
477 uint64_t mem_actual; /* The "One True" physical memory size
478 * actually, it's the highest physical
479 * address + 1 */
480 uint64_t max_mem; /* Size of physical memory (bytes), adjusted
481 * by maxmem */
482 uint64_t max_mem_actual; /* Actual size of physical memory (bytes),
483 * adjusted by the maxmem boot-arg */
484 uint64_t sane_size; /* Memory size to use for defaults
485 * calculations */
486 /* This no longer appears to be used; kill it? */
487 addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel
488 * virtual address known
489 * to the VM system */
490
491 SECURITY_READ_ONLY_LATE(vm_offset_t) segEXTRADATA;
492 SECURITY_READ_ONLY_LATE(unsigned long) segSizeEXTRADATA;
493
494 /* Trust cache portion of EXTRADATA (if within it) */
495 SECURITY_READ_ONLY_LATE(vm_offset_t) segTRUSTCACHE;
496 SECURITY_READ_ONLY_LATE(unsigned long) segSizeTRUSTCACHE;
497
498 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTTEXT;
499 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWEST;
500 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTRO;
501 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTRO;
502
503 /* Only set when booted from MH_FILESET kernel collections */
504 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTKC;
505 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTKC;
506 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTROKC;
507 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTROKC;
508 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTAuxKC;
509 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTAuxKC;
510 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTROAuxKC;
511 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTROAuxKC;
512 SECURITY_READ_ONLY_LATE(vm_offset_t) segLOWESTRXAuxKC;
513 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTRXAuxKC;
514 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIGHESTNLEAuxKC;
515
516 SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTB;
517 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXT;
518
519 SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATACONSTB;
520 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATACONST;
521
522 SECURITY_READ_ONLY_LATE(vm_offset_t) segTEXTEXECB;
523 SECURITY_READ_ONLY_LATE(unsigned long) segSizeTEXTEXEC;
524
525 SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATAB;
526 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATA;
527
528 SECURITY_READ_ONLY_LATE(vm_offset_t) segBOOTDATAB;
529 SECURITY_READ_ONLY_LATE(unsigned long) segSizeBOOTDATA;
530 extern vm_offset_t intstack_low_guard;
531 extern vm_offset_t intstack_high_guard;
532 extern vm_offset_t excepstack_high_guard;
533
534 SECURITY_READ_ONLY_LATE(vm_offset_t) segLINKB;
535 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLINK;
536
537 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDB;
538 SECURITY_READ_ONLY_LATE(unsigned long) segSizeKLD;
539 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDDATAB;
540 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKLDDATA;
541 SECURITY_READ_ONLY_LATE(vm_offset_t) segLASTB;
542 SECURITY_READ_ONLY_LATE(unsigned long) segSizeLAST;
543 SECURITY_READ_ONLY_LATE(vm_offset_t) segLASTDATACONSTB;
544 SECURITY_READ_ONLY_LATE(unsigned long) segSizeLASTDATACONST;
545
546 SECURITY_READ_ONLY_LATE(vm_offset_t) sectHIBTEXTB;
547 SECURITY_READ_ONLY_LATE(unsigned long) sectSizeHIBTEXT;
548 SECURITY_READ_ONLY_LATE(vm_offset_t) segHIBDATAB;
549 SECURITY_READ_ONLY_LATE(unsigned long) segSizeHIBDATA;
550 SECURITY_READ_ONLY_LATE(vm_offset_t) sectHIBDATACONSTB;
551 SECURITY_READ_ONLY_LATE(unsigned long) sectSizeHIBDATACONST;
552
553 SECURITY_READ_ONLY_LATE(vm_offset_t) segPRELINKTEXTB;
554 SECURITY_READ_ONLY_LATE(unsigned long) segSizePRELINKTEXT;
555
556 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKTEXTEXECB;
557 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKTEXTEXEC;
558
559 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKDATACONSTB;
560 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKDATACONST;
561
562 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKDATAB;
563 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKDATA;
564
565 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLLVMCOVB = 0;
566 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLLVMCOV = 0;
567
568 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLINKEDITB;
569 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLINKEDIT;
570
571 SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKINFOB;
572 SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKINFO;
573
574 /* Only set when booted from MH_FILESET primary kernel collection */
575 SECURITY_READ_ONLY_LATE(vm_offset_t) segKCTEXTEXECB;
576 SECURITY_READ_ONLY_LATE(unsigned long) segSizeKCTEXTEXEC;
577 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKCDATACONSTB;
578 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKCDATACONST;
579 SECURITY_READ_ONLY_LATE(static vm_offset_t) segKCDATAB;
580 SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKCDATA;
581
582 SECURITY_READ_ONLY_LATE(static boolean_t) use_contiguous_hint = TRUE;
583
584 SECURITY_READ_ONLY_LATE(int) PAGE_SHIFT_CONST;
585
586 SECURITY_READ_ONLY_LATE(vm_offset_t) end_kern;
587 SECURITY_READ_ONLY_LATE(vm_offset_t) etext;
588 SECURITY_READ_ONLY_LATE(vm_offset_t) sdata;
589 SECURITY_READ_ONLY_LATE(vm_offset_t) edata;
590
591 SECURITY_READ_ONLY_LATE(static vm_offset_t) auxkc_mh, auxkc_base;
592
593 pmap_paddr_t alloc_ptpage(sptm_pt_level_t level, bool map_static);
594 SECURITY_READ_ONLY_LATE(vm_offset_t) ropage_next;
595 extern int dtrace_keep_kernel_symbols(void);
596
597 /*
598 * Bootstrap the system enough to run with virtual memory.
599 * Map the kernel's code and data, and allocate the system page table.
600 * Page_size must already be set.
601 *
602 * Parameters:
603 * first_avail: first available physical page -
604 * after kernel page tables
605 * avail_start: PA of first physical page
606 * avail_end: PA of last physical page
607 */
608 SECURITY_READ_ONLY_LATE(vm_offset_t) first_avail;
609 SECURITY_READ_ONLY_LATE(vm_offset_t) static_memory_end;
610 SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_start;
611 SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_end;
612 SECURITY_READ_ONLY_LATE(pmap_paddr_t) real_avail_end;
613 SECURITY_READ_ONLY_LATE(unsigned long) real_phys_size;
614 SECURITY_READ_ONLY_LATE(vm_map_address_t) physmap_base = (vm_map_address_t)0;
615 SECURITY_READ_ONLY_LATE(vm_map_address_t) physmap_end = (vm_map_address_t)0;
616
617 typedef struct {
618 pmap_paddr_t pa;
619 vm_map_address_t va;
620 vm_size_t len;
621 } ptov_table_entry;
622
623 SECURITY_READ_ONLY_LATE(static boolean_t) kva_active = FALSE;
624
625 #if HAS_ARM_FEAT_SME
626 static SECURITY_READ_ONLY_LATE(bool) enable_sme = true;
627 #endif
628
629 /**
630 * sptm_supports_local_coredump is set in start_sptm.s when SPTM dispatch logic
631 * calls into XNU to handle a panic from SPTM/TXM/cL4. If this variable is set
632 * to false then osfmk/kern/debug.c:debugger_collect_diagnostic() will skip
633 * taking a local core dump. This defaults to true since as long as the panic
634 * doesn't occur within the SPTM, then the SPTM will support making calls during
635 * the panic path to save the coredump. Only when the panic occurs from within
636 * guarded mode do we let SPTM decide whether it supports local coredumps.
637 */
638 bool sptm_supports_local_coredump = true;
639
640 #if KASAN
641 /* Prototypes for KASAN functions */
642 void kasan_bootstrap(boot_args *, vm_offset_t pgtable, sptm_bootstrap_args_xnu_t *sptm_boot_args);
643
644 /**
645 * Entry point for systems that support an SPTM and are booting a KASAN kernel.
646 * This is required because KASAN kernels need to set up the shadow map before
647 * arm_init() can even run.
648 */
649 void
arm_init_kasan(boot_args * args,sptm_bootstrap_args_xnu_t * sptm_boot_args)650 arm_init_kasan(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_boot_args)
651 {
652 /* Initialize SPTM helper library. */
653 uint8_t ret = libsptm_init(&sptm_boot_args->libsptm_state);
654 if (ret != LIBSPTM_SUCCESS) {
655 panic("%s: libsptm_init failed: %u", __func__, ret);
656 }
657
658 memSize = args->memSize;
659 kasan_bootstrap(args, phystokv(sptm_boot_args->libsptm_state.root_table_paddr), sptm_boot_args);
660
661 arm_init(args, sptm_boot_args);
662 }
663 #endif /* KASAN */
664
665 /**
666 * Entry point for systems that support an SPTM - except on KASAN kernels,
667 * see above. Bootstrap stacks have been set up by the SPTM by this point,
668 * and XNU is responsible for rebasing and signing absolute addresses.
669 */
670 void
arm_init(boot_args * args,sptm_bootstrap_args_xnu_t * sptm_boot_args)671 arm_init(boot_args *args, sptm_bootstrap_args_xnu_t *sptm_boot_args)
672 {
673 unsigned int maxmem;
674 uint32_t memsize;
675 uint64_t xmaxmem;
676 thread_t thread;
677 DTEntry chosen;
678 unsigned int dt_entry_size;
679
680 extern void xnu_return_to_gl2(void);
681 const sptm_vaddr_t handler_addr = (sptm_vaddr_t) ptrauth_strip((void *)xnu_return_to_gl2, ptrauth_key_function_pointer);
682 sptm_register_xnu_exc_return(handler_addr);
683
684 #if defined(HAS_APPLE_PAC)
685 kernel_mach_header_t *kc_mh = PE_get_kc_header(KCKindPrimary);
686 OSRuntimeSignStructorsInFileset(kc_mh);
687 #endif /* defined(HAS_APPLE_PAC) */
688
689 /* If kernel integrity is supported, use a constant copy of the boot args. */
690 const_boot_args = *args;
691 BootArgs = args = &const_boot_args;
692 const_sptm_args = *sptm_boot_args;
693 SPTMArgs = sptm_boot_args = &const_sptm_args;
694 sptm_xnu_triggered_panic_ptr = sptm_boot_args->xnu_triggered_panic;
695 /*
696 * Initialize first_avail_phys from what the SPTM tells us.
697 * We're not using iBoot's topOfKernelData, as SPTM and other
698 * components have consumed pages themselves.
699 */
700 first_avail_phys = sptm_boot_args->first_avail_phys;
701
702 #if APPLEVIRTUALPLATFORM
703 reset_vector_vaddr = (vm_offset_t) sptm_boot_args->sptm_reset_vector_vaddr;
704 #endif /* APPLEVIRTUALPLATFORM */
705
706 cpu_data_init(&BootCpuData);
707 #if defined(HAS_APPLE_PAC)
708 /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
709 BootCpuData.rop_key = ml_default_rop_pid();
710 BootCpuData.jop_key = ml_default_jop_pid();
711 #endif /* defined(HAS_APPLE_PAC) */
712
713 PE_init_platform(FALSE, args); /* Get platform expert set up */
714
715 #if !KASAN
716 memSize = args->memSize;
717
718 /* Initialize SPTM helper library. */
719 uint8_t ret = libsptm_init(&const_sptm_args.libsptm_state);
720 if (ret != LIBSPTM_SUCCESS) {
721 panic("%s: libsptm_init failed: %u", __func__, ret);
722 }
723 #endif
724
725 #if __arm64__
726 configure_timer_apple_regs();
727 wfe_timeout_configure();
728 wfe_timeout_init();
729
730 configure_misc_apple_boot_args();
731 configure_misc_apple_regs(true);
732
733 #if (DEVELOPMENT || DEBUG)
734 unsigned long const *platform_stall_ptr = NULL;
735
736 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
737 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
738 }
739
740 // Not usable TUNABLE here because TUNABLEs are parsed at a later point.
741 if (SecureDTGetProperty(chosen, "xnu_platform_stall", (void const **)&platform_stall_ptr,
742 &dt_entry_size) == kSuccess) {
743 xnu_platform_stall_value = *platform_stall_ptr;
744 }
745
746 platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_ARM_INIT);
747
748 chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
749 dt_entry_size = 0;
750 #endif
751
752 #if HAS_ARM_FEAT_SME
753 (void)PE_parse_boot_argn("enable_sme", &enable_sme, sizeof(enable_sme));
754 if (enable_sme) {
755 arm_sme_init(true);
756 }
757 #endif
758
759
760 {
761 /*
762 * Select the advertised kernel page size.
763 */
764 if (memSize > 1ULL * 1024 * 1024 * 1024) {
765 /*
766 * arm64 device with > 1GB of RAM:
767 * kernel uses 16KB pages.
768 */
769 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
770 } else {
771 /*
772 * arm64 device with <= 1GB of RAM:
773 * kernel uses hardware page size
774 * (4KB for H6/H7, 16KB for H8+).
775 */
776 PAGE_SHIFT_CONST = ARM_PGSHIFT;
777 }
778
779 /* 32-bit apps always see 16KB page size */
780 page_shift_user32 = PAGE_MAX_SHIFT;
781 #ifdef APPLETYPHOON
782 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
783 if ((cpus_defeatures & 0xF) != 0) {
784 cpu_defeatures_set(cpus_defeatures & 0xF);
785 }
786 }
787 #endif
788 }
789 #endif
790
791 /* Enable SPTM stability hacks if requested */
792 PE_parse_boot_argn("sptm_stability_hacks", &sptm_stability_hacks, sizeof(sptm_stability_hacks));
793
794 ml_parse_cpu_topology();
795
796 siq_init();
797
798 master_cpu = ml_get_boot_cpu_number();
799 assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
800
801 BootCpuData.cpu_number = (unsigned short)master_cpu;
802 BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
803 BootCpuData.istackptr = &intstack_top;
804 BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
805 BootCpuData.excepstackptr = &excepstack_top;
806 CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
807 CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
808 + ((uintptr_t)&BootCpuData
809 - (uintptr_t)(args->virtBase)));
810
811 thread = thread_bootstrap();
812 thread->machine.CpuDatap = &BootCpuData;
813 thread->machine.pcpu_data_base_and_cpu_number =
814 ml_make_pcpu_base_and_cpu_number(0, BootCpuData.cpu_number);
815 machine_set_current_thread(thread);
816
817 /*
818 * Preemption is enabled for this thread so that it can lock mutexes without
819 * tripping the preemption check. In reality scheduling is not enabled until
820 * this thread completes, and there are no other threads to switch to, so
821 * preemption level is not really meaningful for the bootstrap thread.
822 */
823 thread->machine.preemption_count = 0;
824 cpu_bootstrap();
825
826 rtclock_early_init();
827
828 kernel_debug_string_early("kernel_startup_bootstrap");
829 kernel_startup_bootstrap();
830
831 /*
832 * Initialize the timer callout world
833 */
834 timer_call_init();
835
836 cpu_init();
837
838 processor_bootstrap();
839
840 if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
841 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
842 } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
843 xmaxmem = (uint64_t) memsize;
844 } else {
845 xmaxmem = 0;
846 }
847
848 #if SCHED_HYGIENE_DEBUG
849 {
850 int wdt_boot_arg = 0;
851 bool const wdt_disabled = (PE_parse_boot_argn("wdt", &wdt_boot_arg, sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1));
852
853 /* Disable if WDT is disabled */
854 if (wdt_disabled || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
855 interrupt_masked_debug_mode = SCHED_HYGIENE_MODE_OFF;
856 }
857 if (wdt_disabled || kern_feature_override(KF_PREEMPTION_DISABLED_DEBUG_OVRD)) {
858 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
859 }
860 }
861 #endif /* SCHED_HYGIENE_DEBUG */
862
863 nanoseconds_to_absolutetime(XCALL_ACK_TIMEOUT_NS, &xcall_ack_timeout_abstime);
864
865 #if HAS_BP_RET
866 PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
867 set_bp_ret(); // Apply branch predictor retention settings to boot CPU
868 #endif
869
870 PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
871
872 #if __ARM_PAN_AVAILABLE__
873 __builtin_arm_wsr("pan", 1);
874 #endif /* __ARM_PAN_AVAILABLE__ */
875
876
877 arm_vm_init(xmaxmem, args);
878
879 if (debug_boot_arg) {
880 patch_low_glo();
881 }
882
883 #if __arm64__ && WITH_CLASSIC_S2R
884 sleep_token_buffer_init();
885 #endif
886
887 PE_consistent_debug_inherit();
888
889 /* Setup debugging output. */
890 const unsigned int serial_exists = serial_init();
891 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
892 kprintf("kprintf initialized\n");
893
894 serialmode = 0;
895 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
896 /* Do we want a serial keyboard and/or console? */
897 kprintf("Serial mode specified: %08X\n", serialmode);
898 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
899 enable_dklog_serial_output = restore_boot || (serialmode & SERIALMODE_DKLOG) != 0;
900 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
901 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
902 if (force_sync) {
903 serialmode |= SERIALMODE_SYNCDRAIN;
904 kprintf(
905 "WARNING: Forcing uart driver to output synchronously."
906 "printf()s/IOLogs will impact kernel performance.\n"
907 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
908 }
909 }
910 }
911 if (kern_feature_override(KF_SERIAL_OVRD)) {
912 serialmode = 0;
913 }
914
915 /* Start serial if requested and a serial device was enumerated in serial_init(). */
916 if ((serialmode & SERIALMODE_OUTPUT) && serial_exists) {
917 serial_console_enabled = true;
918 (void)switch_to_serial_console(); /* Switch into serial mode from video console */
919 disableConsoleOutput = FALSE; /* Allow printfs to happen */
920 }
921 PE_create_console();
922
923 /* setup console output */
924 PE_init_printf(FALSE);
925
926 #if __arm64__
927 #if DEBUG
928 dump_kva_space();
929 #endif
930 #endif
931
932 cpu_machine_idle_init(TRUE);
933
934 PE_init_platform(TRUE, &BootCpuData);
935
936 /* Initialize the debug infrastructure system-wide and on the local core. */
937 pe_arm_debug_init_early(&BootCpuData);
938
939 #if RELEASE
940 /* Validate SPTM variant. */
941 if (const_sptm_args.sptm_variant != SPTM_VARIANT_RELEASE) {
942 panic("arm_init: Development SPTM / Release XNU is not a supported configuration.");
943 }
944 #endif /* RELEASE */
945
946 #if __arm64__
947 extern bool cpu_config_correct;
948 if (!cpu_config_correct) {
949 panic("The cpumask=N boot arg cannot be used together with cpus=N, and the boot CPU must be enabled");
950 }
951
952 ml_map_cpu_pio();
953 #endif
954
955 cpu_timebase_init(TRUE);
956
957 #if KPERF
958 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
959 kptimer_curcpu_up();
960 #endif /* KPERF */
961
962 PE_init_cpu();
963 #if __arm64__
964 apt_msg_init();
965 apt_msg_init_cpu();
966 #endif
967 fiq_context_init(TRUE);
968
969
970 #if HIBERNATION
971 pal_hib_init();
972 #endif /* HIBERNATION */
973
974 /*
975 * gPhysBase/Size only represent kernel-managed memory. These globals represent
976 * the actual DRAM base address and size as reported by iBoot through the
977 * device tree.
978 */
979 unsigned long const *dram_base;
980 unsigned long const *dram_size;
981 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
982 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
983 }
984
985 if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
986 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
987 }
988
989 if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
990 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
991 }
992
993 gDramBase = *dram_base;
994 gDramSize = *dram_size;
995 pmap_first_pnum = (ppnum_t)atop(gDramBase);
996
997 /*
998 * Initialize the stack protector for all future calls
999 * to C code. Since kernel_bootstrap() eventually
1000 * switches stack context without returning through this
1001 * function, we do not risk failing the check even though
1002 * we mutate the guard word during execution.
1003 */
1004 __stack_chk_guard = (unsigned long)early_random();
1005 /* Zero a byte of the protector to guard
1006 * against string vulnerabilities
1007 */
1008 __stack_chk_guard &= ~(0xFFULL << 8);
1009 machine_startup(args);
1010 }
1011
1012 /*
1013 * Routine: arm_init_cpu
1014 * Function:
1015 * Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
1016 */
1017
1018 void
arm_init_cpu(cpu_data_t * cpu_data_ptr,__unused uint64_t hibernation_args)1019 arm_init_cpu(
1020 cpu_data_t *cpu_data_ptr,
1021 __unused uint64_t hibernation_args)
1022 {
1023 #if HIBERNATION
1024 sptm_hibernation_args_xnu_t *hibargs = (sptm_hibernation_args_xnu_t *)hibernation_args;
1025
1026 if ((hibargs != 0) && (hibargs->hib_header_phys != 0) && (hibargs->handoff_page_count > 0)) {
1027 /*
1028 * We must copy the handoff region before anything else because the physical pages
1029 * holding the handoff region are not tracked by xnu as in-use.
1030 */
1031 HibernationCopyHandoffRegionFromPageArray(&hibargs->handoff_pages[0], hibargs->handoff_page_count);
1032 }
1033 #endif /* HIBERNATION */
1034
1035 #if __ARM_PAN_AVAILABLE__
1036 __builtin_arm_wsr("pan", 1);
1037 #endif
1038
1039 #ifdef __arm64__
1040 configure_timer_apple_regs();
1041 configure_misc_apple_regs(false);
1042 #endif
1043 #if HAS_ARM_FEAT_SME
1044 if (enable_sme) {
1045 arm_sme_init(false);
1046 }
1047 #endif
1048
1049 os_atomic_andnot(&cpu_data_ptr->cpu_flags, SleepState, relaxed);
1050
1051 siq_cpu_init();
1052
1053 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
1054
1055
1056 #if HIBERNATION
1057 if (hibargs != 0 && hibargs->hib_header_phys != 0) {
1058 gIOHibernateState = kIOHibernateStateWakingFromHibernate;
1059 uart_hibernation = true;
1060
1061
1062 __nosan_memcpy(gIOHibernateCurrentHeader, (void*)phystokv(hibargs->hib_header_phys), sizeof(IOHibernateImageHeader));
1063
1064 }
1065 if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate) && ml_is_quiescing()) {
1066 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
1067 extern uint64_t wake_abstime;
1068 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
1069
1070 // since the hw clock stops ticking across hibernation, we need to apply an offset;
1071 // iBoot computes this offset for us and passes it via the hibernation header
1072 extern uint64_t hwclock_conttime_offset;
1073 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
1074
1075 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
1076 // fix up its preemption count
1077 unsigned int expected_preemption_count = (gEnforcePlatformActionSafety ? 2 : 1);
1078 if (get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread) !=
1079 expected_preemption_count) {
1080 panic("unexpected preemption count %u on boot cpu thread (should be %u)",
1081 get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread),
1082 expected_preemption_count);
1083 }
1084 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
1085 }
1086 #endif /* HIBERNATION */
1087
1088 #if __arm64__
1089 wfe_timeout_init();
1090 flush_mmu_tlb();
1091 #endif
1092
1093 cpu_machine_idle_init(FALSE);
1094
1095 cpu_init();
1096
1097 #ifdef APPLETYPHOON
1098 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
1099 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
1100 }
1101 #endif
1102 /* Initialize the timebase before serial_init, as some serial
1103 * drivers use mach_absolute_time() to implement rate control
1104 */
1105 cpu_timebase_init(FALSE);
1106
1107 #if KPERF
1108 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
1109 kptimer_curcpu_up();
1110 #endif /* KPERF */
1111
1112 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
1113 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
1114 /*
1115 * Prevent CPUs from going into deep sleep until all
1116 * CPUs are ready to do so.
1117 */
1118 arm64_stall_sleep = TRUE;
1119 #endif
1120 serial_init();
1121 PE_init_platform(TRUE, NULL);
1122 commpage_update_timebase();
1123
1124 exclaves_update_timebase(EXCLAVES_CLOCK_ABSOLUTE,
1125 rtclock_base_abstime);
1126 #if HIBERNATION
1127 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
1128 exclaves_update_timebase(EXCLAVES_CLOCK_CONTINUOUS,
1129 hwclock_conttime_offset);
1130 }
1131 #endif /* HIBERNATION */
1132 }
1133 PE_init_cpu();
1134 #if __arm64__
1135 apt_msg_init_cpu();
1136 #endif
1137
1138 fiq_context_init(TRUE);
1139 cpu_data_ptr->rtcPop = EndOfAllTime;
1140 timer_resync_deadlines();
1141
1142 /* Start tracing (secondary CPU). */
1143 #if DEVELOPMENT || DEBUG
1144 PE_arm_debug_enable_trace(true);
1145 #endif /* DEVELOPMENT || DEBUG */
1146
1147 kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
1148
1149 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
1150 if (kdebug_enable == 0) {
1151 __kdebug_only uint64_t elapsed = kdebug_wake();
1152 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
1153 }
1154
1155 #if CONFIG_TELEMETRY
1156 bootprofile_wake_from_sleep();
1157 #endif /* CONFIG_TELEMETRY */
1158 }
1159 #if CONFIG_CPU_COUNTERS
1160 mt_wake_per_core();
1161 #endif /* CONFIG_CPU_COUNTERS */
1162
1163 #if defined(KERNEL_INTEGRITY_CTRR)
1164 if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
1165 lck_spin_lock(&ctrr_cpu_start_lck);
1166 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
1167 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
1168 lck_spin_unlock(&ctrr_cpu_start_lck);
1169 }
1170 #endif
1171
1172
1173 secondary_cpu_main(NULL);
1174 }
1175
1176 /*
1177 * Routine: arm_init_idle_cpu
1178 * Function: Resume from non-retention WFI. Called from the reset vector.
1179 */
1180 void __attribute__((noreturn))
arm_init_idle_cpu(cpu_data_t * cpu_data_ptr)1181 arm_init_idle_cpu(
1182 cpu_data_t *cpu_data_ptr)
1183 {
1184 #if __ARM_PAN_AVAILABLE__
1185 __builtin_arm_wsr("pan", 1);
1186 #endif
1187
1188 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
1189
1190 #if __arm64__
1191 wfe_timeout_init();
1192 #endif
1193
1194 #ifdef APPLETYPHOON
1195 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
1196 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
1197 }
1198 #endif
1199
1200 /*
1201 * Update the active debug object to reflect that debug registers have been reset.
1202 * This will force any thread with active debug state to resync the debug registers
1203 * if it returns to userspace on this CPU.
1204 */
1205 if (cpu_data_ptr->cpu_user_debug != NULL) {
1206 arm_debug_set(NULL);
1207 }
1208
1209 fiq_context_init(FALSE);
1210
1211 cpu_idle_exit(TRUE);
1212 }
1213
1214 vm_map_address_t
phystokv(pmap_paddr_t pa)1215 phystokv(pmap_paddr_t pa)
1216 {
1217 sptm_papt_t va;
1218 if (sptm_phystokv(pa, &va) != LIBSPTM_SUCCESS) {
1219 return 0;
1220 }
1221 return (vm_map_address_t)va;
1222 }
1223
1224 vm_map_address_t
phystokv_range(pmap_paddr_t pa,vm_size_t * max_len)1225 phystokv_range(pmap_paddr_t pa, vm_size_t *max_len)
1226 {
1227
1228 vm_size_t len;
1229
1230 len = PAGE_SIZE - (pa & PAGE_MASK);
1231 if (*max_len > len) {
1232 *max_len = len;
1233 }
1234
1235 return phystokv((sptm_paddr_t)pa);
1236 }
1237
1238 vm_offset_t
ml_static_vtop(vm_offset_t va)1239 ml_static_vtop(vm_offset_t va)
1240 {
1241 return (vm_offset_t)kvtophys_nofail((sptm_papt_t)va);
1242 }
1243
1244 #define ARM64_GRANULE_ALLOW_BLOCK (1 << 0)
1245 #define ARM64_GRANULE_ALLOW_HINT (1 << 1)
1246
1247 // Populate seg...AuxKC and fixup AuxKC permissions
1248 static bool
arm_vm_auxkc_init(void)1249 arm_vm_auxkc_init(void)
1250 {
1251 if (auxkc_mh == 0 || auxkc_base == 0) {
1252 return false; // no auxKC.
1253 }
1254
1255 /* Fixup AuxKC and populate seg*AuxKC globals used below */
1256 arm_auxkc_init((void*)auxkc_mh, (void*)auxkc_base);
1257
1258 /*
1259 * The AuxKC LINKEDIT segment needs to be covered by the RO region but is excluded
1260 * from the RO address range returned by kernel_collection_adjust_mh_addrs().
1261 * Ensure the highest non-LINKEDIT address in the AuxKC is the current end of
1262 * its RO region before extending it.
1263 */
1264 assert(segHIGHESTROAuxKC == segHIGHESTNLEAuxKC);
1265 assert(segHIGHESTAuxKC >= segHIGHESTROAuxKC);
1266 if (segHIGHESTAuxKC > segHIGHESTROAuxKC) {
1267 segHIGHESTROAuxKC = segHIGHESTAuxKC;
1268 }
1269
1270 /*
1271 * The AuxKC RO region must be right below the device tree/trustcache so that it can be covered
1272 * by CTRR, and the AuxKC RX region must be within the RO region.
1273 */
1274 assert(segHIGHESTRXAuxKC <= segHIGHESTROAuxKC);
1275 assert(segLOWESTRXAuxKC <= segHIGHESTRXAuxKC);
1276 assert(segLOWESTROAuxKC <= segLOWESTRXAuxKC);
1277 assert(segLOWESTAuxKC <= segLOWESTROAuxKC);
1278
1279 return true;
1280 }
1281
1282 /*
1283 * Looks up the set of properties that describe the physical load addresses and sizes of the boot
1284 * kernelcache's loaded segments in the device tree and returns (1) the number of segments found
1285 * in *arm_vm_kernelcache_numrangesp and (2) their starting/ending addresses as an array of type
1286 * arm_physrange_t in *arm_vm_kernelcache_rangesp.
1287 * The function returns the total number of pages across all loaded boot kernelcache segments.
1288 * If there is a problem looking up the /chosen/memory-map node in the DT, all arguments are
1289 * zeroed and the function returns 0.
1290 */
1291 static unsigned int
arm_get_bootkc_ranges_from_DT(const arm_physrange_t ** arm_vm_kernelcache_rangesp,int * arm_vm_kernelcache_numrangesp)1292 arm_get_bootkc_ranges_from_DT(const arm_physrange_t **arm_vm_kernelcache_rangesp, int *arm_vm_kernelcache_numrangesp)
1293 {
1294 DTEntry memory_map;
1295 int err;
1296 DTMemoryMapRange const *range;
1297 unsigned int rangeSize;
1298 #define NUM_BOOTKC_RANGES 5
1299 static arm_physrange_t bootkc_physranges[NUM_BOOTKC_RANGES] = { {0, } };
1300 static int bootkc_numranges = 0;
1301 static unsigned int bootkc_total_pages = 0;
1302
1303 assert(arm_vm_kernelcache_rangesp != NULL);
1304 assert(arm_vm_kernelcache_numrangesp != NULL);
1305
1306 /* return cached values if previously computed */
1307 if (bootkc_numranges == 0) {
1308 err = SecureDTLookupEntry(NULL, "chosen/memory-map", &memory_map);
1309 if (err != kSuccess) {
1310 *arm_vm_kernelcache_numrangesp = 0;
1311 *arm_vm_kernelcache_rangesp = NULL;
1312 return 0;
1313 }
1314
1315 /* We're looking for 5 ranges: BootKC-ro, BootKC-rx, BootKC-bx, BootKC-rw, and BootKC-le */
1316 const char *BootKC_Properties[NUM_BOOTKC_RANGES] = {
1317 "BootKC-ro", "BootKC-rx", "BootKC-bx", "BootKC-rw", "BootKC-le"
1318 };
1319
1320 for (int i = 0; i < NUM_BOOTKC_RANGES; i++) {
1321 err = SecureDTGetProperty(memory_map, BootKC_Properties[i], (void const **)&range, &rangeSize);
1322 if (err == kSuccess && rangeSize == sizeof(DTMemoryMapRange)) {
1323 bootkc_physranges[i].start_phys = range->paddr;
1324 bootkc_physranges[i].end_phys = range->paddr + range->length;
1325 assert((bootkc_physranges[i].end_phys & PAGE_MASK) == 0);
1326 bootkc_numranges++;
1327 bootkc_total_pages += (unsigned int) atop_64(bootkc_physranges[i].end_phys - bootkc_physranges[i].start_phys);
1328 }
1329 }
1330 }
1331
1332 *arm_vm_kernelcache_numrangesp = bootkc_numranges;
1333 *arm_vm_kernelcache_rangesp = &bootkc_physranges[0];
1334 return bootkc_total_pages;
1335 }
1336
1337 void
arm_vm_prot_init(__unused boot_args * args)1338 arm_vm_prot_init(__unused boot_args * args)
1339 {
1340 segLOWESTTEXT = UINT64_MAX;
1341 if (segSizePRELINKTEXT && (segPRELINKTEXTB < segLOWESTTEXT)) {
1342 segLOWESTTEXT = segPRELINKTEXTB;
1343 }
1344 assert(segSizeTEXT);
1345 if (segTEXTB < segLOWESTTEXT) {
1346 segLOWESTTEXT = segTEXTB;
1347 }
1348 assert(segLOWESTTEXT < UINT64_MAX);
1349
1350 segEXTRADATA = 0;
1351 segSizeEXTRADATA = 0;
1352 segTRUSTCACHE = 0;
1353 segSizeTRUSTCACHE = 0;
1354
1355 segLOWEST = segLOWESTTEXT;
1356 segLOWESTRO = segLOWESTTEXT;
1357
1358 if (segLOWESTKC && segLOWESTKC < segLOWEST) {
1359 /*
1360 * kernel collections have segments below the kernel. In particular the collection mach header
1361 * is below PRELINK_TEXT and is not covered by any other segments already tracked.
1362 */
1363 segLOWEST = segLOWESTKC;
1364 if (segLOWESTROKC && segLOWESTROKC < segLOWESTRO) {
1365 segLOWESTRO = segLOWESTROKC;
1366 }
1367 if (segHIGHESTROKC && segHIGHESTROKC > segHIGHESTRO) {
1368 segHIGHESTRO = segHIGHESTROKC;
1369 }
1370 }
1371
1372 DTEntry memory_map;
1373 int err;
1374
1375 // Device Tree portion of EXTRADATA
1376 if (SecureDTIsLockedDown()) {
1377 segEXTRADATA = (vm_offset_t)PE_state.deviceTreeHead;
1378 segSizeEXTRADATA = PE_state.deviceTreeSize;
1379 }
1380
1381 // Trust Caches portion of EXTRADATA
1382 {
1383 DTMemoryMapRange const *trustCacheRange;
1384 unsigned int trustCacheRangeSize;
1385
1386 err = SecureDTLookupEntry(NULL, "chosen/memory-map", &memory_map);
1387 assert(err == kSuccess);
1388
1389 err = SecureDTGetProperty(memory_map, "TrustCache", (void const **)&trustCacheRange, &trustCacheRangeSize);
1390 if (err == kSuccess) {
1391 if (trustCacheRangeSize != sizeof(DTMemoryMapRange)) {
1392 panic("Unexpected /chosen/memory-map/TrustCache property size %u != %zu", trustCacheRangeSize, sizeof(DTMemoryMapRange));
1393 }
1394
1395 vm_offset_t const trustCacheRegion = phystokv(trustCacheRange->paddr);
1396 if (trustCacheRegion < segLOWEST) {
1397 if (segEXTRADATA != 0) {
1398 if (trustCacheRegion != segEXTRADATA + segSizeEXTRADATA) {
1399 panic("Unexpected location of TrustCache region: %#lx != %#lx",
1400 trustCacheRegion, segEXTRADATA + segSizeEXTRADATA);
1401 }
1402 segSizeEXTRADATA += trustCacheRange->length;
1403 } else {
1404 // Not all devices support CTRR device trees.
1405 segEXTRADATA = trustCacheRegion;
1406 segSizeEXTRADATA = trustCacheRange->length;
1407 }
1408 }
1409 segTRUSTCACHE = trustCacheRegion;
1410 segSizeTRUSTCACHE = trustCacheRange->length;
1411 }
1412 }
1413
1414 if (segSizeEXTRADATA != 0) {
1415 if (segEXTRADATA <= segLOWEST) {
1416 segLOWEST = segEXTRADATA;
1417 if (segEXTRADATA <= segLOWESTRO) {
1418 segLOWESTRO = segEXTRADATA;
1419 }
1420 } else {
1421 panic("EXTRADATA is in an unexpected place: %#lx > %#lx", segEXTRADATA, segLOWEST);
1422 }
1423 }
1424
1425 /* Record the bounds of the kernelcache. */
1426 vm_kernelcache_base = segLOWEST;
1427
1428 auxkc_mh = SPTMArgs->auxkc_mh;
1429 auxkc_base = SPTMArgs->auxkc_base;
1430 end_kern = SPTMArgs->auxkc_end;
1431
1432 vm_kernelcache_top = end_kern;
1433 }
1434
1435 /*
1436 * return < 0 for a < b
1437 * 0 for a == b
1438 * > 0 for a > b
1439 */
1440 typedef int (*cmpfunc_t)(const void *a, const void *b);
1441
1442 extern void
1443 qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
1444
1445 SECURITY_READ_ONLY_LATE(static unsigned int) ptov_index = 0;
1446
1447 #define ROUND_L1(addr) (((addr) + ARM_TT_L1_OFFMASK) & ~(ARM_TT_L1_OFFMASK))
1448 #define ROUND_TWIG(addr) (((addr) + ARM_TT_TWIG_OFFMASK) & ~(ARM_TT_TWIG_OFFMASK))
1449
1450 void
arm_vm_prot_finalize(boot_args * args __unused)1451 arm_vm_prot_finalize(boot_args * args __unused)
1452 {
1453 /*
1454 * At this point, we are far enough along in the boot process that it will be
1455 * safe to free up all of the memory preceeding the kernel. It may in fact
1456 * be safe to do this earlier.
1457 *
1458 * This keeps the memory in the V-to-P mapping, but advertises it to the VM
1459 * as usable.
1460 */
1461
1462 /* Slide KLDDATA */
1463 sptm_slide_region(segKLDDATAB, (unsigned int)(segSizeKLDDATA >> PAGE_SHIFT));
1464
1465 /*
1466 * Replace the boot CPU's stacks with properly-guarded dynamically allocated stacks.
1467 * This must happen prior to sliding segBOOTDATAB, which will effectively remove
1468 * the existing boot stacks.
1469 */
1470 cpu_stack_alloc(&BootCpuData);
1471 arm64_replace_bootstack(&BootCpuData);
1472
1473 /* Slide early-boot data */
1474 sptm_slide_region(segBOOTDATAB, (unsigned int)(segSizeBOOTDATA >> PAGE_SHIFT));
1475
1476 /* Slide linkedit, unless otherwise requested */
1477 bool keep_linkedit = false;
1478 PE_parse_boot_argn("keepsyms", &keep_linkedit, sizeof(keep_linkedit));
1479 #if CONFIG_DTRACE
1480 if (dtrace_keep_kernel_symbols()) {
1481 keep_linkedit = true;
1482 }
1483 #endif /* CONFIG_DTRACE */
1484 #if KASAN_DYNAMIC_BLACKLIST
1485 /* KASAN's dynamic blacklist needs to query the LINKEDIT segment at runtime. As such, the
1486 * kext bootstrap code will not jettison LINKEDIT on kasan kernels, so don't bother to relocate it. */
1487 keep_linkedit = true;
1488 #endif
1489
1490 if (!keep_linkedit) {
1491 sptm_slide_region(segLINKB, (unsigned int)(segSizeLINK >> PAGE_SHIFT));
1492 if (segSizePLKLINKEDIT) {
1493 /* Prelinked kernel LINKEDIT */
1494 sptm_slide_region(segPLKLINKEDITB, (unsigned int)(segSizePLKLINKEDIT >> PAGE_SHIFT));
1495 }
1496 }
1497
1498 /* Slide prelinked kernel plists */
1499 sptm_slide_region(segPRELINKINFOB, (unsigned int)(segSizePRELINKINFO >> PAGE_SHIFT));
1500
1501 /*
1502 * Free the portion of memory that precedes the first usable region, known
1503 * as the physical slide.
1504 */
1505 ml_static_mfree(SPTMArgs->phys_slide_papt, SPTMArgs->phys_slide_size);
1506
1507 /*
1508 * KTRR support means we will be mucking with these pages and trying to
1509 * protect them; we cannot free the pages to the VM if we do this.
1510 */
1511 if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC && segSizePRELINKTEXT) {
1512 /* If new segments not present, PRELINK_TEXT is not dynamically sized, free DRAM between it and xnu TEXT */
1513 ml_static_mfree(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT));
1514 }
1515
1516 ml_static_mfree(segBOOTDATAB, segSizeBOOTDATA);
1517
1518 #if __ARM_KERNEL_PROTECT__
1519 arm_vm_populate_kernel_el0_mappings();
1520 #endif /* __ARM_KERNEL_PROTECT__ */
1521 }
1522
1523 /* allocate a page for a page table: we support static and dynamic mappings.
1524 *
1525 * returns a physical address for the allocated page
1526 *
1527 * for static mappings, we allocate from the region ropagetable_begin to ro_pagetable_end-1,
1528 * which is defined in the DATA_CONST segment and will be protected RNX when vm_prot_finalize runs.
1529 *
1530 * for dynamic mappings, we allocate from avail_start, which should remain RWNX.
1531 */
1532 pmap_paddr_t
alloc_ptpage(sptm_pt_level_t level,bool map_static)1533 alloc_ptpage(sptm_pt_level_t level, bool map_static)
1534 {
1535 pmap_paddr_t paddr = 0;
1536
1537 #if !(defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR))
1538 map_static = FALSE;
1539 #endif
1540
1541 /* Set the next free ropage if this is the first call to this function */
1542 if (!ropage_next) {
1543 ropage_next = (vm_offset_t)&ropagetable_begin;
1544 }
1545
1546 if (map_static) {
1547 /* This is a RO allocation. Make sure we have room in the ropagetable area */
1548 assert(ropage_next < (vm_offset_t)&ropagetable_end);
1549
1550 /* Obtain physical address and increment the index into the ropagetable area */
1551 paddr = (pmap_paddr_t)kvtophys((sptm_papt_t)ropage_next);
1552 ropage_next += ARM_PGBYTES;
1553 } else {
1554 /* This is a RW allocation. Simply grab a page from [avail_start] */
1555 paddr = avail_start;
1556 avail_start += ARM_PGBYTES;
1557 }
1558
1559 /* Retype the page to XNU_PAGE_TABLE, with the desired level */
1560 sptm_retype_params_t retype_params;
1561 retype_params.level = level;
1562 sptm_retype(paddr, XNU_DEFAULT, XNU_PAGE_TABLE, retype_params);
1563
1564 return paddr;
1565 }
1566
1567 /**
1568 * Initialize a vm_image_offsets structure with information obtained from a
1569 * Mach-O header for the wanted image.
1570 *
1571 * @param debug_header_entry The entry in the debug header images list to obtain
1572 * a pointer to the Mach-O header from. This must be
1573 * either the SPTM or TXM debug header entry.
1574 * @param offsets Output pointer of the vm_image_offsets structure to fill in.
1575 */
1576 static void
init_image_offsets(size_t debug_header_entry,vm_image_offsets * offsets)1577 init_image_offsets(size_t debug_header_entry, vm_image_offsets *offsets)
1578 {
1579 assert(offsets != NULL);
1580 assert((debug_header_entry == DEBUG_HEADER_ENTRY_SPTM) ||
1581 (debug_header_entry == DEBUG_HEADER_ENTRY_TXM));
1582
1583 offsets->slid_base = (vm_offset_t)SPTMArgs->debug_header->image[debug_header_entry];
1584 kernel_mach_header_t *macho = (kernel_mach_header_t*)offsets->slid_base;
1585 offsets->unslid_base = (vm_offset_t)getsegbynamefromheader(macho, "__TEXT")->vmaddr;
1586 assert((offsets->slid_base != 0) && (offsets->unslid_base != 0));
1587 offsets->slide = offsets->slid_base - offsets->unslid_base;
1588 offsets->unslid_top = getlastaddr(macho);
1589 offsets->slid_top = offsets->unslid_top + offsets->slide;
1590 }
1591
1592 #define ARM64_PHYSMAP_SLIDE_RANGE (1ULL << 30) // 1 GB
1593 #define ARM64_PHYSMAP_SLIDE_MASK (ARM64_PHYSMAP_SLIDE_RANGE - 1)
1594
1595 void
arm_vm_init(uint64_t memory_size,boot_args * args)1596 arm_vm_init(uint64_t memory_size, boot_args * args)
1597 {
1598 vm_map_address_t va_l1, va_l1_end;
1599 tt_entry_t *cpu_l1_tte;
1600 tt_entry_t *cpu_l2_tte;
1601 vm_map_address_t va_l2, va_l2_end;
1602 vm_map_address_t dynamic_memory_begin;
1603 uint64_t mem_segments;
1604
1605 /* Get the virtual and physical kernel-managed memory base from boot_args */
1606 gVirtBase = args->virtBase;
1607 gPhysBase = args->physBase;
1608
1609 /* Get the memory size */
1610 #if KASAN
1611 real_phys_size = memSize + (shadow_ptop - shadow_pbase);
1612 #else
1613 real_phys_size = memSize;
1614 #endif
1615
1616 /**
1617 * Ensure the physical region we specify for the VM to manage ends on a
1618 * software page boundary. Note that the software page size (PAGE_SIZE)
1619 * may be a multiple of the hardware page size specified in ARM_PGBYTES.
1620 * We must round the reported memory size down to the nearest PAGE_SIZE
1621 * boundary to ensure the VM does not try to manage a page it does not
1622 * completely own. The KASAN shadow region, if present, is managed entirely
1623 * in units of the hardware page size and should not need similar treatment.
1624 */
1625 gPhysSize = mem_size = ((gPhysBase + memSize) & ~PAGE_MASK) - gPhysBase;
1626
1627
1628
1629 /* Obtain total memory size, including non-managed memory */
1630 mem_actual = args->memSizeActual ? args->memSizeActual : mem_size;
1631
1632 if ((memory_size != 0) && (mem_size > memory_size)) {
1633 mem_size = memory_size;
1634 max_mem_actual = memory_size;
1635 } else {
1636 max_mem_actual = mem_actual;
1637 }
1638
1639 /* Make sure the system does not have more physical memory than what can be mapped */
1640 if (mem_size >= ((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 2)) {
1641 panic("Unsupported memory configuration %lx", mem_size);
1642 }
1643
1644 physmap_base = SPTMArgs->physmap_base;
1645 physmap_end = static_memory_end = SPTMArgs->physmap_end;
1646
1647 #if KASAN && !defined(ARM_LARGE_MEMORY) && !defined(CONFIG_SPTM)
1648 /* add the KASAN stolen memory to the physmap */
1649 dynamic_memory_begin = static_memory_end + (shadow_ptop - shadow_pbase);
1650 #else
1651 dynamic_memory_begin = static_memory_end;
1652 #endif
1653
1654 if (dynamic_memory_begin > VM_MAX_KERNEL_ADDRESS) {
1655 panic("Unsupported memory configuration %lx", mem_size);
1656 }
1657
1658 /*
1659 * TODO: free bootstrap table memory back to allocator.
1660 * on large memory systems bootstrap tables could be quite large.
1661 * after bootstrap complete, xnu can warm start with a single 16KB page mapping
1662 * to trampoline to KVA. this requires only 3 pages to stay resident.
1663 */
1664 avail_start = first_avail_phys;
1665
1666 /*
1667 * Initialize l1 page table page.
1668 *
1669 * SPTM TODO: Have a separate root_table_paddr field in the sptm_args
1670 * instead of snooping the libsptm_state (XNU should not be
1671 * snooping the libsptm_state directly in general).
1672 */
1673 cpu_ttep = (pmap_paddr_t)const_sptm_args.libsptm_state.root_table_paddr;
1674 cpu_tte = (tt_entry_t *)phystokv(cpu_ttep);
1675 avail_end = gPhysBase + mem_size;
1676 assert(!(avail_end & PAGE_MASK));
1677
1678 /* These need to be set early so pa_valid() works */
1679 vm_first_phys = gPhysBase;
1680 vm_last_phys = trunc_page(avail_end);
1681
1682 #if KASAN
1683 real_avail_end = gPhysBase + real_phys_size;
1684 #else
1685 real_avail_end = avail_end;
1686 #endif
1687
1688 /*
1689 * Now retrieve addresses for various segments from kernel mach-o header
1690 */
1691 segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT);
1692 segPLKDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_DATA_CONST", &segSizePLKDATACONST);
1693 segPLKTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_TEXT_EXEC", &segSizePLKTEXTEXEC);
1694 segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT);
1695 segDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA_CONST", &segSizeDATACONST);
1696 segTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT_EXEC", &segSizeTEXTEXEC);
1697 segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA);
1698
1699 segBOOTDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__BOOTDATA", &segSizeBOOTDATA);
1700 segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK);
1701 segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD);
1702 segKLDDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLDDATA", &segSizeKLDDATA);
1703 segPRELINKDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_DATA", &segSizePRELINKDATA);
1704 segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO);
1705 segPLKLLVMCOVB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LLVM_COV", &segSizePLKLLVMCOV);
1706 segPLKLINKEDITB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LINKEDIT", &segSizePLKLINKEDIT);
1707 segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST);
1708 segLASTDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LASTDATA_CONST", &segSizeLASTDATACONST);
1709
1710 sectHIBTEXTB = (vm_offset_t) getsectdatafromheader(&_mh_execute_header, "__TEXT_EXEC", "__hib_text", §SizeHIBTEXT);
1711 sectHIBDATACONSTB = (vm_offset_t) getsectdatafromheader(&_mh_execute_header, "__DATA_CONST", "__hib_const", §SizeHIBDATACONST);
1712 segHIBDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__HIBDATA", &segSizeHIBDATA);
1713
1714 if (kernel_mach_header_is_in_fileset(&_mh_execute_header)) {
1715 kernel_mach_header_t *kc_mh = PE_get_kc_header(KCKindPrimary);
1716
1717 // fileset has kext PLK_TEXT_EXEC under kernel collection TEXT_EXEC following kernel's LAST
1718 segKCTEXTEXECB = (vm_offset_t) getsegdatafromheader(kc_mh, "__TEXT_EXEC", &segSizeKCTEXTEXEC);
1719 assert(segPLKTEXTEXECB && !segSizePLKTEXTEXEC); // kernel PLK_TEXT_EXEC must be empty
1720
1721 assert(segLASTB); // kernel LAST can be empty, but it must have
1722 // a valid address for computations below.
1723
1724 assert(segKCTEXTEXECB <= segLASTB); // KC TEXT_EXEC must contain kernel LAST
1725 assert(segKCTEXTEXECB + segSizeKCTEXTEXEC >= segLASTB + segSizeLAST);
1726 segPLKTEXTEXECB = segLASTB + segSizeLAST;
1727 segSizePLKTEXTEXEC = segSizeKCTEXTEXEC - (segPLKTEXTEXECB - segKCTEXTEXECB);
1728
1729 // fileset has kext PLK_DATA_CONST under kernel collection DATA_CONST following kernel's LASTDATA_CONST
1730 segKCDATACONSTB = (vm_offset_t) getsegdatafromheader(kc_mh, "__DATA_CONST", &segSizeKCDATACONST);
1731 assert(segPLKDATACONSTB && !segSizePLKDATACONST); // kernel PLK_DATA_CONST must be empty
1732 assert(segLASTDATACONSTB && segSizeLASTDATACONST); // kernel LASTDATA_CONST must be non-empty
1733 assert(segKCDATACONSTB <= segLASTDATACONSTB); // KC DATA_CONST must contain kernel LASTDATA_CONST
1734 assert(segKCDATACONSTB + segSizeKCDATACONST >= segLASTDATACONSTB + segSizeLASTDATACONST);
1735 segPLKDATACONSTB = segLASTDATACONSTB + segSizeLASTDATACONST;
1736 segSizePLKDATACONST = segSizeKCDATACONST - (segPLKDATACONSTB - segKCDATACONSTB);
1737
1738 // fileset has kext PRELINK_DATA under kernel collection DATA following kernel's empty PRELINK_DATA
1739 segKCDATAB = (vm_offset_t) getsegdatafromheader(kc_mh, "__DATA", &segSizeKCDATA);
1740 assert(segPRELINKDATAB && !segSizePRELINKDATA); // kernel PRELINK_DATA must be empty
1741 assert(segKCDATAB <= segPRELINKDATAB); // KC DATA must contain kernel PRELINK_DATA
1742 assert(segKCDATAB + segSizeKCDATA >= segPRELINKDATAB + segSizePRELINKDATA);
1743 segSizePRELINKDATA = segSizeKCDATA - (segPRELINKDATAB - segKCDATAB);
1744
1745 // fileset has consolidated PRELINK_TEXT, PRELINK_INFO and LINKEDIT at the kernel collection level
1746 assert(segPRELINKTEXTB && !segSizePRELINKTEXT); // kernel PRELINK_TEXT must be empty
1747 segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(kc_mh, "__PRELINK_TEXT", &segSizePRELINKTEXT);
1748 assert(segPRELINKINFOB && !segSizePRELINKINFO); // kernel PRELINK_INFO must be empty
1749 segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(kc_mh, "__PRELINK_INFO", &segSizePRELINKINFO);
1750 segLINKB = (vm_offset_t) getsegdatafromheader(kc_mh, "__LINKEDIT", &segSizeLINK);
1751 }
1752
1753 /* if one of the new segments is present, the other one better be as well */
1754 if (segSizePLKDATACONST || segSizePLKTEXTEXEC) {
1755 assert(segSizePLKDATACONST && segSizePLKTEXTEXEC);
1756 }
1757
1758 etext = (vm_offset_t) segTEXTB + segSizeTEXT;
1759 sdata = (vm_offset_t) segDATAB;
1760 edata = (vm_offset_t) segDATAB + segSizeDATA;
1761 end_kern = round_page(segHIGHESTKC ? segHIGHESTKC : getlastkerneladdr()); /* Force end to next page */
1762
1763 vm_set_page_size();
1764
1765 vm_kernel_base = segTEXTB;
1766 vm_kernel_top = (vm_offset_t) &last_kernel_symbol;
1767 vm_kext_base = segPRELINKTEXTB;
1768 vm_kext_top = vm_kext_base + segSizePRELINKTEXT;
1769
1770 vm_prelink_stext = segPRELINKTEXTB;
1771 if (!segSizePLKTEXTEXEC && !segSizePLKDATACONST) {
1772 vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT;
1773 } else {
1774 vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT + segSizePLKDATACONST + segSizePLKTEXTEXEC;
1775 }
1776 vm_prelink_sinfo = segPRELINKINFOB;
1777 vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO;
1778 vm_slinkedit = segLINKB;
1779 vm_elinkedit = segLINKB + segSizeLINK;
1780
1781 vm_prelink_sdata = segPRELINKDATAB;
1782 vm_prelink_edata = segPRELINKDATAB + segSizePRELINKDATA;
1783
1784 arm_vm_prot_init(args);
1785
1786 /**
1787 * Count the number of pages the boot kernelcache occupies. Additionally,
1788 * ml_static_mfree() uses the BootKC ranges from the DT to account for freed kernelcache pages.
1789 */
1790 vm_page_kernelcache_count = arm_get_bootkc_ranges_from_DT(&arm_vm_kernelcache_ranges, &arm_vm_kernelcache_numranges);
1791
1792 assert(vm_page_kernelcache_count > 0);
1793
1794 #if KASAN
1795 /* record the extent of the physmap */
1796 physmap_vbase = physmap_base;
1797 physmap_vtop = physmap_end;
1798 kasan_init();
1799 #endif /* KASAN */
1800
1801 #if CONFIG_CPU_COUNTERS
1802 mt_early_init();
1803 #endif /* CONFIG_CPU_COUNTERS */
1804
1805 kva_active = TRUE;
1806
1807 if (arm_vm_auxkc_init()) {
1808 if (segLOWESTROAuxKC < segLOWESTRO) {
1809 segLOWESTRO = segLOWESTROAuxKC;
1810 }
1811 if (segHIGHESTROAuxKC > segHIGHESTRO) {
1812 segHIGHESTRO = segHIGHESTROAuxKC;
1813 }
1814 if (segLOWESTRXAuxKC < segLOWESTTEXT) {
1815 segLOWESTTEXT = segLOWESTRXAuxKC;
1816 }
1817
1818 #if XNU_TARGET_OS_OSX
1819 /**
1820 * If we are on macOS with 3P kexts, we disable
1821 * XNU_KERNEL_RESTRICTED for now.
1822 */
1823 use_xnu_restricted = false;
1824
1825 #endif /* XNU_TARGET_OS_OSX */
1826 }
1827
1828 sane_size = mem_size - (avail_start - gPhysBase);
1829 max_mem = mem_size;
1830 // vm_kernel_slide is set by arm_init()->arm_slide_rebase_and_sign_image()
1831 vm_kernel_slid_base = segLOWESTTEXT;
1832 vm_kernel_stext = segTEXTB;
1833
1834 if (kernel_mach_header_is_in_fileset(&_mh_execute_header)) {
1835 vm_kernel_etext = segTEXTEXECB + segSizeTEXTEXEC;
1836 vm_kernel_slid_top = vm_slinkedit;
1837 } else {
1838 assert(segDATACONSTB == segTEXTB + segSizeTEXT);
1839 assert(segTEXTEXECB == segDATACONSTB + segSizeDATACONST);
1840 vm_kernel_etext = segTEXTB + segSizeTEXT + segSizeDATACONST + segSizeTEXTEXEC;
1841 vm_kernel_slid_top = vm_prelink_einfo;
1842 }
1843
1844 /**
1845 * Calculate the address ranges used to determine whether an address is an
1846 * SPTM or TXM address, as well as the slides used to slide/unslide those
1847 * addresses.
1848 *
1849 * The debug header contains pointers to the beginning of the images loaded
1850 * up by iBoot (which always start with the Mach-O header). The __TEXT
1851 * segment should be the first (and lowest) segment in both of these
1852 * binaries (the addresses in the Mach-O header are all unslid).
1853 */
1854 init_image_offsets(DEBUG_HEADER_ENTRY_SPTM, &vm_sptm_offsets);
1855 init_image_offsets(DEBUG_HEADER_ENTRY_TXM, &vm_txm_offsets);
1856
1857 dynamic_memory_begin = ROUND_TWIG(dynamic_memory_begin);
1858
1859 /* TODO: CONFIG_XNUPOST CTRR test */
1860
1861 pmap_bootstrap(dynamic_memory_begin);
1862
1863 disable_preemption();
1864
1865 /*
1866 * Initialize l3 page table pages :
1867 * cover this address range:
1868 * 2MB + FrameBuffer size + 10MB for each 256MB segment
1869 *
1870 * Note: This does not allocate L3 page tables, since page tables for all static
1871 * memory is allocated and inserted into the hierarchy by the SPTM beforehand.
1872 * Instead, this code simply walks the page tables to find those pre-allocated
1873 * tables and allocates PTD objects for them.
1874 */
1875
1876 mem_segments = (mem_size + 0x0FFFFFFF) >> 28;
1877
1878 va_l1 = dynamic_memory_begin;
1879 va_l1_end = va_l1 + ((2 + (mem_segments * 10)) << 20);
1880 va_l1_end += round_page(args->Video.v_height * args->Video.v_rowBytes);
1881 va_l1_end = (va_l1_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL;
1882
1883 cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
1884
1885 while (va_l1 < va_l1_end) {
1886 va_l2 = va_l1;
1887
1888 if (((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE) < va_l1) {
1889 /* If this is the last L1 entry, it must cover the last mapping. */
1890 va_l2_end = va_l1_end;
1891 } else {
1892 va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE, va_l1_end);
1893 }
1894
1895 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
1896
1897 while (va_l2 < va_l2_end) {
1898 /* Obtain pre-allocated page and setup L3 Table TTE in L2 */
1899 tt_entry_t *ttp = pmap_tt2e(kernel_pmap, va_l2);
1900 pt_entry_t *ptp = (pt_entry_t *)phystokv(tte_to_pa(*ttp));
1901 pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE);
1902
1903 va_l2 += ARM_TT_L2_SIZE;
1904 cpu_l2_tte++;
1905 }
1906
1907 va_l1 = va_l2_end;
1908 cpu_l1_tte++;
1909 }
1910
1911 /*
1912 * Initialize l3 page table pages :
1913 * cover this address range:
1914 * ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - PE_EARLY_BOOT_VA) to VM_MAX_KERNEL_ADDRESS
1915 *
1916 * Note: This does not allocate L3 page tables, since page tables for all static
1917 * memory is allocated and inserted into the hierarchy by the SPTM beforehand.
1918 * Instead, this code simply walks the page tables to find those pre-allocated
1919 * tables and allocates PTD objects for them.
1920 */
1921 va_l1 = (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - PE_EARLY_BOOT_VA;
1922 va_l1_end = VM_MAX_KERNEL_ADDRESS;
1923
1924 cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
1925
1926 while (va_l1 < va_l1_end) {
1927 va_l2 = va_l1;
1928
1929 if (((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE) < va_l1) {
1930 /* If this is the last L1 entry, it must cover the last mapping. */
1931 va_l2_end = va_l1_end;
1932 } else {
1933 va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE, va_l1_end);
1934 }
1935
1936 cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
1937
1938 while (va_l2 < va_l2_end) {
1939 /* Obtain pre-allocated page and setup L3 Table TTE in L2 */
1940 tt_entry_t *ttp = pmap_tt2e(kernel_pmap, va_l2);
1941 pt_entry_t *ptp = (pt_entry_t *)phystokv(tte_to_pa(*ttp));
1942 pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE);
1943
1944 va_l2 += ARM_TT_L2_SIZE;
1945 cpu_l2_tte++;
1946 }
1947
1948 va_l1 = va_l2_end;
1949 cpu_l1_tte++;
1950 }
1951
1952 /*
1953 * Adjust avail_start so that the range that the VM owns
1954 * starts on a PAGE_SIZE aligned boundary.
1955 */
1956 avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK;
1957
1958 /* TODO pmap_static_allocations_done() */
1959
1960 first_avail = avail_start;
1961 patch_low_glo_static_region(first_avail_phys, avail_start - first_avail_phys);
1962 enable_preemption();
1963 }
1964
1965 /*
1966 * Returns true if the address lies within __TEXT, __TEXT_EXEC or __DATA_CONST
1967 * segment range. This is what [vm_kernel_stext, vm_kernel_etext) used to cover.
1968 * The segments together may not make a continuous address space anymore and so
1969 * individual intervals are inspected.
1970 */
1971 bool
kernel_text_contains(vm_offset_t addr)1972 kernel_text_contains(vm_offset_t addr)
1973 {
1974 if (segTEXTB <= addr && addr < (segTEXTB + segSizeTEXT)) {
1975 return true;
1976 }
1977 if (segTEXTEXECB <= addr && addr < (segTEXTEXECB + segSizeTEXTEXEC)) {
1978 return true;
1979 }
1980 return segDATACONSTB <= addr && addr < (segDATACONSTB + segSizeDATACONST);
1981 }
1982