1 /*
2 * Copyright (c) 2007-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 #include <debug.h>
33 #include <mach_ldebug.h>
34 #include <mach_kdp.h>
35
36 #include <kern/misc_protos.h>
37 #include <kern/thread.h>
38 #include <kern/timer_queue.h>
39 #include <kern/processor.h>
40 #include <kern/startup.h>
41 #include <kern/debug.h>
42 #include <kern/monotonic.h>
43 #include <prng/random.h>
44 #include <kern/ecc.h>
45 #include <machine/machine_routines.h>
46 #include <machine/commpage.h>
47 #include <machine/config.h>
48 #if HIBERNATION
49 #include <machine/pal_hibernate.h>
50 #endif /* HIBERNATION */
51 /* ARM64_TODO unify boot.h */
52 #if __arm64__
53 #include <pexpert/arm64/apt_msg.h>
54 #include <pexpert/arm64/boot.h>
55 #include <arm64/amcc_rorgn.h>
56 #else
57 #error Unsupported arch
58 #endif
59 #include <pexpert/arm/consistent_debug.h>
60 #include <pexpert/device_tree.h>
61 #include <arm64/proc_reg.h>
62 #include <arm/pmap.h>
63 #include <arm/caches_internal.h>
64 #include <arm/cpu_internal.h>
65 #include <arm/cpu_data_internal.h>
66 #include <arm/cpuid_internal.h>
67 #include <arm/misc_protos.h>
68 #include <arm/machine_cpu.h>
69 #include <arm/rtclock.h>
70 #include <vm/vm_map.h>
71
72 #include <libkern/kernel_mach_header.h>
73 #include <libkern/stack_protector.h>
74 #include <libkern/section_keywords.h>
75 #include <san/kasan.h>
76 #include <sys/kdebug.h>
77
78 #include <pexpert/pexpert.h>
79
80 #include <console/serial_protos.h>
81
82 #if CONFIG_TELEMETRY
83 #include <kern/telemetry.h>
84 #endif
85
86 #if KPERF
87 #include <kperf/kptimer.h>
88 #endif /* KPERF */
89
90 #if HIBERNATION
91 #include <IOKit/IOPlatformExpert.h>
92 #endif /* HIBERNATION */
93
94 extern void patch_low_glo(void);
95 extern int serial_init(void);
96 extern void sleep_token_buffer_init(void);
97
98 extern vm_offset_t intstack_top;
99 #if __arm64__
100 extern vm_offset_t excepstack_top;
101 #endif
102
103 extern const char version[];
104 extern const char version_variant[];
105 extern int disableConsoleOutput;
106
107 int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
108 int pc_trace_cnt = PC_TRACE_BUF_SIZE;
109 int debug_task;
110
111 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
112
113 TUNABLE(bool, restore_boot, "-restore", false);
114
115 #if HAS_BP_RET
116 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
117 uint32_t bp_ret = 3;
118 extern void set_bp_ret(void);
119 #endif
120
121 #if SCHED_HYGIENE_DEBUG
122
123 #if XNU_PLATFORM_iPhoneOS
124 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
125 #elif XNU_PLATFORM_XROS
126 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
127 #else
128 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 0xd0000 /* 35.499ms */
129 #endif /* XNU_PLATFORM_iPhoneOS */
130
131 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, interrupt_masked_debug_mode,
132 "machine-timeouts", "interrupt-masked-debug-mode",
133 "interrupt-masked-debug-mode",
134 SCHED_HYGIENE_MODE_PANIC,
135 TUNABLE_DT_CHECK_CHOSEN);
136
137 MACHINE_TIMEOUT_DEV_WRITEABLE(interrupt_masked_timeout, "interrupt-masked",
138 DEFAULT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
139 NULL);
140 #if __arm64__
141 #define SSHOT_INTERRUPT_MASKED_TIMEOUT 0xf9999 /* 64-bit: 42.599ms */
142 #endif
143 MACHINE_TIMEOUT_DEV_WRITEABLE(stackshot_interrupt_masked_timeout, "sshot-interrupt-masked",
144 SSHOT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
145 NULL);
146 #undef SSHOT_INTERRUPT_MASKED_TIMEOUT
147 #endif
148
149 /*
150 * A 6-second timeout will give the watchdog code a chance to run
151 * before a panic is triggered by the xcall routine.
152 */
153 #define XCALL_ACK_TIMEOUT_NS ((uint64_t) 6000000000)
154 uint64_t xcall_ack_timeout_abstime;
155
156 #ifndef __BUILDING_XNU_LIBRARY__
157 #define BOOTARGS_SECTION_ATTR __attribute__((section("__DATA, __const")))
158 #else /* __BUILDING_XNU_LIBRARY__ */
159 /* Special segments are not used when building for user-mode */
160 #define BOOTARGS_SECTION_ATTR
161 #endif /* __BUILDING_XNU_LIBRARY__ */
162
163 boot_args const_boot_args BOOTARGS_SECTION_ATTR;
164 boot_args *BootArgs BOOTARGS_SECTION_ATTR;
165
166 TUNABLE(uint32_t, arm_diag, "diag", 0);
167 #ifdef APPLETYPHOON
168 static unsigned cpus_defeatures = 0x0;
169 extern void cpu_defeatures_set(unsigned int);
170 #endif
171
172 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
173 extern volatile boolean_t arm64_stall_sleep;
174 #endif
175
176 extern boolean_t force_immediate_debug_halt;
177
178 #if HAS_APPLE_PAC
179 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
180 #endif
181
182 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
183 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
184 SECURITY_READ_ONLY_LATE(ppnum_t) pmap_first_pnum;
185
186 SECURITY_READ_ONLY_LATE(bool) serial_console_enabled = false;
187
188 #if HAS_ARM_FEAT_SME
189 static SECURITY_READ_ONLY_LATE(bool) enable_sme = true;
190 #endif
191
192 #if APPLEVIRTUALPLATFORM
193 SECURITY_READ_ONLY_LATE(vm_offset_t) reset_vector_vaddr = 0;
194 #endif /* APPLEVIRTUALPLATFORM */
195
196 /*
197 * Forward definition
198 */
199 void arm_init(boot_args * args);
200
201 #if __arm64__
202 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
203
204 extern void configure_misc_apple_boot_args(void);
205 extern void configure_misc_apple_regs(bool is_boot_cpu);
206 extern void configure_timer_apple_regs(void);
207 extern void configure_late_apple_regs(bool cold_boot);
208 #endif /* __arm64__ */
209
210
211 /*
212 * JOP rebasing
213 */
214
215 #define dyldLogFunc(msg, ...)
216 #include <mach/dyld_kernel_fixups.h>
217
218 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
219 extern uint32_t __thread_starts_sect_end[] __asm("section$end$__TEXT$__thread_starts");
220 #if defined(HAS_APPLE_PAC)
221 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
222 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
223 #endif /* defined(HAS_APPLE_PAC) */
224
225 extern vm_offset_t vm_kernel_slide;
226 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
227 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
228 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
229
230 static void
arm_slide_rebase_and_sign_image(void)231 arm_slide_rebase_and_sign_image(void)
232 {
233 kernel_mach_header_t *k_mh, *kc_mh = NULL;
234 kernel_segment_command_t *seg;
235 uintptr_t slide;
236
237 k_mh = &_mh_execute_header;
238 if (kernel_mach_header_is_in_fileset(k_mh)) {
239 /*
240 * The kernel is part of a MH_FILESET kernel collection, determine slide
241 * based on first segment's mach-o vmaddr (requires first kernel load
242 * command to be LC_SEGMENT_64 of the __TEXT segment)
243 */
244 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
245 assert(seg->cmd == LC_SEGMENT_KERNEL);
246 slide = (uintptr_t)k_mh - seg->vmaddr;
247
248 /*
249 * The kernel collection linker guarantees that the boot collection mach
250 * header vmaddr is the hardcoded kernel link address (as specified to
251 * ld64 when linking the kernel).
252 */
253 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
254 assert(kc_mh->filetype == MH_FILESET);
255
256 /*
257 * rebase and sign jops
258 * Note that we can't call any functions before this point, so
259 * we have to hard-code the knowledge that the base of the KC
260 * is the KC's mach-o header. This would change if any
261 * segment's VA started *before* the text segment
262 * (as the HIB segment does on x86).
263 */
264 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
265 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
266
267 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
268
269 /*
270 * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
271 * collection, so adjust them now, and determine the vmaddr range
272 * covered by read-only segments for the CTRR rorgn.
273 */
274 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
275 (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
276 (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
277 NULL, NULL, NULL);
278 #if defined(HAS_APPLE_PAC)
279 OSRuntimeSignStructorsInFileset(kc_mh);
280 #endif /* defined(HAS_APPLE_PAC) */
281 } else {
282 /*
283 * Static kernelcache: iBoot slid kernel MachO vmaddrs, determine slide
284 * using hardcoded kernel link address
285 */
286 slide = (uintptr_t)k_mh - VM_KERNEL_LINK_ADDRESS;
287
288 /* rebase and sign jops */
289 static_kernelcache = &__thread_starts_sect_end[0] != &__thread_starts_sect_start[0];
290 if (static_kernelcache) {
291 rebase_threaded_starts( &__thread_starts_sect_start[0],
292 &__thread_starts_sect_end[0],
293 (uintptr_t)k_mh, (uintptr_t)k_mh - slide, slide);
294 }
295 #if defined(HAS_APPLE_PAC)
296 OSRuntimeSignStructors(&_mh_execute_header);
297 #endif /* defined(HAS_APPLE_PAC) */
298 }
299
300
301 /*
302 * Initialize slide global here to avoid duplicating this logic in
303 * arm_vm_init()
304 */
305 vm_kernel_slide = slide;
306 }
307
308 void
arm_auxkc_init(void * mh,void * base)309 arm_auxkc_init(void *mh, void *base)
310 {
311 /*
312 * The kernel collection linker guarantees that the lowest vmaddr in an
313 * AuxKC collection is 0 (but note that the mach header is higher up since
314 * RW segments precede RO segments in the AuxKC).
315 */
316 uintptr_t slide = (uintptr_t)base;
317 kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
318
319 assert(akc_mh->filetype == MH_FILESET);
320 PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
321
322 /* rebase and sign jops */
323 const void *collection_base_pointers[KCNumKinds];
324 memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
325 kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
326
327 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
328 (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
329 (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
330 (uintptr_t *)&segHIGHESTNLEAuxKC);
331 #if defined(HAS_APPLE_PAC)
332 OSRuntimeSignStructorsInFileset(akc_mh);
333 #endif /* defined(HAS_APPLE_PAC) */
334 }
335
336 /*
337 * Routine: arm_init
338 * Function: Runs on the boot CPU, once, on entry from iBoot.
339 */
340
341 __startup_func
342 void
arm_init(boot_args * args)343 arm_init(
344 boot_args *args)
345 {
346 unsigned int maxmem;
347 uint32_t memsize;
348 uint64_t xmaxmem;
349 thread_t thread;
350 DTEntry chosen = NULL;
351 unsigned int dt_entry_size = 0;
352
353 arm_slide_rebase_and_sign_image();
354
355 /* If kernel integrity is supported, use a constant copy of the boot args. */
356 const_boot_args = *args;
357 BootArgs = args = &const_boot_args;
358
359 #if APPLEVIRTUALPLATFORM
360 reset_vector_vaddr = (vm_offset_t) &LowResetVectorBase;
361 #endif /* APPLEVIRTUALPLATFORM */
362
363 cpu_data_init(&BootCpuData);
364 #if defined(HAS_APPLE_PAC)
365 /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
366 BootCpuData.rop_key = ml_default_rop_pid();
367 BootCpuData.jop_key = ml_default_jop_pid();
368 #endif /* defined(HAS_APPLE_PAC) */
369
370 PE_init_platform(FALSE, args); /* Get platform expert set up */
371
372 #if __arm64__
373 configure_timer_apple_regs();
374 wfe_timeout_configure();
375 wfe_timeout_init();
376
377 configure_misc_apple_boot_args();
378 configure_misc_apple_regs(true);
379
380 #if HAS_UPSI_FAILURE_INJECTION
381 /* UPSI (Universal Panic and Stall Injection) Logic
382 * iBoot/XNU are both configured for failure injection at specific stages
383 * The injected failure and stage is populated through EDT properties by iBoot
384 *
385 * iBoot populates the EDT properties for XNU based upon PMU scratch bits
386 * This is done because the EDT is available sooner in XNU than the PMU Kext
387 */
388 uint64_t const *upsi_info = NULL;
389
390 /* Not usable TUNABLE here because TUNABLEs are parsed at a later point. */
391 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
392 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
393 }
394
395 /* Check if there is a requested injection stage */
396 if (SecureDTGetProperty(chosen, "injection_stage", (void const **)&upsi_info,
397 &dt_entry_size) == kSuccess) {
398 assert3u(dt_entry_size, ==, 8);
399 xnu_upsi_injection_stage = *upsi_info;
400 }
401
402 /* Check if there is a requested injection action */
403 if (SecureDTGetProperty(chosen, "injection_action", (void const **)&upsi_info,
404 &dt_entry_size) == kSuccess) {
405 assert3u(dt_entry_size, ==, 8);
406 xnu_upsi_injection_action = *upsi_info;
407 }
408
409 check_for_failure_injection(XNU_STAGE_ARM_INIT);
410
411 chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
412 dt_entry_size = 0;
413 #endif // HAS_UPSI_FAILURE_INJECTION
414
415
416 {
417 /*
418 * Select the advertised kernel page size.
419 */
420 if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
421 /*
422 * arm64 device with > 1GB of RAM:
423 * kernel uses 16KB pages.
424 */
425 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
426 } else {
427 /*
428 * arm64 device with <= 1GB of RAM:
429 * kernel uses hardware page size
430 * (4KB for H6/H7, 16KB for H8+).
431 */
432 PAGE_SHIFT_CONST = ARM_PGSHIFT;
433 }
434
435 /* 32-bit apps always see 16KB page size */
436 page_shift_user32 = PAGE_MAX_SHIFT;
437 #ifdef APPLETYPHOON
438 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
439 if ((cpus_defeatures & 0xF) != 0) {
440 cpu_defeatures_set(cpus_defeatures & 0xF);
441 }
442 }
443 #endif
444 }
445 #endif
446 #if HAS_ARM_FEAT_SME
447 (void)PE_parse_boot_argn("enable_sme", &enable_sme, sizeof(enable_sme));
448 if (enable_sme) {
449 arm_sme_init(true);
450 }
451 #endif
452
453 ml_parse_cpu_topology();
454
455
456 master_cpu = ml_get_boot_cpu_number();
457 assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
458
459 BootCpuData.cpu_number = (unsigned short)master_cpu;
460 BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
461 BootCpuData.istackptr = &intstack_top;
462 #if __arm64__
463 BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
464 BootCpuData.excepstackptr = &excepstack_top;
465 #endif
466 CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
467 CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
468 + ((uintptr_t)&BootCpuData
469 - (uintptr_t)(args->virtBase)));
470
471 thread = thread_bootstrap();
472 thread->machine.CpuDatap = &BootCpuData;
473 thread->machine.pcpu_data_base_and_cpu_number =
474 ml_make_pcpu_base_and_cpu_number(0, BootCpuData.cpu_number);
475 machine_set_current_thread(thread);
476
477 /*
478 * Preemption is enabled for this thread so that it can lock mutexes without
479 * tripping the preemption check. In reality scheduling is not enabled until
480 * this thread completes, and there are no other threads to switch to, so
481 * preemption level is not really meaningful for the bootstrap thread.
482 */
483 thread->machine.preemption_count = 0;
484 cpu_bootstrap();
485
486 rtclock_early_init();
487
488 kernel_debug_string_early("kernel_startup_bootstrap");
489 kernel_startup_bootstrap();
490
491 /*
492 * Initialize the timer callout world
493 */
494 timer_call_init();
495
496 cpu_init();
497
498 processor_bootstrap();
499
500 if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
501 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
502 } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
503 xmaxmem = (uint64_t) memsize;
504 } else {
505 xmaxmem = 0;
506 }
507
508 #if SCHED_HYGIENE_DEBUG
509 {
510 int wdt_boot_arg = 0;
511 bool const wdt_disabled = (PE_parse_boot_argn("wdt", &wdt_boot_arg, sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1));
512
513 /* Disable if WDT is disabled */
514 if (wdt_disabled || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
515 interrupt_masked_debug_mode = SCHED_HYGIENE_MODE_OFF;
516 }
517 if (wdt_disabled || kern_feature_override(KF_PREEMPTION_DISABLED_DEBUG_OVRD)) {
518 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
519 }
520 }
521 #endif /* SCHED_HYGIENE_DEBUG */
522
523 nanoseconds_to_absolutetime(XCALL_ACK_TIMEOUT_NS, &xcall_ack_timeout_abstime);
524
525 #if HAS_BP_RET
526 PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
527 set_bp_ret(); // Apply branch predictor retention settings to boot CPU
528 #endif
529
530 PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
531
532 #if __ARM_PAN_AVAILABLE__
533 __builtin_arm_wsr("pan", 1);
534 #endif /* __ARM_PAN_AVAILABLE__ */
535
536 #if HAS_MTE
537 panic("This code path should never be hit: MTE devices should always flow through the SPTM-assisted init");
538 #endif /* HAS_MTE */
539
540 /*
541 * gPhysBase/Size only represent kernel-managed memory. These globals represent
542 * the actual DRAM base address and size as reported by iBoot through the
543 * device tree.
544 */
545 unsigned long const *dram_base;
546 unsigned long const *dram_size;
547
548 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
549 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
550 }
551
552 if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
553 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
554 }
555
556 if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
557 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
558 }
559
560 gDramBase = *dram_base;
561 gDramSize = *dram_size;
562 pmap_first_pnum = (ppnum_t)atop(gDramBase);
563
564 arm_vm_init(xmaxmem, args);
565
566 if (debug_boot_arg) {
567 patch_low_glo();
568 }
569
570 #if __arm64__ && WITH_CLASSIC_S2R
571 sleep_token_buffer_init();
572 #endif
573
574 PE_consistent_debug_inherit();
575
576 /* Setup debugging output. */
577 const unsigned int serial_exists = serial_init();
578 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
579 kprintf("kprintf initialized\n");
580
581 serialmode = 0;
582 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
583 /* Do we want a serial keyboard and/or console? */
584 kprintf("Serial mode specified: %08X\n", serialmode);
585 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
586 enable_dklog_serial_output = restore_boot || (serialmode & SERIALMODE_DKLOG) != 0;
587 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
588 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
589 if (force_sync) {
590 serialmode |= SERIALMODE_SYNCDRAIN;
591 kprintf(
592 "WARNING: Forcing uart driver to output synchronously."
593 "printf()s/IOLogs will impact kernel performance.\n"
594 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
595 }
596 }
597 /* If on-demand is selected, disable serials until reception. */
598 bool on_demand = !!(serialmode & SERIALMODE_ON_DEMAND);
599 if (on_demand && !(serialmode & SERIALMODE_INPUT)) {
600 kprintf(
601 "WARNING: invalid serial boot-args : ON_DEMAND (0x%x) flag "
602 "requires INPUT(0x%x). Ignoring ON_DEMAND.\n",
603 SERIALMODE_ON_DEMAND, SERIALMODE_INPUT
604 );
605 on_demand = 0;
606 }
607 serial_set_on_demand(on_demand);
608 }
609 if (kern_feature_override(KF_SERIAL_OVRD)) {
610 serialmode = 0;
611 }
612
613 /* Start serial if requested and a serial device was enumerated in serial_init(). */
614 if ((serialmode & SERIALMODE_OUTPUT) && serial_exists) {
615 serial_console_enabled = true;
616 (void)switch_to_serial_console(); /* Switch into serial mode from video console */
617 disableConsoleOutput = FALSE; /* Allow printfs to happen */
618 }
619 PE_create_console();
620
621 /* setup console output */
622 PE_init_printf(FALSE);
623
624 #if __arm64__
625 #if DEBUG
626 dump_kva_space();
627 #endif
628 #endif
629
630 cpu_machine_idle_init(TRUE);
631
632 PE_init_platform(TRUE, &BootCpuData);
633
634 /* Initialize the debug infrastructure system-wide and on the local core. */
635 pe_arm_debug_init_early(&BootCpuData);
636
637 #if __arm64__
638 extern bool cpu_config_correct;
639 if (!cpu_config_correct) {
640 panic("The cpumask=N boot arg cannot be used together with cpus=N, and the boot CPU must be enabled");
641 }
642
643 ml_map_cpu_pio();
644
645 #if APPLE_ARM64_ARCH_FAMILY
646 configure_late_apple_regs(true);
647 #endif
648
649 #endif
650
651 cpu_timebase_init(TRUE);
652
653 #if KPERF
654 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
655 kptimer_curcpu_up();
656 #endif /* KPERF */
657
658 PE_init_cpu();
659 #if __arm64__
660 apt_msg_init();
661 apt_msg_init_cpu();
662 #endif
663 fiq_context_init(TRUE);
664
665
666 #if HIBERNATION
667 pal_hib_init();
668 #endif /* HIBERNATION */
669
670 /*
671 * Initialize the stack protector for all future calls
672 * to C code. Since kernel_bootstrap() eventually
673 * switches stack context without returning through this
674 * function, we do not risk failing the check even though
675 * we mutate the guard word during execution.
676 */
677 __stack_chk_guard = (unsigned long)early_random();
678 /* Zero a byte of the protector to guard
679 * against string vulnerabilities
680 */
681 __stack_chk_guard &= ~(0xFFULL << 8);
682 machine_startup(args);
683 }
684
685 /*
686 * Routine: arm_init_cpu
687 * Function:
688 * Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
689 */
690
691 void
arm_init_cpu(cpu_data_t * cpu_data_ptr,uint64_t __unused hib_header_phys)692 arm_init_cpu(
693 cpu_data_t *cpu_data_ptr,
694 uint64_t __unused hib_header_phys)
695 {
696 #if __ARM_PAN_AVAILABLE__
697 __builtin_arm_wsr("pan", 1);
698 #endif
699
700 #ifdef __arm64__
701 configure_timer_apple_regs();
702 configure_misc_apple_regs(false);
703 #endif
704 #if HAS_ARM_FEAT_SME
705 if (enable_sme) {
706 arm_sme_init(false);
707 }
708 #endif
709
710 os_atomic_andnot(&cpu_data_ptr->cpu_flags, SleepState, relaxed);
711
712
713 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
714
715 #if APPLE_ARM64_ARCH_FAMILY
716 configure_late_apple_regs(false);
717 #endif
718
719 #if HIBERNATION
720 if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate) && ml_is_quiescing()) {
721 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
722 extern uint64_t wake_abstime;
723 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
724
725 // since the hw clock stops ticking across hibernation, we need to apply an offset;
726 // iBoot computes this offset for us and passes it via the hibernation header
727 extern uint64_t hwclock_conttime_offset;
728 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
729
730 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
731 // fix up its preemption count
732 unsigned int expected_preemption_count = (gEnforcePlatformActionSafety ? 2 : 1);
733 if (get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread) !=
734 expected_preemption_count) {
735 panic("unexpected preemption count %u on boot cpu thread (should be %u)",
736 get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread),
737 expected_preemption_count);
738 }
739 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
740 }
741 #endif /* HIBERNATION */
742
743 #if __arm64__
744 wfe_timeout_init();
745 pmap_clear_user_ttb();
746 flush_mmu_tlb();
747 #endif
748
749 cpu_machine_idle_init(FALSE);
750
751 cpu_init();
752
753 #ifdef APPLETYPHOON
754 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
755 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
756 }
757 #endif
758 /* Initialize the timebase before serial_init, as some serial
759 * drivers use mach_absolute_time() to implement rate control
760 */
761 cpu_timebase_init(FALSE);
762
763 #if KPERF
764 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
765 kptimer_curcpu_up();
766 #endif /* KPERF */
767
768 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
769 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
770 /*
771 * Prevent CPUs from going into deep sleep until all
772 * CPUs are ready to do so.
773 */
774 arm64_stall_sleep = TRUE;
775 #endif
776 serial_init();
777 PE_init_platform(TRUE, NULL);
778 commpage_update_timebase();
779 }
780 PE_init_cpu();
781 #if __arm64__
782 apt_msg_init_cpu();
783 #endif
784
785 fiq_context_init(TRUE);
786 cpu_data_ptr->rtcPop = EndOfAllTime;
787 timer_resync_deadlines();
788
789 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
790 bool should_kprintf = processor_should_kprintf(processor, true);
791
792 /* Start tracing (secondary CPU). */
793 #if DEVELOPMENT || DEBUG
794 PE_arm_debug_enable_trace(should_kprintf);
795 #endif /* DEVELOPMENT || DEBUG */
796
797 #if KERNEL_INTEGRITY_KTRR || KERNEL_INTEGRITY_CTRR || KERNEL_INTEGRITY_PV_CTRR
798 rorgn_validate_core();
799 #endif
800
801
802 if (should_kprintf) {
803 kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
804 }
805
806 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
807 if (kdebug_enable == 0) {
808 __kdebug_only uint64_t elapsed = kdebug_wake();
809 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
810 }
811
812 #if CONFIG_TELEMETRY
813 bootprofile_wake_from_sleep();
814 #endif /* CONFIG_TELEMETRY */
815 }
816 #if CONFIG_CPU_COUNTERS
817 mt_wake_per_core();
818 #endif /* CONFIG_CPU_COUNTERS */
819
820 #if defined(KERNEL_INTEGRITY_CTRR)
821 if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
822 lck_spin_lock(&ctrr_cpu_start_lck);
823 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
824 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
825 lck_spin_unlock(&ctrr_cpu_start_lck);
826 }
827 #endif
828
829
830 secondary_cpu_main(NULL);
831 }
832
833 /*
834 * Routine: arm_init_idle_cpu
835 * Function: Resume from non-retention WFI. Called from the reset vector.
836 */
837 void __attribute__((noreturn))
arm_init_idle_cpu(cpu_data_t * cpu_data_ptr)838 arm_init_idle_cpu(
839 cpu_data_t *cpu_data_ptr)
840 {
841 #if __ARM_PAN_AVAILABLE__
842 __builtin_arm_wsr("pan", 1);
843 #endif
844
845 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
846
847 #if __arm64__
848 wfe_timeout_init();
849 pmap_clear_user_ttb();
850 flush_mmu_tlb();
851 #endif
852
853 #ifdef APPLETYPHOON
854 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
855 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
856 }
857 #endif
858
859 /*
860 * Update the active debug object to reflect that debug registers have been reset.
861 * This will force any thread with active debug state to resync the debug registers
862 * if it returns to userspace on this CPU.
863 */
864 if (cpu_data_ptr->cpu_user_debug != NULL) {
865 arm_debug_set(NULL);
866 }
867
868 fiq_context_init(FALSE);
869
870 cpu_idle_exit(TRUE);
871 }
872