1 /*
2 * Copyright (c) 2007-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 #include <debug.h>
33 #include <mach_ldebug.h>
34 #include <mach_kdp.h>
35
36 #include <kern/misc_protos.h>
37 #include <kern/thread.h>
38 #include <kern/timer_queue.h>
39 #include <kern/processor.h>
40 #include <kern/startup.h>
41 #include <kern/debug.h>
42 #include <kern/monotonic.h>
43 #include <prng/random.h>
44 #include <kern/ecc.h>
45 #include <machine/machine_routines.h>
46 #include <machine/commpage.h>
47 #include <machine/config.h>
48 #if HIBERNATION
49 #include <machine/pal_hibernate.h>
50 #endif /* HIBERNATION */
51 /* ARM64_TODO unify boot.h */
52 #if __arm64__
53 #include <pexpert/arm64/boot.h>
54 #include <arm64/amcc_rorgn.h>
55 #else
56 #error Unsupported arch
57 #endif
58 #include <pexpert/arm/consistent_debug.h>
59 #include <pexpert/device_tree.h>
60 #include <arm64/proc_reg.h>
61 #include <arm/pmap.h>
62 #include <arm/caches_internal.h>
63 #include <arm/cpu_internal.h>
64 #include <arm/cpu_data_internal.h>
65 #include <arm/cpuid_internal.h>
66 #include <arm/misc_protos.h>
67 #include <arm/machine_cpu.h>
68 #include <arm/rtclock.h>
69 #include <vm/vm_map.h>
70
71 #include <libkern/kernel_mach_header.h>
72 #include <libkern/stack_protector.h>
73 #include <libkern/section_keywords.h>
74 #include <san/kasan.h>
75 #include <sys/kdebug.h>
76
77 #include <pexpert/pexpert.h>
78
79 #include <console/serial_protos.h>
80
81 #if CONFIG_TELEMETRY
82 #include <kern/telemetry.h>
83 #endif
84
85 #if KPERF
86 #include <kperf/kptimer.h>
87 #endif /* KPERF */
88
89 #if HIBERNATION
90 #include <IOKit/IOPlatformExpert.h>
91 #endif /* HIBERNATION */
92
93 extern void patch_low_glo(void);
94 extern int serial_init(void);
95 extern void sleep_token_buffer_init(void);
96
97 extern vm_offset_t intstack_top;
98 #if __arm64__
99 extern vm_offset_t excepstack_top;
100 #endif
101
102 extern const char version[];
103 extern const char version_variant[];
104 extern int disableConsoleOutput;
105
106 int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
107 int pc_trace_cnt = PC_TRACE_BUF_SIZE;
108 int debug_task;
109
110 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
111
112 TUNABLE(bool, restore_boot, "-restore", false);
113
114 #if HAS_BP_RET
115 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
116 uint32_t bp_ret = 3;
117 extern void set_bp_ret(void);
118 #endif
119
120 #if SCHED_HYGIENE_DEBUG
121 boolean_t sched_hygiene_debug_pmc = 1;
122 #endif
123
124 #if SCHED_HYGIENE_DEBUG
125
126 #if XNU_PLATFORM_iPhoneOS
127 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
128 #elif XNU_PLATFORM_XROS
129 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
130 #else
131 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 0xd0000 /* 35.499ms */
132 #endif /* XNU_PLATFORM_iPhoneOS */
133
134 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, interrupt_masked_debug_mode,
135 "machine-timeouts", "interrupt-masked-debug-mode",
136 "interrupt-masked-debug-mode",
137 SCHED_HYGIENE_MODE_PANIC,
138 TUNABLE_DT_CHECK_CHOSEN);
139
140 MACHINE_TIMEOUT_DEV_WRITEABLE(interrupt_masked_timeout, "interrupt-masked",
141 DEFAULT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
142 NULL);
143 #if __arm64__
144 #define SSHOT_INTERRUPT_MASKED_TIMEOUT 0xf9999 /* 64-bit: 42.599ms */
145 #endif
146 MACHINE_TIMEOUT_DEV_WRITEABLE(stackshot_interrupt_masked_timeout, "sshot-interrupt-masked",
147 SSHOT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
148 NULL);
149 #undef SSHOT_INTERRUPT_MASKED_TIMEOUT
150 #endif
151
152 /*
153 * A 6-second timeout will give the watchdog code a chance to run
154 * before a panic is triggered by the xcall routine.
155 */
156 #define XCALL_ACK_TIMEOUT_NS ((uint64_t) 6000000000)
157 uint64_t xcall_ack_timeout_abstime;
158
159 boot_args const_boot_args __attribute__((section("__DATA, __const")));
160 boot_args *BootArgs __attribute__((section("__DATA, __const")));
161
162 TUNABLE(uint32_t, arm_diag, "diag", 0);
163 #ifdef APPLETYPHOON
164 static unsigned cpus_defeatures = 0x0;
165 extern void cpu_defeatures_set(unsigned int);
166 #endif
167
168 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
169 extern volatile boolean_t arm64_stall_sleep;
170 #endif
171
172 extern boolean_t force_immediate_debug_halt;
173
174 #if HAS_APPLE_PAC
175 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
176 #endif
177
178 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
179 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
180
181 SECURITY_READ_ONLY_LATE(bool) serial_console_enabled = false;
182
183 #if HAS_ARM_FEAT_SME
184 static SECURITY_READ_ONLY_LATE(bool) enable_sme = true;
185 #endif
186
187 #if APPLEVIRTUALPLATFORM
188 SECURITY_READ_ONLY_LATE(vm_offset_t) reset_vector_vaddr = 0;
189 #endif /* APPLEVIRTUALPLATFORM */
190
191
192 /*
193 * Forward definition
194 */
195 void arm_init(boot_args * args);
196
197 #if __arm64__
198 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
199
200 extern void configure_misc_apple_boot_args(void);
201 extern void configure_misc_apple_regs(bool is_boot_cpu);
202 extern void configure_timer_apple_regs(void);
203 extern void configure_late_apple_regs(bool cold_boot);
204 #endif /* __arm64__ */
205
206
207 /*
208 * JOP rebasing
209 */
210
211 #define dyldLogFunc(msg, ...)
212 #include <mach/dyld_kernel_fixups.h>
213
214 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
215 extern uint32_t __thread_starts_sect_end[] __asm("section$end$__TEXT$__thread_starts");
216 #if defined(HAS_APPLE_PAC)
217 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
218 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
219 #endif /* defined(HAS_APPLE_PAC) */
220
221 extern vm_offset_t vm_kernel_slide;
222 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
223 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
224 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
225
226 static void
arm_slide_rebase_and_sign_image(void)227 arm_slide_rebase_and_sign_image(void)
228 {
229 kernel_mach_header_t *k_mh, *kc_mh = NULL;
230 kernel_segment_command_t *seg;
231 uintptr_t slide;
232
233 k_mh = &_mh_execute_header;
234 if (kernel_mach_header_is_in_fileset(k_mh)) {
235 /*
236 * The kernel is part of a MH_FILESET kernel collection, determine slide
237 * based on first segment's mach-o vmaddr (requires first kernel load
238 * command to be LC_SEGMENT_64 of the __TEXT segment)
239 */
240 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
241 assert(seg->cmd == LC_SEGMENT_KERNEL);
242 slide = (uintptr_t)k_mh - seg->vmaddr;
243
244 /*
245 * The kernel collection linker guarantees that the boot collection mach
246 * header vmaddr is the hardcoded kernel link address (as specified to
247 * ld64 when linking the kernel).
248 */
249 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
250 assert(kc_mh->filetype == MH_FILESET);
251
252 /*
253 * rebase and sign jops
254 * Note that we can't call any functions before this point, so
255 * we have to hard-code the knowledge that the base of the KC
256 * is the KC's mach-o header. This would change if any
257 * segment's VA started *before* the text segment
258 * (as the HIB segment does on x86).
259 */
260 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
261 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
262
263 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
264
265 /*
266 * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
267 * collection, so adjust them now, and determine the vmaddr range
268 * covered by read-only segments for the CTRR rorgn.
269 */
270 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
271 (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
272 (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
273 NULL, NULL, NULL);
274 #if defined(HAS_APPLE_PAC)
275 OSRuntimeSignStructorsInFileset(kc_mh);
276 #endif /* defined(HAS_APPLE_PAC) */
277 } else {
278 /*
279 * Static kernelcache: iBoot slid kernel MachO vmaddrs, determine slide
280 * using hardcoded kernel link address
281 */
282 slide = (uintptr_t)k_mh - VM_KERNEL_LINK_ADDRESS;
283
284 /* rebase and sign jops */
285 static_kernelcache = &__thread_starts_sect_end[0] != &__thread_starts_sect_start[0];
286 if (static_kernelcache) {
287 rebase_threaded_starts( &__thread_starts_sect_start[0],
288 &__thread_starts_sect_end[0],
289 (uintptr_t)k_mh, (uintptr_t)k_mh - slide, slide);
290 }
291 #if defined(HAS_APPLE_PAC)
292 OSRuntimeSignStructors(&_mh_execute_header);
293 #endif /* defined(HAS_APPLE_PAC) */
294 }
295
296
297 /*
298 * Initialize slide global here to avoid duplicating this logic in
299 * arm_vm_init()
300 */
301 vm_kernel_slide = slide;
302 }
303
304 void
arm_auxkc_init(void * mh,void * base)305 arm_auxkc_init(void *mh, void *base)
306 {
307 /*
308 * The kernel collection linker guarantees that the lowest vmaddr in an
309 * AuxKC collection is 0 (but note that the mach header is higher up since
310 * RW segments precede RO segments in the AuxKC).
311 */
312 uintptr_t slide = (uintptr_t)base;
313 kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
314
315 assert(akc_mh->filetype == MH_FILESET);
316 PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
317
318 /* rebase and sign jops */
319 const void *collection_base_pointers[KCNumKinds];
320 memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
321 kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
322
323 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
324 (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
325 (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
326 (uintptr_t *)&segHIGHESTNLEAuxKC);
327 #if defined(HAS_APPLE_PAC)
328 OSRuntimeSignStructorsInFileset(akc_mh);
329 #endif /* defined(HAS_APPLE_PAC) */
330 }
331
332 /*
333 * Routine: arm_setup_pre_sign
334 * Function: Perform HW initialization that must happen ahead of the first PAC sign
335 * operation.
336 */
337 static void
arm_setup_pre_sign(void)338 arm_setup_pre_sign(void)
339 {
340 #if __arm64__
341 /* DATA TBI, if enabled, affects the number of VA bits that contain the signature */
342 arm_set_kernel_tbi();
343 #endif /* __arm64 */
344 }
345
346 /*
347 * Routine: arm_init
348 * Function: Runs on the boot CPU, once, on entry from iBoot.
349 */
350
351 __startup_func
352 void
arm_init(boot_args * args)353 arm_init(
354 boot_args *args)
355 {
356 unsigned int maxmem;
357 uint32_t memsize;
358 uint64_t xmaxmem;
359 thread_t thread;
360 DTEntry chosen = NULL;
361 unsigned int dt_entry_size = 0;
362
363 arm_setup_pre_sign();
364
365 arm_slide_rebase_and_sign_image();
366
367 /* If kernel integrity is supported, use a constant copy of the boot args. */
368 const_boot_args = *args;
369 BootArgs = args = &const_boot_args;
370
371 #if APPLEVIRTUALPLATFORM
372 reset_vector_vaddr = (vm_offset_t) &LowResetVectorBase;
373 #endif /* APPLEVIRTUALPLATFORM */
374
375 cpu_data_init(&BootCpuData);
376 #if defined(HAS_APPLE_PAC)
377 /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
378 BootCpuData.rop_key = ml_default_rop_pid();
379 BootCpuData.jop_key = ml_default_jop_pid();
380 #endif /* defined(HAS_APPLE_PAC) */
381
382 PE_init_platform(FALSE, args); /* Get platform expert set up */
383
384 #if __arm64__
385 configure_timer_apple_regs();
386 wfe_timeout_configure();
387 wfe_timeout_init();
388
389 configure_misc_apple_boot_args();
390 configure_misc_apple_regs(true);
391
392 #if (DEVELOPMENT || DEBUG)
393 unsigned long const *platform_stall_ptr = NULL;
394
395 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
396 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
397 }
398
399 // Not usable TUNABLE here because TUNABLEs are parsed at a later point.
400 if (SecureDTGetProperty(chosen, "xnu_platform_stall", (void const **)&platform_stall_ptr,
401 &dt_entry_size) == kSuccess) {
402 xnu_platform_stall_value = *platform_stall_ptr;
403 }
404
405 platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_ARM_INIT);
406
407 chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
408 dt_entry_size = 0;
409 #endif
410
411
412 {
413 /*
414 * Select the advertised kernel page size.
415 */
416 if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
417 /*
418 * arm64 device with > 1GB of RAM:
419 * kernel uses 16KB pages.
420 */
421 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
422 } else {
423 /*
424 * arm64 device with <= 1GB of RAM:
425 * kernel uses hardware page size
426 * (4KB for H6/H7, 16KB for H8+).
427 */
428 PAGE_SHIFT_CONST = ARM_PGSHIFT;
429 }
430
431 /* 32-bit apps always see 16KB page size */
432 page_shift_user32 = PAGE_MAX_SHIFT;
433 #ifdef APPLETYPHOON
434 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
435 if ((cpus_defeatures & 0xF) != 0) {
436 cpu_defeatures_set(cpus_defeatures & 0xF);
437 }
438 }
439 #endif
440 }
441 #endif
442 #if HAS_ARM_FEAT_SME
443 (void)PE_parse_boot_argn("enable_sme", &enable_sme, sizeof(enable_sme));
444 if (enable_sme) {
445 arm_sme_init(true);
446 }
447 #endif
448
449 ml_parse_cpu_topology();
450
451
452 master_cpu = ml_get_boot_cpu_number();
453 assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
454
455 BootCpuData.cpu_number = (unsigned short)master_cpu;
456 BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
457 BootCpuData.istackptr = &intstack_top;
458 #if __arm64__
459 BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
460 BootCpuData.excepstackptr = &excepstack_top;
461 #endif
462 CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
463 CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
464 + ((uintptr_t)&BootCpuData
465 - (uintptr_t)(args->virtBase)));
466
467 thread = thread_bootstrap();
468 thread->machine.CpuDatap = &BootCpuData;
469 thread->machine.pcpu_data_base_and_cpu_number =
470 ml_make_pcpu_base_and_cpu_number(0, BootCpuData.cpu_number);
471 machine_set_current_thread(thread);
472
473 /*
474 * Preemption is enabled for this thread so that it can lock mutexes without
475 * tripping the preemption check. In reality scheduling is not enabled until
476 * this thread completes, and there are no other threads to switch to, so
477 * preemption level is not really meaningful for the bootstrap thread.
478 */
479 thread->machine.preemption_count = 0;
480 cpu_bootstrap();
481
482 rtclock_early_init();
483
484 kernel_debug_string_early("kernel_startup_bootstrap");
485 kernel_startup_bootstrap();
486
487 /*
488 * Initialize the timer callout world
489 */
490 timer_call_init();
491
492 cpu_init();
493
494 processor_bootstrap();
495
496 if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
497 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
498 } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
499 xmaxmem = (uint64_t) memsize;
500 } else {
501 xmaxmem = 0;
502 }
503
504 #if SCHED_HYGIENE_DEBUG
505 {
506 int wdt_boot_arg = 0;
507 bool const wdt_disabled = (PE_parse_boot_argn("wdt", &wdt_boot_arg, sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1));
508
509 /* Disable if WDT is disabled */
510 if (wdt_disabled || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
511 interrupt_masked_debug_mode = SCHED_HYGIENE_MODE_OFF;
512 } else if (kern_feature_override(KF_SCHED_HYGIENE_DEBUG_PMC_OVRD)) {
513 /*
514 * The sched hygiene facility can, in adition to checking time, capture
515 * metrics provided by the cycle and instruction counters available in some
516 * systems. Check if we should enable this feature based on the validation
517 * overrides.
518 */
519 sched_hygiene_debug_pmc = 0;
520 }
521
522 if (wdt_disabled || kern_feature_override(KF_PREEMPTION_DISABLED_DEBUG_OVRD)) {
523 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
524 }
525 }
526 #endif /* SCHED_HYGIENE_DEBUG */
527
528 nanoseconds_to_absolutetime(XCALL_ACK_TIMEOUT_NS, &xcall_ack_timeout_abstime);
529
530 #if HAS_BP_RET
531 PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
532 set_bp_ret(); // Apply branch predictor retention settings to boot CPU
533 #endif
534
535 PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
536
537 #if __ARM_PAN_AVAILABLE__
538 __builtin_arm_wsr("pan", 1);
539 #endif /* __ARM_PAN_AVAILABLE__ */
540
541
542 /*
543 * gPhysBase/Size only represent kernel-managed memory. These globals represent
544 * the actual DRAM base address and size as reported by iBoot through the
545 * device tree.
546 */
547 unsigned long const *dram_base;
548 unsigned long const *dram_size;
549
550 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
551 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
552 }
553
554 if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
555 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
556 }
557
558 if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
559 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
560 }
561
562 gDramBase = *dram_base;
563 gDramSize = *dram_size;
564
565
566 arm_vm_init(xmaxmem, args);
567
568 if (debug_boot_arg) {
569 patch_low_glo();
570 }
571
572 #if __arm64__ && WITH_CLASSIC_S2R
573 sleep_token_buffer_init();
574 #endif
575
576 PE_consistent_debug_inherit();
577
578 /* Setup debugging output. */
579 const unsigned int serial_exists = serial_init();
580 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
581 kprintf("kprintf initialized\n");
582
583 serialmode = 0;
584 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
585 /* Do we want a serial keyboard and/or console? */
586 kprintf("Serial mode specified: %08X\n", serialmode);
587 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
588 enable_dklog_serial_output = restore_boot || (serialmode & SERIALMODE_DKLOG) != 0;
589 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
590 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
591 if (force_sync) {
592 serialmode |= SERIALMODE_SYNCDRAIN;
593 kprintf(
594 "WARNING: Forcing uart driver to output synchronously."
595 "printf()s/IOLogs will impact kernel performance.\n"
596 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
597 }
598 }
599 /* If on-demand is selected, disable serials until reception. */
600 bool on_demand = !!(serialmode & SERIALMODE_ON_DEMAND);
601 if (on_demand && !(serialmode & SERIALMODE_INPUT)) {
602 kprintf(
603 "WARNING: invalid serial boot-args : ON_DEMAND (0x%x) flag "
604 "requires INPUT(0x%x). Ignoring ON_DEMAND.\n",
605 SERIALMODE_ON_DEMAND, SERIALMODE_INPUT
606 );
607 on_demand = 0;
608 }
609 serial_set_on_demand(on_demand);
610 }
611 if (kern_feature_override(KF_SERIAL_OVRD)) {
612 serialmode = 0;
613 }
614
615 /* Start serial if requested and a serial device was enumerated in serial_init(). */
616 if ((serialmode & SERIALMODE_OUTPUT) && serial_exists) {
617 serial_console_enabled = true;
618 (void)switch_to_serial_console(); /* Switch into serial mode from video console */
619 disableConsoleOutput = FALSE; /* Allow printfs to happen */
620 }
621 PE_create_console();
622
623 /* setup console output */
624 PE_init_printf(FALSE);
625
626 #if __arm64__
627 #if DEBUG
628 dump_kva_space();
629 #endif
630 #endif
631
632 cpu_machine_idle_init(TRUE);
633
634 PE_init_platform(TRUE, &BootCpuData);
635
636 #if __arm64__
637 extern bool cpu_config_correct;
638 if (!cpu_config_correct) {
639 panic("The cpumask=N boot arg cannot be used together with cpus=N, and the boot CPU must be enabled");
640 }
641
642 ml_map_cpu_pio();
643
644 #if APPLE_ARM64_ARCH_FAMILY
645 configure_late_apple_regs(true);
646 #endif
647
648 #endif
649
650 cpu_timebase_init(TRUE);
651
652 #if KPERF
653 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
654 kptimer_curcpu_up();
655 #endif /* KPERF */
656
657 PE_init_cpu();
658 fiq_context_init(TRUE);
659
660
661 #if HIBERNATION
662 pal_hib_init();
663 #endif /* HIBERNATION */
664
665 /*
666 * Initialize the stack protector for all future calls
667 * to C code. Since kernel_bootstrap() eventually
668 * switches stack context without returning through this
669 * function, we do not risk failing the check even though
670 * we mutate the guard word during execution.
671 */
672 __stack_chk_guard = (unsigned long)early_random();
673 /* Zero a byte of the protector to guard
674 * against string vulnerabilities
675 */
676 __stack_chk_guard &= ~(0xFFULL << 8);
677 machine_startup(args);
678 }
679
680 /*
681 * Routine: arm_init_cpu
682 * Function:
683 * Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
684 */
685
686 void
arm_init_cpu(cpu_data_t * cpu_data_ptr,uint64_t __unused hib_header_phys)687 arm_init_cpu(
688 cpu_data_t *cpu_data_ptr,
689 uint64_t __unused hib_header_phys)
690 {
691 #if __ARM_PAN_AVAILABLE__
692 __builtin_arm_wsr("pan", 1);
693 #endif
694
695
696 #ifdef __arm64__
697 configure_timer_apple_regs();
698 configure_misc_apple_regs(false);
699 #endif
700 #if HAS_ARM_FEAT_SME
701 if (enable_sme) {
702 arm_sme_init(false);
703 }
704 #endif
705
706 os_atomic_andnot(&cpu_data_ptr->cpu_flags, SleepState, relaxed);
707
708
709 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
710
711
712 #if APPLE_ARM64_ARCH_FAMILY
713 configure_late_apple_regs(false);
714 #endif
715
716 #if HIBERNATION
717 if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate) && ml_is_quiescing()) {
718 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
719 extern uint64_t wake_abstime;
720 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
721
722 // since the hw clock stops ticking across hibernation, we need to apply an offset;
723 // iBoot computes this offset for us and passes it via the hibernation header
724 extern uint64_t hwclock_conttime_offset;
725 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
726
727 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
728 // fix up its preemption count
729 unsigned int expected_preemption_count = (gEnforcePlatformActionSafety ? 2 : 1);
730 if (get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread) !=
731 expected_preemption_count) {
732 panic("unexpected preemption count %u on boot cpu thread (should be %u)",
733 get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread),
734 expected_preemption_count);
735 }
736 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
737 }
738 #endif /* HIBERNATION */
739
740 #if __arm64__
741 wfe_timeout_init();
742 pmap_clear_user_ttb();
743 flush_mmu_tlb();
744 #endif
745
746 cpu_machine_idle_init(FALSE);
747
748 cpu_init();
749
750 #ifdef APPLETYPHOON
751 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
752 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
753 }
754 #endif
755 /* Initialize the timebase before serial_init, as some serial
756 * drivers use mach_absolute_time() to implement rate control
757 */
758 cpu_timebase_init(FALSE);
759
760 #if KPERF
761 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
762 kptimer_curcpu_up();
763 #endif /* KPERF */
764
765 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
766 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
767 /*
768 * Prevent CPUs from going into deep sleep until all
769 * CPUs are ready to do so.
770 */
771 arm64_stall_sleep = TRUE;
772 #endif
773 serial_init();
774 PE_init_platform(TRUE, NULL);
775 commpage_update_timebase();
776 }
777 PE_init_cpu();
778
779 fiq_context_init(TRUE);
780 cpu_data_ptr->rtcPop = EndOfAllTime;
781 timer_resync_deadlines();
782
783 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
784 bool should_kprintf = processor_should_kprintf(processor, true);
785
786 #if DEVELOPMENT || DEBUG
787 PE_arm_debug_enable_trace(should_kprintf);
788 #endif /* DEVELOPMENT || DEBUG */
789
790 #if KERNEL_INTEGRITY_KTRR || KERNEL_INTEGRITY_CTRR
791 rorgn_validate_core();
792 #endif
793
794
795 if (should_kprintf) {
796 kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
797 }
798
799 if (cpu_data_ptr == &BootCpuData && ml_is_quiescing()) {
800 if (kdebug_enable == 0) {
801 __kdebug_only uint64_t elapsed = kdebug_wake();
802 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
803 }
804
805 #if CONFIG_TELEMETRY
806 bootprofile_wake_from_sleep();
807 #endif /* CONFIG_TELEMETRY */
808 }
809 #if CONFIG_CPU_COUNTERS
810 mt_wake_per_core();
811 #endif /* CONFIG_CPU_COUNTERS */
812
813 #if defined(KERNEL_INTEGRITY_CTRR)
814 if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
815 lck_spin_lock(&ctrr_cpu_start_lck);
816 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
817 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
818 lck_spin_unlock(&ctrr_cpu_start_lck);
819 }
820 #endif
821
822
823 secondary_cpu_main(NULL);
824 }
825
826 /*
827 * Routine: arm_init_idle_cpu
828 * Function: Resume from non-retention WFI. Called from the reset vector.
829 */
830 void __attribute__((noreturn))
arm_init_idle_cpu(cpu_data_t * cpu_data_ptr)831 arm_init_idle_cpu(
832 cpu_data_t *cpu_data_ptr)
833 {
834 #if __ARM_PAN_AVAILABLE__
835 __builtin_arm_wsr("pan", 1);
836 #endif
837
838 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
839
840 #if __arm64__
841 wfe_timeout_init();
842 pmap_clear_user_ttb();
843 flush_mmu_tlb();
844 #endif
845
846 #ifdef APPLETYPHOON
847 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
848 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
849 }
850 #endif
851
852 /*
853 * Update the active debug object to reflect that debug registers have been reset.
854 * This will force any thread with active debug state to resync the debug registers
855 * if it returns to userspace on this CPU.
856 */
857 if (cpu_data_ptr->cpu_user_debug != NULL) {
858 arm_debug_set(NULL);
859 }
860
861 fiq_context_init(FALSE);
862
863 cpu_idle_exit(TRUE);
864 }
865