1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 #include <debug.h>
33 #include <mach_ldebug.h>
34 #include <mach_kdp.h>
35
36 #include <kern/misc_protos.h>
37 #include <kern/thread.h>
38 #include <kern/timer_queue.h>
39 #include <kern/processor.h>
40 #include <kern/startup.h>
41 #include <kern/debug.h>
42 #include <prng/random.h>
43 #include <kern/ecc.h>
44 #include <machine/machine_routines.h>
45 #include <machine/commpage.h>
46 #include <machine/config.h>
47 #if HIBERNATION
48 #include <machine/pal_hibernate.h>
49 #endif /* HIBERNATION */
50 /* ARM64_TODO unify boot.h */
51 #if __arm64__
52 #include <pexpert/arm64/boot.h>
53 #include <arm64/amcc_rorgn.h>
54 #else
55 #error Unsupported arch
56 #endif
57 #include <pexpert/arm/consistent_debug.h>
58 #include <pexpert/device_tree.h>
59 #include <arm64/proc_reg.h>
60 #include <arm/pmap.h>
61 #include <arm/caches_internal.h>
62 #include <arm/cpu_internal.h>
63 #include <arm/cpu_data_internal.h>
64 #include <arm/cpuid_internal.h>
65 #include <arm/misc_protos.h>
66 #include <arm/machine_cpu.h>
67 #include <arm/rtclock.h>
68 #include <vm/vm_map.h>
69
70 #include <libkern/kernel_mach_header.h>
71 #include <libkern/stack_protector.h>
72 #include <libkern/section_keywords.h>
73 #include <san/kasan.h>
74 #include <sys/kdebug.h>
75
76 #include <pexpert/pexpert.h>
77
78 #include <console/serial_protos.h>
79
80 #if CONFIG_TELEMETRY
81 #include <kern/telemetry.h>
82 #endif
83 #if MONOTONIC
84 #include <kern/monotonic.h>
85 #endif /* MONOTONIC */
86
87 #if KPERF
88 #include <kperf/kptimer.h>
89 #endif /* KPERF */
90
91 #if HIBERNATION
92 #include <IOKit/IOPlatformExpert.h>
93 #endif /* HIBERNATION */
94
95 extern void patch_low_glo(void);
96 extern int serial_init(void);
97 extern void sleep_token_buffer_init(void);
98
99 extern vm_offset_t intstack_top;
100 #if __arm64__
101 extern vm_offset_t excepstack_top;
102 #endif
103
104 extern const char version[];
105 extern const char version_variant[];
106 extern int disableConsoleOutput;
107
108 int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
109 int pc_trace_cnt = PC_TRACE_BUF_SIZE;
110 int debug_task;
111
112 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
113
114 #if HAS_BP_RET
115 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
116 uint32_t bp_ret = 3;
117 extern void set_bp_ret(void);
118 #endif
119
120 #if SCHED_HYGIENE_DEBUG
121 boolean_t sched_hygiene_debug_pmc = 1;
122 #endif
123
124 #if SCHED_HYGIENE_DEBUG
125
126 #if XNU_PLATFORM_iPhoneOS
127 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 12000 /* 500us */
128 #else
129 #define DEFAULT_INTERRUPT_MASKED_TIMEOUT 0xd0000 /* 35.499ms */
130 #endif /* XNU_PLATFORM_iPhoneOS */
131
132 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, interrupt_masked_debug_mode,
133 "machine-timeouts", "interrupt-masked-debug-mode",
134 "interrupt-masked-debug-mode",
135 SCHED_HYGIENE_MODE_PANIC,
136 TUNABLE_DT_CHECK_CHOSEN);
137
138 MACHINE_TIMEOUT_DEV_WRITEABLE(interrupt_masked_timeout, "interrupt-masked",
139 DEFAULT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
140 NULL);
141 #if __arm64__
142 #define SSHOT_INTERRUPT_MASKED_TIMEOUT 0xf9999 /* 64-bit: 42.599ms */
143 #endif
144 MACHINE_TIMEOUT_DEV_WRITEABLE(stackshot_interrupt_masked_timeout, "sshot-interrupt-masked",
145 SSHOT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
146 NULL);
147 #undef SSHOT_INTERRUPT_MASKED_TIMEOUT
148 #endif
149
150 /*
151 * A 6-second timeout will give the watchdog code a chance to run
152 * before a panic is triggered by the xcall routine.
153 */
154 #define XCALL_ACK_TIMEOUT_NS ((uint64_t) 6000000000)
155 uint64_t xcall_ack_timeout_abstime;
156
157 boot_args const_boot_args __attribute__((section("__DATA, __const")));
158 boot_args *BootArgs __attribute__((section("__DATA, __const")));
159
160 TUNABLE(uint32_t, arm_diag, "diag", 0);
161 #ifdef APPLETYPHOON
162 static unsigned cpus_defeatures = 0x0;
163 extern void cpu_defeatures_set(unsigned int);
164 #endif
165
166 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
167 extern volatile boolean_t arm64_stall_sleep;
168 #endif
169
170 extern boolean_t force_immediate_debug_halt;
171
172 #if HAS_APPLE_PAC
173 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
174 #endif
175
176 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
177 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
178
179 SECURITY_READ_ONLY_LATE(bool) serial_console_enabled = false;
180 #ifdef XNU_ENABLE_PROCESSOR_EXIT
181 SECURITY_READ_ONLY_LATE(bool) enable_processor_exit = true;
182 #else
183 SECURITY_READ_ONLY_LATE(bool) enable_processor_exit = false;
184 #endif
185
186
187 /*
188 * Forward definition
189 */
190 void arm_init(boot_args * args);
191
192 #if __arm64__
193 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
194
195 extern void configure_misc_apple_boot_args(void);
196 extern void configure_misc_apple_regs(bool is_boot_cpu);
197 extern void configure_timer_apple_regs(void);
198 extern void configure_late_apple_regs(bool cold_boot);
199 #endif /* __arm64__ */
200
201
202 /*
203 * JOP rebasing
204 */
205
206 #define dyldLogFunc(msg, ...)
207 #include <mach/dyld_kernel_fixups.h>
208
209 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
210 extern uint32_t __thread_starts_sect_end[] __asm("section$end$__TEXT$__thread_starts");
211 #if defined(HAS_APPLE_PAC)
212 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
213 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
214 #endif /* defined(HAS_APPLE_PAC) */
215
216 extern vm_offset_t vm_kernel_slide;
217 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
218 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
219 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
220
221 static void
arm_slide_rebase_and_sign_image(void)222 arm_slide_rebase_and_sign_image(void)
223 {
224 kernel_mach_header_t *k_mh, *kc_mh = NULL;
225 kernel_segment_command_t *seg;
226 uintptr_t slide;
227
228 k_mh = &_mh_execute_header;
229 if (kernel_mach_header_is_in_fileset(k_mh)) {
230 /*
231 * The kernel is part of a MH_FILESET kernel collection, determine slide
232 * based on first segment's mach-o vmaddr (requires first kernel load
233 * command to be LC_SEGMENT_64 of the __TEXT segment)
234 */
235 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
236 assert(seg->cmd == LC_SEGMENT_KERNEL);
237 slide = (uintptr_t)k_mh - seg->vmaddr;
238
239 /*
240 * The kernel collection linker guarantees that the boot collection mach
241 * header vmaddr is the hardcoded kernel link address (as specified to
242 * ld64 when linking the kernel).
243 */
244 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
245 assert(kc_mh->filetype == MH_FILESET);
246
247 /*
248 * rebase and sign jops
249 * Note that we can't call any functions before this point, so
250 * we have to hard-code the knowledge that the base of the KC
251 * is the KC's mach-o header. This would change if any
252 * segment's VA started *before* the text segment
253 * (as the HIB segment does on x86).
254 */
255 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
256 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
257
258 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
259
260 /*
261 * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
262 * collection, so adjust them now, and determine the vmaddr range
263 * covered by read-only segments for the CTRR rorgn.
264 */
265 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
266 (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
267 (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
268 NULL, NULL, NULL);
269 #if defined(HAS_APPLE_PAC)
270 OSRuntimeSignStructorsInFileset(kc_mh);
271 #endif /* defined(HAS_APPLE_PAC) */
272 } else {
273 /*
274 * Static kernelcache: iBoot slid kernel MachO vmaddrs, determine slide
275 * using hardcoded kernel link address
276 */
277 slide = (uintptr_t)k_mh - VM_KERNEL_LINK_ADDRESS;
278
279 /* rebase and sign jops */
280 static_kernelcache = &__thread_starts_sect_end[0] != &__thread_starts_sect_start[0];
281 if (static_kernelcache) {
282 rebase_threaded_starts( &__thread_starts_sect_start[0],
283 &__thread_starts_sect_end[0],
284 (uintptr_t)k_mh, (uintptr_t)k_mh - slide, slide);
285 }
286 #if defined(HAS_APPLE_PAC)
287 OSRuntimeSignStructors(&_mh_execute_header);
288 #endif /* defined(HAS_APPLE_PAC) */
289 }
290
291
292 /*
293 * Initialize slide global here to avoid duplicating this logic in
294 * arm_vm_init()
295 */
296 vm_kernel_slide = slide;
297 }
298
299 void
arm_auxkc_init(void * mh,void * base)300 arm_auxkc_init(void *mh, void *base)
301 {
302 /*
303 * The kernel collection linker guarantees that the lowest vmaddr in an
304 * AuxKC collection is 0 (but note that the mach header is higher up since
305 * RW segments precede RO segments in the AuxKC).
306 */
307 uintptr_t slide = (uintptr_t)base;
308 kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
309
310 assert(akc_mh->filetype == MH_FILESET);
311 PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
312
313 /* rebase and sign jops */
314 const void *collection_base_pointers[KCNumKinds];
315 memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
316 kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
317
318 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
319 (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
320 (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
321 (uintptr_t *)&segHIGHESTNLEAuxKC);
322 #if defined(HAS_APPLE_PAC)
323 OSRuntimeSignStructorsInFileset(akc_mh);
324 #endif /* defined(HAS_APPLE_PAC) */
325 }
326
327 /*
328 * Routine: arm_setup_pre_sign
329 * Function: Perform HW initialization that must happen ahead of the first PAC sign
330 * operation.
331 */
332 static void
arm_setup_pre_sign(void)333 arm_setup_pre_sign(void)
334 {
335 #if __arm64__
336 /* DATA TBI, if enabled, affects the number of VA bits that contain the signature */
337 arm_set_kernel_tbi();
338 #endif /* __arm64 */
339 }
340
341 /*
342 * Routine: arm_init
343 * Function: Runs on the boot CPU, once, on entry from iBoot.
344 */
345
346 __startup_func
347 void
arm_init(boot_args * args)348 arm_init(
349 boot_args *args)
350 {
351 unsigned int maxmem;
352 uint32_t memsize;
353 uint64_t xmaxmem;
354 thread_t thread;
355 DTEntry chosen = NULL;
356 unsigned int dt_entry_size = 0;
357
358 arm_setup_pre_sign();
359
360 arm_slide_rebase_and_sign_image();
361
362 /* If kernel integrity is supported, use a constant copy of the boot args. */
363 const_boot_args = *args;
364 BootArgs = args = &const_boot_args;
365
366 cpu_data_init(&BootCpuData);
367 #if defined(HAS_APPLE_PAC)
368 /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
369 BootCpuData.rop_key = ml_default_rop_pid();
370 BootCpuData.jop_key = ml_default_jop_pid();
371 #endif /* defined(HAS_APPLE_PAC) */
372
373 PE_init_platform(FALSE, args); /* Get platform expert set up */
374
375 #if __arm64__
376 configure_timer_apple_regs();
377 wfe_timeout_configure();
378 wfe_timeout_init();
379
380 configure_misc_apple_boot_args();
381 configure_misc_apple_regs(true);
382
383 #if (DEVELOPMENT || DEBUG)
384 unsigned long const *platform_stall_ptr = NULL;
385
386 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
387 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
388 }
389
390 // Not usable TUNABLE here because TUNABLEs are parsed at a later point.
391 if (SecureDTGetProperty(chosen, "xnu_platform_stall", (void const **)&platform_stall_ptr,
392 &dt_entry_size) == kSuccess) {
393 xnu_platform_stall_value = *platform_stall_ptr;
394 }
395
396 platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_ARM_INIT);
397
398 chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
399 dt_entry_size = 0;
400 #endif
401
402
403 {
404 /*
405 * Select the advertised kernel page size.
406 */
407 if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
408 /*
409 * arm64 device with > 1GB of RAM:
410 * kernel uses 16KB pages.
411 */
412 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
413 } else {
414 /*
415 * arm64 device with <= 1GB of RAM:
416 * kernel uses hardware page size
417 * (4KB for H6/H7, 16KB for H8+).
418 */
419 PAGE_SHIFT_CONST = ARM_PGSHIFT;
420 }
421
422 /* 32-bit apps always see 16KB page size */
423 page_shift_user32 = PAGE_MAX_SHIFT;
424 #ifdef APPLETYPHOON
425 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
426 if ((cpus_defeatures & 0xF) != 0) {
427 cpu_defeatures_set(cpus_defeatures & 0xF);
428 }
429 }
430 #endif
431 }
432 #endif
433
434 ml_parse_cpu_topology();
435
436
437 master_cpu = ml_get_boot_cpu_number();
438 assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
439
440 BootCpuData.cpu_number = (unsigned short)master_cpu;
441 BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
442 BootCpuData.istackptr = &intstack_top;
443 #if __arm64__
444 BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
445 #endif
446 CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
447 CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
448 + ((uintptr_t)&BootCpuData
449 - (uintptr_t)(args->virtBase)));
450
451 thread = thread_bootstrap();
452 thread->machine.CpuDatap = &BootCpuData;
453 thread->machine.pcpu_data_base = (vm_offset_t)0;
454 machine_set_current_thread(thread);
455
456 /*
457 * Preemption is enabled for this thread so that it can lock mutexes without
458 * tripping the preemption check. In reality scheduling is not enabled until
459 * this thread completes, and there are no other threads to switch to, so
460 * preemption level is not really meaningful for the bootstrap thread.
461 */
462 thread->machine.preemption_count = 0;
463 cpu_bootstrap();
464
465 rtclock_early_init();
466
467 kernel_debug_string_early("kernel_startup_bootstrap");
468 kernel_startup_bootstrap();
469
470 /*
471 * Initialize the timer callout world
472 */
473 timer_call_init();
474
475 cpu_init();
476
477 processor_bootstrap();
478
479 if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
480 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
481 } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
482 xmaxmem = (uint64_t) memsize;
483 } else {
484 xmaxmem = 0;
485 }
486
487 #if SCHED_HYGIENE_DEBUG
488 {
489 int wdt_boot_arg = 0;
490 bool const wdt_disabled = (PE_parse_boot_argn("wdt", &wdt_boot_arg, sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1));
491
492 /* Disable if WDT is disabled */
493 if (wdt_disabled || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
494 interrupt_masked_debug_mode = SCHED_HYGIENE_MODE_OFF;
495 } else if (kern_feature_override(KF_SCHED_HYGIENE_DEBUG_PMC_OVRD)) {
496 /*
497 * The sched hygiene facility can, in adition to checking time, capture
498 * metrics provided by the cycle and instruction counters available in some
499 * systems. Check if we should enable this feature based on the validation
500 * overrides.
501 */
502 sched_hygiene_debug_pmc = 0;
503 }
504
505 if (wdt_disabled || kern_feature_override(KF_PREEMPTION_DISABLED_DEBUG_OVRD)) {
506 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
507 }
508 }
509 #endif /* SCHED_HYGIENE_DEBUG */
510
511 nanoseconds_to_absolutetime(XCALL_ACK_TIMEOUT_NS, &xcall_ack_timeout_abstime);
512
513 #if HAS_BP_RET
514 PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
515 set_bp_ret(); // Apply branch predictor retention settings to boot CPU
516 #endif
517
518 PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
519
520 #if __ARM_PAN_AVAILABLE__
521 __builtin_arm_wsr("pan", 1);
522 #endif /* __ARM_PAN_AVAILABLE__ */
523
524 arm_vm_init(xmaxmem, args);
525
526 if (debug_boot_arg) {
527 patch_low_glo();
528 }
529
530 #if __arm64__ && WITH_CLASSIC_S2R
531 sleep_token_buffer_init();
532 #endif
533
534 PE_consistent_debug_inherit();
535
536 /* Setup debugging output. */
537 const unsigned int serial_exists = serial_init();
538 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
539 kprintf("kprintf initialized\n");
540
541 serialmode = 0;
542 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
543 /* Do we want a serial keyboard and/or console? */
544 kprintf("Serial mode specified: %08X\n", serialmode);
545 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
546 enable_dklog_serial_output = (serialmode & SERIALMODE_DKLOG) != 0;
547 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
548 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
549 if (force_sync) {
550 serialmode |= SERIALMODE_SYNCDRAIN;
551 kprintf(
552 "WARNING: Forcing uart driver to output synchronously."
553 "printf()s/IOLogs will impact kernel performance.\n"
554 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
555 }
556 }
557 /* If on-demand is selected, disable serials until reception. */
558 bool on_demand = !!(serialmode & SERIALMODE_ON_DEMAND);
559 if (on_demand && !(serialmode & SERIALMODE_INPUT)) {
560 kprintf(
561 "WARNING: invalid serial boot-args : ON_DEMAND (0x%x) flag "
562 "requires INPUT(0x%x). Ignoring ON_DEMAND.\n",
563 SERIALMODE_ON_DEMAND, SERIALMODE_INPUT
564 );
565 on_demand = 0;
566 }
567 serial_set_on_demand(on_demand);
568 }
569 if (kern_feature_override(KF_SERIAL_OVRD)) {
570 serialmode = 0;
571 }
572
573 /* Start serial if requested and a serial device was enumerated in serial_init(). */
574 if ((serialmode & SERIALMODE_OUTPUT) && serial_exists) {
575 serial_console_enabled = true;
576 (void)switch_to_serial_console(); /* Switch into serial mode from video console */
577 disableConsoleOutput = FALSE; /* Allow printfs to happen */
578 }
579 PE_create_console();
580
581 /* setup console output */
582 PE_init_printf(FALSE);
583
584 #if __arm64__
585 #if DEBUG
586 dump_kva_space();
587 #endif
588 #endif
589
590 cpu_machine_idle_init(TRUE);
591
592 PE_init_platform(TRUE, &BootCpuData);
593
594 #if __arm64__
595 extern bool cpu_config_correct;
596 if (!cpu_config_correct) {
597 panic("The cpumask=N boot arg cannot be used together with cpus=N, and the boot CPU must be enabled");
598 }
599
600 ml_map_cpu_pio();
601
602 #if APPLE_ARM64_ARCH_FAMILY
603 configure_late_apple_regs(true);
604 #endif
605
606 #endif
607
608 cpu_timebase_init(TRUE);
609
610 #if KPERF
611 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
612 kptimer_curcpu_up();
613 #endif /* KPERF */
614
615 PE_init_cpu();
616 fiq_context_init(TRUE);
617
618
619 #if HIBERNATION
620 pal_hib_init();
621 #endif /* HIBERNATION */
622
623 /*
624 * gPhysBase/Size only represent kernel-managed memory. These globals represent
625 * the actual DRAM base address and size as reported by iBoot through the
626 * device tree.
627 */
628 unsigned long const *dram_base;
629 unsigned long const *dram_size;
630
631 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
632 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
633 }
634
635 if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
636 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
637 }
638
639 if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
640 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
641 }
642
643 gDramBase = *dram_base;
644 gDramSize = *dram_size;
645
646 /*
647 * Initialize the stack protector for all future calls
648 * to C code. Since kernel_bootstrap() eventually
649 * switches stack context without returning through this
650 * function, we do not risk failing the check even though
651 * we mutate the guard word during execution.
652 */
653 __stack_chk_guard = (unsigned long)early_random();
654 /* Zero a byte of the protector to guard
655 * against string vulnerabilities
656 */
657 __stack_chk_guard &= ~(0xFFULL << 8);
658 machine_startup(args);
659 }
660
661 /*
662 * Routine: arm_init_cpu
663 * Function:
664 * Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
665 */
666
667 void
arm_init_cpu(cpu_data_t * cpu_data_ptr)668 arm_init_cpu(
669 cpu_data_t *cpu_data_ptr)
670 {
671 #if __ARM_PAN_AVAILABLE__
672 __builtin_arm_wsr("pan", 1);
673 #endif
674
675 #ifdef __arm64__
676 configure_timer_apple_regs();
677 configure_misc_apple_regs(false);
678 #endif
679
680 cpu_data_ptr->cpu_flags &= ~SleepState;
681
682
683 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
684
685 #if APPLE_ARM64_ARCH_FAMILY
686 configure_late_apple_regs(false);
687 #endif
688
689 #if HIBERNATION
690 if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate)) {
691 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
692 extern uint64_t wake_abstime;
693 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
694
695 // since the hw clock stops ticking across hibernation, we need to apply an offset;
696 // iBoot computes this offset for us and passes it via the hibernation header
697 extern uint64_t hwclock_conttime_offset;
698 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
699
700 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
701 // fix up its preemption count
702 unsigned int expected_preemption_count = (gEnforcePlatformActionSafety ? 2 : 1);
703 if (get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread) !=
704 expected_preemption_count) {
705 panic("unexpected preemption count %u on boot cpu thread (should be %u)",
706 get_preemption_level_for_thread(cpu_data_ptr->cpu_active_thread),
707 expected_preemption_count);
708 }
709 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
710 }
711 #endif /* HIBERNATION */
712
713 #if __arm64__
714 wfe_timeout_init();
715 pmap_clear_user_ttb();
716 flush_mmu_tlb();
717 #endif
718
719 cpu_machine_idle_init(FALSE);
720
721 cpu_init();
722
723 #ifdef APPLETYPHOON
724 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
725 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
726 }
727 #endif
728 /* Initialize the timebase before serial_init, as some serial
729 * drivers use mach_absolute_time() to implement rate control
730 */
731 cpu_timebase_init(FALSE);
732
733 #if KPERF
734 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
735 kptimer_curcpu_up();
736 #endif /* KPERF */
737
738 if (cpu_data_ptr == &BootCpuData) {
739 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
740 /*
741 * Prevent CPUs from going into deep sleep until all
742 * CPUs are ready to do so.
743 */
744 arm64_stall_sleep = TRUE;
745 #endif
746 serial_init();
747 PE_init_platform(TRUE, NULL);
748 commpage_update_timebase();
749 }
750 PE_init_cpu();
751
752 fiq_context_init(TRUE);
753 cpu_data_ptr->rtcPop = EndOfAllTime;
754 timer_resync_deadlines();
755
756 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
757 bool should_kprintf = processor_should_kprintf(processor, true);
758
759 #if DEVELOPMENT || DEBUG
760 PE_arm_debug_enable_trace(should_kprintf);
761 #endif /* DEVELOPMENT || DEBUG */
762
763 #if KERNEL_INTEGRITY_KTRR || KERNEL_INTEGRITY_CTRR
764 rorgn_validate_core();
765 #endif
766
767
768 if (should_kprintf) {
769 kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
770 }
771
772 if (cpu_data_ptr == &BootCpuData) {
773 if (kdebug_enable == 0) {
774 __kdebug_only uint64_t elapsed = kdebug_wake();
775 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
776 }
777
778 #if CONFIG_TELEMETRY
779 bootprofile_wake_from_sleep();
780 #endif /* CONFIG_TELEMETRY */
781 }
782 #if MONOTONIC && defined(__arm64__)
783 mt_wake_per_core();
784 #endif /* MONOTONIC && defined(__arm64__) */
785
786 #if defined(KERNEL_INTEGRITY_CTRR)
787 if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
788 lck_spin_lock(&ctrr_cpu_start_lck);
789 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
790 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
791 lck_spin_unlock(&ctrr_cpu_start_lck);
792 }
793 #endif
794
795
796 slave_main(NULL);
797 }
798
799 /*
800 * Routine: arm_init_idle_cpu
801 * Function: Resume from non-retention WFI. Called from the reset vector.
802 */
803 void __attribute__((noreturn))
arm_init_idle_cpu(cpu_data_t * cpu_data_ptr)804 arm_init_idle_cpu(
805 cpu_data_t *cpu_data_ptr)
806 {
807 #if __ARM_PAN_AVAILABLE__
808 __builtin_arm_wsr("pan", 1);
809 #endif
810
811 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
812
813 #if __arm64__
814 wfe_timeout_init();
815 pmap_clear_user_ttb();
816 flush_mmu_tlb();
817 /* Enable asynchronous exceptions */
818 __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
819 #endif
820
821 #ifdef APPLETYPHOON
822 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
823 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
824 }
825 #endif
826
827 /*
828 * Update the active debug object to reflect that debug registers have been reset.
829 * This will force any thread with active debug state to resync the debug registers
830 * if it returns to userspace on this CPU.
831 */
832 if (cpu_data_ptr->cpu_user_debug != NULL) {
833 arm_debug_set(NULL);
834 }
835
836 fiq_context_init(FALSE);
837
838 cpu_idle_exit(TRUE);
839 }
840