1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 #include <debug.h>
33 #include <mach_ldebug.h>
34 #include <mach_kdp.h>
35
36 #include <kern/misc_protos.h>
37 #include <kern/thread.h>
38 #include <kern/timer_queue.h>
39 #include <kern/processor.h>
40 #include <kern/startup.h>
41 #include <kern/debug.h>
42 #include <prng/random.h>
43 #include <kern/ecc.h>
44 #include <machine/machine_routines.h>
45 #include <machine/commpage.h>
46 #if HIBERNATION
47 #include <machine/pal_hibernate.h>
48 #endif /* HIBERNATION */
49 /* ARM64_TODO unify boot.h */
50 #if __arm64__
51 #include <pexpert/arm64/apple_arm64_common.h>
52 #include <pexpert/arm64/boot.h>
53 #else
54 #error Unsupported arch
55 #endif
56 #include <pexpert/arm/consistent_debug.h>
57 #include <pexpert/device_tree.h>
58 #include <arm64/proc_reg.h>
59 #include <arm/pmap.h>
60 #include <arm/caches_internal.h>
61 #include <arm/cpu_internal.h>
62 #include <arm/cpu_data_internal.h>
63 #include <arm/cpuid_internal.h>
64 #include <arm/misc_protos.h>
65 #include <arm/machine_cpu.h>
66 #include <arm/rtclock.h>
67 #include <vm/vm_map.h>
68
69 #include <libkern/kernel_mach_header.h>
70 #include <libkern/stack_protector.h>
71 #include <libkern/section_keywords.h>
72 #include <san/kasan.h>
73 #include <sys/kdebug.h>
74
75 #include <pexpert/pexpert.h>
76
77 #include <console/serial_protos.h>
78
79 #if CONFIG_TELEMETRY
80 #include <kern/telemetry.h>
81 #endif
82 #if MONOTONIC
83 #include <kern/monotonic.h>
84 #endif /* MONOTONIC */
85
86 #if KPERF
87 #include <kperf/kptimer.h>
88 #endif /* KPERF */
89
90 #if HIBERNATION
91 #include <IOKit/IOPlatformExpert.h>
92 #endif /* HIBERNATION */
93
94 extern void patch_low_glo(void);
95 extern int serial_init(void);
96 extern void sleep_token_buffer_init(void);
97
98 extern vm_offset_t intstack_top;
99 #if __arm64__
100 extern vm_offset_t excepstack_top;
101 #endif
102
103 extern const char version[];
104 extern const char version_variant[];
105 extern int disableConsoleOutput;
106
107 int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0};
108 int pc_trace_cnt = PC_TRACE_BUF_SIZE;
109 int debug_task;
110
111 SECURITY_READ_ONLY_LATE(bool) static_kernelcache = false;
112
113 #if HAS_BP_RET
114 /* Enable both branch target retention (0x2) and branch direction retention (0x1) across sleep */
115 uint32_t bp_ret = 3;
116 extern void set_bp_ret(void);
117 #endif
118
119 #if SCHED_HYGIENE_DEBUG
120 boolean_t sched_hygiene_debug_pmc = 1;
121 #endif
122
123 #if SCHED_HYGIENE_DEBUG
124 TUNABLE_DT_WRITEABLE(sched_hygiene_mode_t, interrupt_masked_debug_mode,
125 "machine-timeouts", "interrupt-masked-debug-mode",
126 "interrupt-masked-debug-mode",
127 SCHED_HYGIENE_MODE_PANIC,
128 TUNABLE_DT_CHECK_CHOSEN);
129
130 MACHINE_TIMEOUT_WRITEABLE(interrupt_masked_timeout, "interrupt-masked",
131 0xd0000, MACHINE_TIMEOUT_UNIT_TIMEBASE, /* 35.499ms */
132 NULL);
133 #if __arm64__
134 #define SSHOT_INTERRUPT_MASKED_TIMEOUT 0xf9999 /* 64-bit: 42.599ms */
135 #endif
136 MACHINE_TIMEOUT_WRITEABLE(stackshot_interrupt_masked_timeout, "sshot-interrupt-masked",
137 SSHOT_INTERRUPT_MASKED_TIMEOUT, MACHINE_TIMEOUT_UNIT_TIMEBASE,
138 NULL);
139 #undef SSHOT_INTERRUPT_MASKED_TIMEOUT
140 #endif
141
142 /*
143 * A 6-second timeout will give the watchdog code a chance to run
144 * before a panic is triggered by the xcall routine.
145 */
146 #define XCALL_ACK_TIMEOUT_NS ((uint64_t) 6000000000)
147 uint64_t xcall_ack_timeout_abstime;
148
149 boot_args const_boot_args __attribute__((section("__DATA, __const")));
150 boot_args *BootArgs __attribute__((section("__DATA, __const")));
151
152 TUNABLE(uint32_t, arm_diag, "diag", 0);
153 #ifdef APPLETYPHOON
154 static unsigned cpus_defeatures = 0x0;
155 extern void cpu_defeatures_set(unsigned int);
156 #endif
157
158 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
159 extern volatile boolean_t arm64_stall_sleep;
160 #endif
161
162 extern boolean_t force_immediate_debug_halt;
163
164 #if HAS_APPLE_PAC
165 SECURITY_READ_ONLY_LATE(boolean_t) diversify_user_jop = TRUE;
166 #endif
167
168 SECURITY_READ_ONLY_LATE(uint64_t) gDramBase;
169 SECURITY_READ_ONLY_LATE(uint64_t) gDramSize;
170
171 SECURITY_READ_ONLY_LATE(bool) serial_console_enabled = false;
172 #ifdef XNU_ENABLE_PROCESSOR_EXIT
173 SECURITY_READ_ONLY_LATE(bool) enable_processor_exit = true;
174 #else
175 SECURITY_READ_ONLY_LATE(bool) enable_processor_exit = false;
176 #endif
177
178 /*
179 * Forward definition
180 */
181 void arm_init(boot_args * args);
182
183 #if __arm64__
184 unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */
185
186 extern void configure_misc_apple_boot_args(void);
187 extern void configure_misc_apple_regs(void);
188 extern void configure_timer_apple_regs(void);
189 #endif /* __arm64__ */
190
191
192 /*
193 * JOP rebasing
194 */
195
196 #define dyldLogFunc(msg, ...)
197 #include <mach/dyld_kernel_fixups.h>
198
199 extern uint32_t __thread_starts_sect_start[] __asm("section$start$__TEXT$__thread_starts");
200 extern uint32_t __thread_starts_sect_end[] __asm("section$end$__TEXT$__thread_starts");
201 #if defined(HAS_APPLE_PAC)
202 extern void OSRuntimeSignStructors(kernel_mach_header_t * header);
203 extern void OSRuntimeSignStructorsInFileset(kernel_mach_header_t * header);
204 #endif /* defined(HAS_APPLE_PAC) */
205
206 extern vm_offset_t vm_kernel_slide;
207 extern vm_offset_t segLOWESTKC, segHIGHESTKC, segLOWESTROKC, segHIGHESTROKC;
208 extern vm_offset_t segLOWESTAuxKC, segHIGHESTAuxKC, segLOWESTROAuxKC, segHIGHESTROAuxKC;
209 extern vm_offset_t segLOWESTRXAuxKC, segHIGHESTRXAuxKC, segHIGHESTNLEAuxKC;
210
211 static void
arm_slide_rebase_and_sign_image(void)212 arm_slide_rebase_and_sign_image(void)
213 {
214 kernel_mach_header_t *k_mh, *kc_mh = NULL;
215 kernel_segment_command_t *seg;
216 uintptr_t slide;
217
218 k_mh = &_mh_execute_header;
219 if (kernel_mach_header_is_in_fileset(k_mh)) {
220 /*
221 * The kernel is part of a MH_FILESET kernel collection, determine slide
222 * based on first segment's mach-o vmaddr (requires first kernel load
223 * command to be LC_SEGMENT_64 of the __TEXT segment)
224 */
225 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
226 assert(seg->cmd == LC_SEGMENT_KERNEL);
227 slide = (uintptr_t)k_mh - seg->vmaddr;
228
229 /*
230 * The kernel collection linker guarantees that the boot collection mach
231 * header vmaddr is the hardcoded kernel link address (as specified to
232 * ld64 when linking the kernel).
233 */
234 kc_mh = (kernel_mach_header_t*)(VM_KERNEL_LINK_ADDRESS + slide);
235 assert(kc_mh->filetype == MH_FILESET);
236
237 /*
238 * rebase and sign jops
239 * Note that we can't call any functions before this point, so
240 * we have to hard-code the knowledge that the base of the KC
241 * is the KC's mach-o header. This would change if any
242 * segment's VA started *before* the text segment
243 * (as the HIB segment does on x86).
244 */
245 const void *collection_base_pointers[KCNumKinds] = {[0] = kc_mh, };
246 kernel_collection_slide((struct mach_header_64 *)kc_mh, collection_base_pointers);
247
248 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
249
250 /*
251 * iBoot doesn't slide load command vmaddrs in an MH_FILESET kernel
252 * collection, so adjust them now, and determine the vmaddr range
253 * covered by read-only segments for the CTRR rorgn.
254 */
255 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)kc_mh, slide, false,
256 (uintptr_t *)&segLOWESTKC, (uintptr_t *)&segHIGHESTKC,
257 (uintptr_t *)&segLOWESTROKC, (uintptr_t *)&segHIGHESTROKC,
258 NULL, NULL, NULL);
259 #if defined(HAS_APPLE_PAC)
260 OSRuntimeSignStructorsInFileset(kc_mh);
261 #endif /* defined(HAS_APPLE_PAC) */
262 } else {
263 /*
264 * Static kernelcache: iBoot slid kernel MachO vmaddrs, determine slide
265 * using hardcoded kernel link address
266 */
267 slide = (uintptr_t)k_mh - VM_KERNEL_LINK_ADDRESS;
268
269 /* rebase and sign jops */
270 static_kernelcache = &__thread_starts_sect_end[0] != &__thread_starts_sect_start[0];
271 if (static_kernelcache) {
272 rebase_threaded_starts( &__thread_starts_sect_start[0],
273 &__thread_starts_sect_end[0],
274 (uintptr_t)k_mh, (uintptr_t)k_mh - slide, slide);
275 }
276 #if defined(HAS_APPLE_PAC)
277 OSRuntimeSignStructors(&_mh_execute_header);
278 #endif /* defined(HAS_APPLE_PAC) */
279 }
280
281
282 /*
283 * Initialize slide global here to avoid duplicating this logic in
284 * arm_vm_init()
285 */
286 vm_kernel_slide = slide;
287 }
288
289 void
arm_auxkc_init(void * mh,void * base)290 arm_auxkc_init(void *mh, void *base)
291 {
292 /*
293 * The kernel collection linker guarantees that the lowest vmaddr in an
294 * AuxKC collection is 0 (but note that the mach header is higher up since
295 * RW segments precede RO segments in the AuxKC).
296 */
297 uintptr_t slide = (uintptr_t)base;
298 kernel_mach_header_t *akc_mh = (kernel_mach_header_t*)mh;
299
300 assert(akc_mh->filetype == MH_FILESET);
301 PE_set_kc_header_and_base(KCKindAuxiliary, akc_mh, base, slide);
302
303 /* rebase and sign jops */
304 const void *collection_base_pointers[KCNumKinds];
305 memcpy(collection_base_pointers, PE_get_kc_base_pointers(), sizeof(collection_base_pointers));
306 kernel_collection_slide((struct mach_header_64 *)akc_mh, collection_base_pointers);
307
308 kernel_collection_adjust_mh_addrs((struct mach_header_64 *)akc_mh, slide, false,
309 (uintptr_t *)&segLOWESTAuxKC, (uintptr_t *)&segHIGHESTAuxKC, (uintptr_t *)&segLOWESTROAuxKC,
310 (uintptr_t *)&segHIGHESTROAuxKC, (uintptr_t *)&segLOWESTRXAuxKC, (uintptr_t *)&segHIGHESTRXAuxKC,
311 (uintptr_t *)&segHIGHESTNLEAuxKC);
312 #if defined(HAS_APPLE_PAC)
313 OSRuntimeSignStructorsInFileset(akc_mh);
314 #endif /* defined(HAS_APPLE_PAC) */
315 }
316
317 /*
318 * Routine: arm_setup_pre_sign
319 * Function: Perform HW initialization that must happen ahead of the first PAC sign
320 * operation.
321 */
322 static void
arm_setup_pre_sign(void)323 arm_setup_pre_sign(void)
324 {
325 #if __arm64__
326 /* DATA TBI, if enabled, affects the number of VA bits that contain the signature */
327 arm_set_kernel_tbi();
328 #endif /* __arm64 */
329 }
330
331 /*
332 * Routine: arm_init
333 * Function: Runs on the boot CPU, once, on entry from iBoot.
334 */
335
336 __startup_func
337 void
arm_init(boot_args * args)338 arm_init(
339 boot_args *args)
340 {
341 unsigned int maxmem;
342 uint32_t memsize;
343 uint64_t xmaxmem;
344 thread_t thread;
345 DTEntry chosen = NULL;
346 unsigned int dt_entry_size = 0;
347
348 arm_setup_pre_sign();
349
350 arm_slide_rebase_and_sign_image();
351
352 /* If kernel integrity is supported, use a constant copy of the boot args. */
353 const_boot_args = *args;
354 BootArgs = args = &const_boot_args;
355
356 cpu_data_init(&BootCpuData);
357 #if defined(HAS_APPLE_PAC)
358 /* bootstrap cpu process dependent key for kernel has been loaded by start.s */
359 BootCpuData.rop_key = ml_default_rop_pid();
360 BootCpuData.jop_key = ml_default_jop_pid();
361 #endif /* defined(HAS_APPLE_PAC) */
362
363 PE_init_platform(FALSE, args); /* Get platform expert set up */
364
365 #if __arm64__
366 configure_timer_apple_regs();
367 wfe_timeout_configure();
368 wfe_timeout_init();
369
370 configure_misc_apple_boot_args();
371 configure_misc_apple_regs();
372
373 #if (DEVELOPMENT || DEBUG)
374 unsigned long const *platform_stall_ptr = NULL;
375
376 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
377 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
378 }
379
380 // Not usable TUNABLE here because TUNABLEs are parsed at a later point.
381 if (SecureDTGetProperty(chosen, "xnu_platform_stall", (void const **)&platform_stall_ptr,
382 &dt_entry_size) == kSuccess) {
383 xnu_platform_stall_value = *platform_stall_ptr;
384 }
385
386 platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_ARM_INIT);
387
388 chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
389 dt_entry_size = 0;
390 #endif
391
392
393 {
394 /*
395 * Select the advertised kernel page size.
396 */
397 if (args->memSize > 1ULL * 1024 * 1024 * 1024) {
398 /*
399 * arm64 device with > 1GB of RAM:
400 * kernel uses 16KB pages.
401 */
402 PAGE_SHIFT_CONST = PAGE_MAX_SHIFT;
403 } else {
404 /*
405 * arm64 device with <= 1GB of RAM:
406 * kernel uses hardware page size
407 * (4KB for H6/H7, 16KB for H8+).
408 */
409 PAGE_SHIFT_CONST = ARM_PGSHIFT;
410 }
411
412 /* 32-bit apps always see 16KB page size */
413 page_shift_user32 = PAGE_MAX_SHIFT;
414 #ifdef APPLETYPHOON
415 if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) {
416 if ((cpus_defeatures & 0xF) != 0) {
417 cpu_defeatures_set(cpus_defeatures & 0xF);
418 }
419 }
420 #endif
421 }
422 #endif
423
424 ml_parse_cpu_topology();
425
426
427 master_cpu = ml_get_boot_cpu_number();
428 assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
429
430 BootCpuData.cpu_number = (unsigned short)master_cpu;
431 BootCpuData.intstack_top = (vm_offset_t) &intstack_top;
432 BootCpuData.istackptr = BootCpuData.intstack_top;
433 #if __arm64__
434 BootCpuData.excepstack_top = (vm_offset_t) &excepstack_top;
435 BootCpuData.excepstackptr = BootCpuData.excepstack_top;
436 #endif
437 CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData;
438 CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
439 + ((uintptr_t)&BootCpuData
440 - (uintptr_t)(args->virtBase)));
441
442 thread = thread_bootstrap();
443 thread->machine.CpuDatap = &BootCpuData;
444 thread->machine.pcpu_data_base = (vm_offset_t)0;
445 machine_set_current_thread(thread);
446
447 /*
448 * Preemption is enabled for this thread so that it can lock mutexes without
449 * tripping the preemption check. In reality scheduling is not enabled until
450 * this thread completes, and there are no other threads to switch to, so
451 * preemption level is not really meaningful for the bootstrap thread.
452 */
453 thread->machine.preemption_count = 0;
454 cpu_bootstrap();
455
456 rtclock_early_init();
457
458 kernel_debug_string_early("kernel_startup_bootstrap");
459 kernel_startup_bootstrap();
460
461 /*
462 * Initialize the timer callout world
463 */
464 timer_call_init();
465
466 cpu_init();
467
468 processor_bootstrap();
469
470 if (PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
471 xmaxmem = (uint64_t) maxmem * (1024 * 1024);
472 } else if (PE_get_default("hw.memsize", &memsize, sizeof(memsize))) {
473 xmaxmem = (uint64_t) memsize;
474 } else {
475 xmaxmem = 0;
476 }
477
478 #if SCHED_HYGIENE_DEBUG
479 {
480 int wdt_boot_arg = 0;
481 bool const wdt_disabled = (PE_parse_boot_argn("wdt", &wdt_boot_arg, sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1));
482
483 /* Disable if WDT is disabled */
484 if (wdt_disabled || kern_feature_override(KF_INTERRUPT_MASKED_DEBUG_OVRD)) {
485 interrupt_masked_debug_mode = SCHED_HYGIENE_MODE_OFF;
486 } else if (kern_feature_override(KF_SCHED_HYGIENE_DEBUG_PMC_OVRD)) {
487 /*
488 * The sched hygiene facility can, in adition to checking time, capture
489 * metrics provided by the cycle and instruction counters available in some
490 * systems. Check if we should enable this feature based on the validation
491 * overrides.
492 */
493 sched_hygiene_debug_pmc = 0;
494 }
495
496 if (wdt_disabled) {
497 sched_preemption_disable_debug_mode = SCHED_HYGIENE_MODE_OFF;
498 }
499 }
500 #endif /* SCHED_HYGIENE_DEBUG */
501
502 nanoseconds_to_absolutetime(XCALL_ACK_TIMEOUT_NS, &xcall_ack_timeout_abstime);
503
504 #if HAS_BP_RET
505 PE_parse_boot_argn("bpret", &bp_ret, sizeof(bp_ret));
506 set_bp_ret(); // Apply branch predictor retention settings to boot CPU
507 #endif
508
509 PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt));
510
511 #if __ARM_PAN_AVAILABLE__
512 __builtin_arm_wsr("pan", 1);
513 #endif /* __ARM_PAN_AVAILABLE__ */
514
515 arm_vm_init(xmaxmem, args);
516
517 if (debug_boot_arg) {
518 patch_low_glo();
519 }
520
521 #if __arm64__ && WITH_CLASSIC_S2R
522 sleep_token_buffer_init();
523 #endif
524
525 PE_consistent_debug_inherit();
526
527 /* Setup debugging output. */
528 const unsigned int serial_exists = serial_init();
529 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
530 kprintf("kprintf initialized\n");
531
532 serialmode = 0;
533 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
534 /* Do we want a serial keyboard and/or console? */
535 kprintf("Serial mode specified: %08X\n", serialmode);
536 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
537 enable_dklog_serial_output = (serialmode & SERIALMODE_DKLOG) != 0;
538 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
539 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
540 if (force_sync) {
541 serialmode |= SERIALMODE_SYNCDRAIN;
542 kprintf(
543 "WARNING: Forcing uart driver to output synchronously."
544 "printf()s/IOLogs will impact kernel performance.\n"
545 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
546 }
547 }
548 }
549 if (kern_feature_override(KF_SERIAL_OVRD)) {
550 serialmode = 0;
551 }
552
553 /* Start serial if requested and a serial device was enumerated in serial_init(). */
554 if ((serialmode & SERIALMODE_OUTPUT) && serial_exists) {
555 serial_console_enabled = true;
556 (void)switch_to_serial_console(); /* Switch into serial mode from video console */
557 disableConsoleOutput = FALSE; /* Allow printfs to happen */
558 }
559 PE_create_console();
560
561 /* setup console output */
562 PE_init_printf(FALSE);
563
564 #if __arm64__
565 #if DEBUG
566 dump_kva_space();
567 #endif
568 #endif
569
570 cpu_machine_idle_init(TRUE);
571
572 PE_init_platform(TRUE, &BootCpuData);
573
574 #if __arm64__
575 extern bool cpu_config_correct;
576 if (!cpu_config_correct) {
577 panic("The cpumask=N boot arg cannot be used together with cpus=N, and the boot CPU must be enabled");
578 }
579
580 ml_map_cpu_pio();
581 #endif
582
583 cpu_timebase_init(TRUE);
584
585 #if KPERF
586 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
587 kptimer_curcpu_up();
588 #endif /* KPERF */
589
590 PE_init_cpu();
591 fiq_context_init(TRUE);
592
593
594 #if HIBERNATION
595 pal_hib_init();
596 #endif /* HIBERNATION */
597
598 /*
599 * gPhysBase/Size only represent kernel-managed memory. These globals represent
600 * the actual DRAM base address and size as reported by iBoot through the
601 * device tree.
602 */
603 unsigned long const *dram_base;
604 unsigned long const *dram_size;
605
606 if (SecureDTLookupEntry(NULL, "/chosen", &chosen) != kSuccess) {
607 panic("%s: Unable to find 'chosen' DT node", __FUNCTION__);
608 }
609
610 if (SecureDTGetProperty(chosen, "dram-base", (void const **)&dram_base, &dt_entry_size) != kSuccess) {
611 panic("%s: Unable to find 'dram-base' entry in the 'chosen' DT node", __FUNCTION__);
612 }
613
614 if (SecureDTGetProperty(chosen, "dram-size", (void const **)&dram_size, &dt_entry_size) != kSuccess) {
615 panic("%s: Unable to find 'dram-size' entry in the 'chosen' DT node", __FUNCTION__);
616 }
617
618 gDramBase = *dram_base;
619 gDramSize = *dram_size;
620
621 /*
622 * Initialize the stack protector for all future calls
623 * to C code. Since kernel_bootstrap() eventually
624 * switches stack context without returning through this
625 * function, we do not risk failing the check even though
626 * we mutate the guard word during execution.
627 */
628 __stack_chk_guard = (unsigned long)early_random();
629 /* Zero a byte of the protector to guard
630 * against string vulnerabilities
631 */
632 __stack_chk_guard &= ~(0xFFULL << 8);
633 machine_startup(args);
634 }
635
636 /*
637 * Routine: arm_init_cpu
638 * Function:
639 * Runs on S2R resume (all CPUs) and SMP boot (non-boot CPUs only).
640 */
641
642 void
arm_init_cpu(cpu_data_t * cpu_data_ptr)643 arm_init_cpu(
644 cpu_data_t *cpu_data_ptr)
645 {
646 #if __ARM_PAN_AVAILABLE__
647 __builtin_arm_wsr("pan", 1);
648 #endif
649
650 #ifdef __arm64__
651 configure_timer_apple_regs();
652 configure_misc_apple_regs();
653 #endif
654
655 cpu_data_ptr->cpu_flags &= ~SleepState;
656
657
658 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
659
660 #if HIBERNATION
661 if ((cpu_data_ptr == &BootCpuData) && (gIOHibernateState == kIOHibernateStateWakingFromHibernate)) {
662 // the "normal" S2R code captures wake_abstime too early, so on a hibernation resume we fix it up here
663 extern uint64_t wake_abstime;
664 wake_abstime = gIOHibernateCurrentHeader->lastHibAbsTime;
665
666 // since the hw clock stops ticking across hibernation, we need to apply an offset;
667 // iBoot computes this offset for us and passes it via the hibernation header
668 extern uint64_t hwclock_conttime_offset;
669 hwclock_conttime_offset = gIOHibernateCurrentHeader->hwClockOffset;
670
671 // during hibernation, we captured the idle thread's state from inside the PPL context, so we have to
672 // fix up its preemption count
673 unsigned int expected_preemption_count = (gEnforcePlatformActionSafety ? 2 : 1);
674 if (cpu_data_ptr->cpu_active_thread->machine.preemption_count != expected_preemption_count) {
675 panic("unexpected preemption count %u on boot cpu thread (should be %u)",
676 cpu_data_ptr->cpu_active_thread->machine.preemption_count,
677 expected_preemption_count);
678 }
679 cpu_data_ptr->cpu_active_thread->machine.preemption_count--;
680 }
681 #endif /* HIBERNATION */
682
683 #if __arm64__
684 wfe_timeout_init();
685 pmap_clear_user_ttb();
686 flush_mmu_tlb();
687 #endif
688
689 cpu_machine_idle_init(FALSE);
690
691 cpu_init();
692
693 #ifdef APPLETYPHOON
694 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
695 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
696 }
697 #endif
698 /* Initialize the timebase before serial_init, as some serial
699 * drivers use mach_absolute_time() to implement rate control
700 */
701 cpu_timebase_init(FALSE);
702
703 #if KPERF
704 /* kptimer_curcpu_up() must be called after cpu_timebase_init */
705 kptimer_curcpu_up();
706 #endif /* KPERF */
707
708 if (cpu_data_ptr == &BootCpuData) {
709 #if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__
710 /*
711 * Prevent CPUs from going into deep sleep until all
712 * CPUs are ready to do so.
713 */
714 arm64_stall_sleep = TRUE;
715 #endif
716 serial_init();
717 PE_init_platform(TRUE, NULL);
718 commpage_update_timebase();
719 }
720 PE_init_cpu();
721
722 fiq_context_init(TRUE);
723 cpu_data_ptr->rtcPop = EndOfAllTime;
724 timer_resync_deadlines();
725
726 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
727 bool should_kprintf = processor_should_kprintf(processor, true);
728
729 #if DEVELOPMENT || DEBUG
730 PE_arm_debug_enable_trace(should_kprintf);
731 #endif /* DEVELOPMENT || DEBUG */
732
733
734 if (should_kprintf) {
735 kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_number);
736 }
737
738 if (cpu_data_ptr == &BootCpuData) {
739 if (kdebug_enable == 0) {
740 __kdebug_only uint64_t elapsed = kdebug_wake();
741 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 15), mach_absolute_time() - elapsed);
742 }
743
744 #if CONFIG_TELEMETRY
745 bootprofile_wake_from_sleep();
746 #endif /* CONFIG_TELEMETRY */
747 }
748 #if MONOTONIC && defined(__arm64__)
749 mt_wake_per_core();
750 #endif /* MONOTONIC && defined(__arm64__) */
751
752 #if defined(KERNEL_INTEGRITY_CTRR)
753 if (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKED) {
754 lck_spin_lock(&ctrr_cpu_start_lck);
755 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKED;
756 thread_wakeup(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]);
757 lck_spin_unlock(&ctrr_cpu_start_lck);
758 }
759 #endif
760
761
762 slave_main(NULL);
763 }
764
765 /*
766 * Routine: arm_init_idle_cpu
767 * Function: Resume from non-retention WFI. Called from the reset vector.
768 */
769 void __attribute__((noreturn))
arm_init_idle_cpu(cpu_data_t * cpu_data_ptr)770 arm_init_idle_cpu(
771 cpu_data_t *cpu_data_ptr)
772 {
773 #if __ARM_PAN_AVAILABLE__
774 __builtin_arm_wsr("pan", 1);
775 #endif
776
777 machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
778
779 #if __arm64__
780 wfe_timeout_init();
781 pmap_clear_user_ttb();
782 flush_mmu_tlb();
783 /* Enable asynchronous exceptions */
784 __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF);
785 #endif
786
787 #ifdef APPLETYPHOON
788 if ((cpus_defeatures & (0xF << 4 * cpu_data_ptr->cpu_number)) != 0) {
789 cpu_defeatures_set((cpus_defeatures >> 4 * cpu_data_ptr->cpu_number) & 0xF);
790 }
791 #endif
792
793 /*
794 * Update the active debug object to reflect that debug registers have been reset.
795 * This will force any thread with active debug state to resync the debug registers
796 * if it returns to userspace on this CPU.
797 */
798 if (cpu_data_ptr->cpu_user_debug != NULL) {
799 arm_debug_set(NULL);
800 }
801
802 fiq_context_init(FALSE);
803
804 cpu_idle_exit(TRUE);
805 }
806