1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm64/cpu.c
30 *
31 * cpu specific routines
32 */
33
34 #include <pexpert/arm64/board_config.h>
35 #include <kern/kalloc.h>
36 #include <kern/machine.h>
37 #include <kern/cpu_number.h>
38 #include <kern/percpu.h>
39 #include <kern/thread.h>
40 #include <kern/timer_queue.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <arm64/proc_reg.h>
50 #include <mach/processor_info.h>
51 #include <vm/pmap.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58
59 #include <machine/atomic.h>
60
61 #include <san/kasan.h>
62
63 #if KPC
64 #include <kern/kpc.h>
65 #endif
66
67 #if MONOTONIC
68 #include <kern/monotonic.h>
69 #endif /* MONOTONIC */
70
71 #if KPERF
72 #include <kperf/kptimer.h>
73 #endif /* KPERF */
74
75 #if HIBERNATION
76 #include <IOKit/IOPlatformExpert.h>
77 #include <IOKit/IOHibernatePrivate.h>
78 #endif /* HIBERNATION */
79
80
81 #include <libkern/section_keywords.h>
82
83 extern boolean_t idle_enable;
84 extern uint64_t wake_abstime;
85
86 #if WITH_CLASSIC_S2R
87 void sleep_token_buffer_init(void);
88 #endif
89
90
91 extern uintptr_t resume_idle_cpu;
92 extern uintptr_t start_cpu;
93
94 #if __ARM_KERNEL_PROTECT__
95 extern void exc_vectors_table;
96 #endif /* __ARM_KERNEL_PROTECT__ */
97
98 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep, unsigned int cpu, uint64_t entry_pa);
99 extern void arm64_force_wfi_clock_gate(void);
100 #if defined(APPLETYPHOON)
101 // <rdar://problem/15827409>
102 extern void typhoon_prepare_for_wfi(void);
103 extern void typhoon_return_from_wfi(void);
104 #endif
105
106 #if HAS_RETENTION_STATE
107 extern void arm64_retention_wfi(void);
108 #endif
109
110 vm_address_t start_cpu_paddr;
111
112 sysreg_restore_t sysreg_restore __attribute__((section("__DATA, __const"))) = {
113 .tcr_el1 = TCR_EL1_BOOT,
114 };
115
116 // wfi - wfi mode
117 // 0 : disabled
118 // 1 : normal
119 // 2 : overhead simulation (delay & flags)
120 TUNABLE(unsigned int, wfi, "wfi", 1);
121 #if DEVELOPMENT || DEBUG
122
123 // wfi_flags
124 // 1 << 0 : flush L1s
125 // 1 << 1 : flush TLBs
126 static int wfi_flags = 0;
127
128 // wfi_delay - delay ticks after wfi exit
129 static uint64_t wfi_delay = 0;
130
131 #endif /* DEVELOPMENT || DEBUG */
132
133 #define CPUPM_IDLE_WFE 0x5310300
134 #define CPUPM_IDLE_TIMER_WFE 0x5310304
135
136 /* When recommended, issue WFE with [FI]IRQ unmasked in the idle
137 * loop. The default.
138 */
139 uint32_t idle_proximate_io_wfe_unmasked = 1;
140 #if DEVELOPMENT || DEBUG
141 uint32_t idle_proximate_timer_wfe = 1;
142 uint32_t idle_proximate_io_wfe_masked = 0;
143 #else
144 /* Issue WFE in lieu of WFI when awaiting a proximate timer. */
145 static uint32_t idle_proximate_timer_wfe = 1;
146 /* When recommended, issue WFE with [FI]IRQ masked in the idle loop.
147 * Non-default, retained for experimentation.
148 */
149 static uint32_t idle_proximate_io_wfe_masked = 0;
150 #endif
151
152 #if __ARM_GLOBAL_SLEEP_BIT__
153 volatile boolean_t arm64_stall_sleep = TRUE;
154 #endif
155
156 #if WITH_CLASSIC_S2R
157 /*
158 * These must be aligned to avoid issues with calling bcopy_phys on them before
159 * we are done with pmap initialization.
160 */
161 static const uint8_t __attribute__ ((aligned(8))) suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
162 static const uint8_t __attribute__ ((aligned(8))) running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};
163 #endif
164
165 #if WITH_CLASSIC_S2R
166 static vm_offset_t sleepTokenBuffer = (vm_offset_t)NULL;
167 #endif
168 static boolean_t coresight_debug_enabled = FALSE;
169
170 #if defined(CONFIG_XNUPOST)
171 void arm64_ipi_test_callback(void *);
172 void arm64_immediate_ipi_test_callback(void *);
173
174 void
arm64_ipi_test_callback(void * parm)175 arm64_ipi_test_callback(void *parm)
176 {
177 volatile uint64_t *ipi_test_data = parm;
178 cpu_data_t *cpu_data;
179
180 cpu_data = getCpuDatap();
181
182 *ipi_test_data = cpu_data->cpu_number;
183 }
184
185 void
arm64_immediate_ipi_test_callback(void * parm)186 arm64_immediate_ipi_test_callback(void *parm)
187 {
188 volatile uint64_t *ipi_test_data = parm;
189 cpu_data_t *cpu_data;
190
191 cpu_data = getCpuDatap();
192
193 *ipi_test_data = cpu_data->cpu_number + MAX_CPUS;
194 }
195
196 uint64_t arm64_ipi_test_data[MAX_CPUS * 2];
197
198 MACHINE_TIMEOUT(arm64_ipi_test_timeout, "arm64-ipi-test", 100, MACHINE_TIMEOUT_UNIT_MSEC, NULL);
199
200 void
arm64_ipi_test()201 arm64_ipi_test()
202 {
203 volatile uint64_t *ipi_test_data, *immediate_ipi_test_data;
204 uint64_t timeout_ms = os_atomic_load(&arm64_ipi_test_timeout, relaxed);
205 uint64_t then, now, delta;
206 int current_cpu_number = getCpuDatap()->cpu_number;
207
208 /*
209 * probably the only way to have this on most systems is with the
210 * cpus=1 boot-arg, but nonetheless, if we only have 1 CPU active,
211 * IPI is not available
212 */
213 if (real_ncpus == 1) {
214 return;
215 }
216
217 const unsigned int max_cpu_id = ml_get_max_cpu_number();
218 for (unsigned int i = 0; i <= max_cpu_id; ++i) {
219 ipi_test_data = &arm64_ipi_test_data[i];
220 immediate_ipi_test_data = &arm64_ipi_test_data[i + MAX_CPUS];
221 *ipi_test_data = ~i;
222 kern_return_t error = cpu_xcall((int)i, (void *)arm64_ipi_test_callback, (void *)(uintptr_t)ipi_test_data);
223 if (error != KERN_SUCCESS) {
224 panic("CPU %d was unable to IPI CPU %u: error %d", current_cpu_number, i, error);
225 }
226
227 while ((error = cpu_immediate_xcall((int)i, (void *)arm64_immediate_ipi_test_callback,
228 (void *)(uintptr_t)immediate_ipi_test_data)) == KERN_ALREADY_WAITING) {
229 now = mach_absolute_time();
230 absolutetime_to_nanoseconds(now - then, &delta);
231 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
232 panic("CPU %d was unable to immediate-IPI CPU %u within %lldms", current_cpu_number, i, timeout_ms);
233 }
234 }
235
236 if (error != KERN_SUCCESS) {
237 panic("CPU %d was unable to immediate-IPI CPU %u: error %d", current_cpu_number, i, error);
238 }
239
240 then = mach_absolute_time();
241
242 while ((*ipi_test_data != i) || (*immediate_ipi_test_data != (i + MAX_CPUS))) {
243 now = mach_absolute_time();
244 absolutetime_to_nanoseconds(now - then, &delta);
245 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
246 panic("CPU %d tried to IPI CPU %d but didn't get correct responses within %lldms, responses: %llx, %llx",
247 current_cpu_number, i, timeout_ms, *ipi_test_data, *immediate_ipi_test_data);
248 }
249 }
250 }
251 }
252 #endif /* defined(CONFIG_XNUPOST) */
253
254 static void
configure_coresight_registers(cpu_data_t * cdp)255 configure_coresight_registers(cpu_data_t *cdp)
256 {
257 int i;
258
259 assert(cdp);
260 vm_offset_t coresight_regs = ml_get_topology_info()->cpus[cdp->cpu_number].coresight_regs;
261
262 /*
263 * ARMv8 coresight registers are optional. If the device tree did not
264 * provide either cpu_regmap_paddr (from the legacy "reg-private" EDT property)
265 * or coresight_regs (from the new "coresight-reg" property), assume that
266 * coresight registers are not supported.
267 */
268 if (cdp->cpu_regmap_paddr || coresight_regs) {
269 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
270 if (i == CORESIGHT_CTI) {
271 continue;
272 }
273 /* Skip debug-only registers on production chips */
274 if (((i == CORESIGHT_ED) || (i == CORESIGHT_UTT)) && !coresight_debug_enabled) {
275 continue;
276 }
277
278 if (!cdp->coresight_base[i]) {
279 if (coresight_regs) {
280 cdp->coresight_base[i] = coresight_regs + CORESIGHT_OFFSET(i);
281 } else {
282 uint64_t addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i);
283 cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE);
284 }
285 }
286 /* Unlock EDLAR, CTILAR, PMLAR */
287 if (i != CORESIGHT_UTT) {
288 *(volatile uint32_t *)(cdp->coresight_base[i] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
289 }
290 }
291 }
292 }
293
294
295 /*
296 * Routine: cpu_bootstrap
297 * Function:
298 */
299 void
cpu_bootstrap(void)300 cpu_bootstrap(void)
301 {
302 }
303
304 /*
305 * Routine: cpu_sleep
306 * Function:
307 */
308 void
cpu_sleep(void)309 cpu_sleep(void)
310 {
311 cpu_data_t *cpu_data_ptr = getCpuDatap();
312
313 cpu_data_ptr->cpu_active_thread = current_thread();
314 cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr;
315 cpu_data_ptr->cpu_flags |= SleepState;
316
317 if (cpu_data_ptr->cpu_user_debug != NULL) {
318 arm_debug_set(NULL);
319 }
320
321 #if KPC
322 kpc_idle();
323 #endif /* KPC */
324 #if MONOTONIC
325 mt_cpu_down(cpu_data_ptr);
326 #endif /* MONOTONIC */
327 #if KPERF
328 kptimer_stop_curcpu();
329 #endif /* KPERF */
330
331 CleanPoC_Dcache();
332
333 #if USE_APPLEARMSMP
334 if (ml_is_quiescing()) {
335 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
336 } else {
337 bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id);
338 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
339 // hang CPU on spurious wakeup
340 cpu_data_ptr->cpu_reset_handler = (uintptr_t)0;
341 __builtin_arm_dsb(DSB_ISH);
342 CleanPoU_Dcache();
343 arm64_prepare_for_sleep(deep_sleep, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBase));
344 }
345 #else
346 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
347 #endif
348 /*NOTREACHED*/
349 }
350
351 /*
352 * Routine: cpu_interrupt_is_pending
353 * Function: Returns a bool signifying a non-zero ISR_EL1,
354 * indicating a pending IRQ, FIQ or external abort.
355 */
356
357 bool
cpu_interrupt_is_pending(void)358 cpu_interrupt_is_pending(void)
359 {
360 uint64_t isr_value;
361 isr_value = __builtin_arm_rsr64("ISR_EL1");
362 return isr_value != 0;
363 }
364
365 static bool
cpu_proximate_timer(void)366 cpu_proximate_timer(void)
367 {
368 return !SetIdlePop();
369 }
370
371 #ifdef ARM64_BOARD_CONFIG_T6000
372 int wfe_allowed = 0;
373 #else
374 int wfe_allowed = 1;
375 #endif /* ARM64_BOARD_CONFIG_T6000 */
376
377 #if DEVELOPMENT || DEBUG
378 #define WFE_STAT(x) \
379 do { \
380 (x); \
381 } while(0)
382 #else
383 #define WFE_STAT(x) do {} while(0)
384 #endif /* DEVELOPMENT || DEBUG */
385
386 bool
wfe_to_deadline_or_interrupt(uint32_t cid,uint64_t wfe_deadline,cpu_data_t * cdp,bool unmask)387 wfe_to_deadline_or_interrupt(uint32_t cid, uint64_t wfe_deadline, cpu_data_t *cdp, bool unmask)
388 {
389 bool ipending = false;
390 uint64_t irqc = 0, nirqc = 0;
391
392 /* The ARMv8 architecture permits a processor dwelling in WFE
393 * with F/IRQ masked to ignore a pending interrupt, i.e.
394 * not classify it as an 'event'. This is potentially
395 * problematic with AICv2's IRQ distribution model, as
396 * a transient interrupt masked interval can cause an SIQ
397 * query rejection, possibly routing the interrupt to
398 * another core/cluster in a powergated state.
399 * Hence, optionally unmask IRQs+FIQs across WFE.
400 */
401 if (unmask) {
402 /* Latch SW IRQ+FIQ counter prior to unmasking
403 * interrupts.
404 */
405 irqc = nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
406 /* Unmask IRQ+FIQ. Mirrors mask used by machine_idle()
407 * with ASYNCF omission. Consider that this could
408 * delay recognition of an async abort, including
409 * those triggered by ISRs
410 */
411 __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF));
412 }
413
414 while ((ipending = (cpu_interrupt_is_pending())) == false) {
415 if (unmask) {
416 /* If WFE was issued with IRQs unmasked, an
417 * interrupt may have been processed.
418 * Consult the SW IRQ counter to determine
419 * whether the 'idle loop' must be
420 * re-evaluated.
421 */
422 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
423 if (nirqc != irqc) {
424 break;
425 }
426 }
427
428 if (__probable(wfe_allowed)) {
429 /*
430 * If IRQs are unmasked, there's a small window
431 * where an 'extra' WFE may be issued after
432 * the consultation of the SW interrupt counter
433 * and new interrupt arrival. Hence this WFE
434 * relies on the [FI]RQ interrupt handler
435 * epilogue issuing a 'SEVL', to post an
436 * event which causes the next WFE on the same
437 * PE to retire immediately.
438 */
439
440 __builtin_arm_wfe();
441 }
442
443 WFE_STAT(cdp->wfe_count++);
444 if (wfe_deadline != ~0ULL) {
445 WFE_STAT(cdp->wfe_deadline_checks++);
446 /* Check if the WFE recommendation has expired.
447 * We do not recompute the deadline here.
448 */
449 if ((ml_cluster_wfe_timeout(cid) == 0) ||
450 mach_absolute_time() >= wfe_deadline) {
451 WFE_STAT(cdp->wfe_terminations++);
452 break;
453 }
454 }
455 }
456
457 if (unmask) {
458 /* Re-mask IRQ+FIQ
459 * Mirrors mask used by machine_idle(), with ASYNCF
460 * omission
461 */
462 __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));
463 /* Refetch SW interrupt counter with IRQs masked
464 * It is important that this routine accurately flags
465 * any observed interrupts via its return value,
466 * inaccuracy may lead to an erroneous WFI fallback.
467 */
468 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
469 }
470
471 return ipending || (nirqc != irqc);
472 }
473
474 /*
475 * Routine: cpu_idle
476 * Function:
477 */
478 void __attribute__((noreturn))
cpu_idle(void)479 cpu_idle(void)
480 {
481 cpu_data_t *cpu_data_ptr = getCpuDatap();
482 uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop;
483 bool idle_disallowed = false;
484
485 if (__improbable((!idle_enable))) {
486 idle_disallowed = true;
487 } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) {
488 idle_disallowed = true;
489 }
490
491 if (__improbable(idle_disallowed)) {
492 Idle_load_context();
493 }
494
495 bool ipending = false;
496 uint32_t cid = cpu_data_ptr->cpu_cluster_id;
497
498 if (idle_proximate_io_wfe_masked == 1) {
499 uint64_t wfe_deadline = 0;
500 /* Check for an active perf. controller generated
501 * WFE recommendation for this cluster.
502 */
503 uint64_t wfe_ttd = 0;
504 if ((wfe_ttd = ml_cluster_wfe_timeout(cid)) != 0) {
505 wfe_deadline = mach_absolute_time() + wfe_ttd;
506 }
507
508 if (wfe_deadline != 0) {
509 /* Poll issuing event-bounded WFEs until an interrupt
510 * arrives or the WFE recommendation expires
511 */
512 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, wfe_ttd, cid);
513 ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr, false);
514 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, wfe_deadline);
515 if (ipending == true) {
516 /* Back to machine_idle() */
517 Idle_load_context();
518 }
519 }
520 }
521
522 if (__improbable(cpu_proximate_timer())) {
523 if (idle_proximate_timer_wfe == 1) {
524 /* Poll issuing WFEs until the expected
525 * timer FIQ arrives.
526 */
527 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, ~0ULL);
528 ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr, false);
529 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, ~0ULL);
530 assert(ipending == true);
531 }
532 Idle_load_context();
533 }
534
535 lastPop = cpu_data_ptr->rtcPop;
536
537 cpu_data_ptr->cpu_active_thread = current_thread();
538
539 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
540 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
541 }
542
543 if (cpu_data_ptr->idle_timer_notify != NULL) {
544 if (new_idle_timeout_ticks == 0x0ULL) {
545 /* turn off the idle timer */
546 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
547 } else {
548 /* set the new idle timeout */
549 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
550 }
551 timer_resync_deadlines();
552 if (cpu_data_ptr->rtcPop != lastPop) {
553 SetIdlePop();
554 }
555 }
556
557 #if KPC
558 kpc_idle();
559 #endif
560 #if MONOTONIC
561 mt_cpu_idle(cpu_data_ptr);
562 #endif /* MONOTONIC */
563
564 if (wfi) {
565 #if !defined(APPLE_ARM64_ARCH_FAMILY)
566 platform_cache_idle_enter();
567 #endif
568
569 #if DEVELOPMENT || DEBUG
570 // When simulating wfi overhead,
571 // force wfi to clock gating only
572 if (wfi == 2) {
573 arm64_force_wfi_clock_gate();
574 }
575 #endif /* DEVELOPMENT || DEBUG */
576
577 #if defined(APPLETYPHOON)
578 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
579 typhoon_prepare_for_wfi();
580 #endif
581 __builtin_arm_dsb(DSB_SY);
582 #if HAS_RETENTION_STATE
583 arm64_retention_wfi();
584 #else
585 __builtin_arm_wfi();
586 #endif
587
588 #if defined(APPLETYPHOON)
589 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
590 typhoon_return_from_wfi();
591 #endif
592
593 #if DEVELOPMENT || DEBUG
594 // Handle wfi overhead simulation
595 if (wfi == 2) {
596 uint64_t deadline;
597
598 // Calculate wfi delay deadline
599 clock_absolutetime_interval_to_deadline(wfi_delay, &deadline);
600
601 // Flush L1 caches
602 if ((wfi_flags & 1) != 0) {
603 InvalidatePoU_Icache();
604 FlushPoC_Dcache();
605 }
606
607 // Flush TLBs
608 if ((wfi_flags & 2) != 0) {
609 flush_core_tlb();
610 }
611
612 // Wait for the ballance of the wfi delay
613 clock_delay_until(deadline);
614 }
615 #endif /* DEVELOPMENT || DEBUG */
616 }
617
618 ClearIdlePop(TRUE);
619
620 cpu_idle_exit(FALSE);
621 }
622
623 /*
624 * Routine: cpu_idle_exit
625 * Function:
626 */
627 void
cpu_idle_exit(boolean_t from_reset)628 cpu_idle_exit(boolean_t from_reset)
629 {
630 uint64_t new_idle_timeout_ticks = 0x0ULL;
631 cpu_data_t *cpu_data_ptr = getCpuDatap();
632
633 assert(exception_stack_pointer() != 0);
634
635 /* Back from WFI, unlock OSLAR and EDLAR. */
636 if (from_reset) {
637 configure_coresight_registers(cpu_data_ptr);
638 }
639
640 #if KPC
641 kpc_idle_exit();
642 #endif
643
644 #if MONOTONIC
645 mt_cpu_run(cpu_data_ptr);
646 #endif /* MONOTONIC */
647
648 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
649 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
650 }
651
652 if (cpu_data_ptr->idle_timer_notify != NULL) {
653 if (new_idle_timeout_ticks == 0x0ULL) {
654 /* turn off the idle timer */
655 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
656 } else {
657 /* set the new idle timeout */
658 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
659 }
660 timer_resync_deadlines();
661 }
662
663 #if CONFIG_KERNEL_TBI && KASAN
664 kasan_unpoison_curstack(false);
665 #endif /* CONFIG_KERNEL_TBI && KASAN */
666
667 Idle_load_context();
668 }
669
670 void
cpu_init(void)671 cpu_init(void)
672 {
673 cpu_data_t *cdp = getCpuDatap();
674 arm_cpu_info_t *cpu_info_p;
675
676 assert(exception_stack_pointer() != 0);
677
678 if (cdp->cpu_type != CPU_TYPE_ARM64) {
679 cdp->cpu_type = CPU_TYPE_ARM64;
680
681 timer_call_queue_init(&cdp->rtclock_timer.queue);
682 cdp->rtclock_timer.deadline = EndOfAllTime;
683
684 if (cdp == &BootCpuData) {
685 do_cpuid();
686 do_mvfpid();
687 } else {
688 /*
689 * We initialize non-boot CPUs here; the boot CPU is
690 * dealt with as part of pmap_bootstrap.
691 */
692 pmap_cpu_data_init();
693 }
694
695 do_cacheid();
696
697 /* ARM_SMP: Assuming identical cpu */
698 do_debugid();
699
700 cpu_info_p = cpuid_info();
701
702 /* switch based on CPU's reported architecture */
703 switch (cpu_info_p->arm_info.arm_arch) {
704 case CPU_ARCH_ARMv8:
705 cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8;
706 break;
707 case CPU_ARCH_ARMv8E:
708 cdp->cpu_subtype = CPU_SUBTYPE_ARM64E;
709 break;
710 default:
711 //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL;
712 /* this panic doesn't work this early in startup */
713 panic("Unknown CPU subtype...");
714 break;
715 }
716
717 cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
718 }
719 cdp->cpu_stat.irq_ex_cnt_wake = 0;
720 cdp->cpu_stat.ipi_cnt_wake = 0;
721 #if MONOTONIC
722 cdp->cpu_stat.pmi_cnt_wake = 0;
723 #endif /* MONOTONIC */
724 cdp->cpu_running = TRUE;
725 cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
726 cdp->cpu_sleep_token = 0x0UL;
727 #if KPC
728 kpc_idle_exit();
729 #endif /* KPC */
730 #if MONOTONIC
731 mt_cpu_up(cdp);
732 #endif /* MONOTONIC */
733 }
734
735 void
cpu_stack_alloc(cpu_data_t * cpu_data_ptr)736 cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
737 {
738 vm_offset_t irq_stack = 0;
739 vm_offset_t exc_stack = 0;
740
741 kmem_alloc(kernel_map, &irq_stack,
742 INTSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
743 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
744 VM_KERN_MEMORY_STACK);
745
746 cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
747 cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top;
748
749 kmem_alloc(kernel_map, &exc_stack,
750 EXCEPSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
751 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
752 VM_KERN_MEMORY_STACK);
753
754 cpu_data_ptr->excepstack_top = exc_stack + PAGE_SIZE + EXCEPSTACK_SIZE;
755 cpu_data_ptr->excepstackptr = cpu_data_ptr->excepstack_top;
756 }
757
758 void
cpu_data_free(cpu_data_t * cpu_data_ptr)759 cpu_data_free(cpu_data_t *cpu_data_ptr)
760 {
761 if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) {
762 return;
763 }
764
765 int cpu_number = cpu_data_ptr->cpu_number;
766
767 if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
768 CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
769 CpuDataEntries[cpu_number].cpu_data_paddr = 0;
770 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
771 }
772 kmem_free(kernel_map,
773 cpu_data_ptr->intstack_top - INTSTACK_SIZE - PAGE_SIZE,
774 INTSTACK_SIZE + 2 * PAGE_SIZE);
775 kmem_free(kernel_map,
776 cpu_data_ptr->excepstack_top - EXCEPSTACK_SIZE - PAGE_SIZE,
777 EXCEPSTACK_SIZE + 2 * PAGE_SIZE);
778 }
779
780 void
cpu_data_init(cpu_data_t * cpu_data_ptr)781 cpu_data_init(cpu_data_t *cpu_data_ptr)
782 {
783 uint32_t i;
784
785 cpu_data_ptr->cpu_flags = 0;
786 cpu_data_ptr->cpu_int_state = 0;
787 cpu_data_ptr->cpu_pending_ast = AST_NONE;
788 cpu_data_ptr->cpu_cache_dispatch = NULL;
789 cpu_data_ptr->rtcPop = EndOfAllTime;
790 cpu_data_ptr->rtclock_datap = &RTClockData;
791 cpu_data_ptr->cpu_user_debug = NULL;
792
793
794 cpu_data_ptr->cpu_base_timebase = 0;
795 cpu_data_ptr->cpu_idle_notify = NULL;
796 cpu_data_ptr->cpu_idle_latency = 0x0ULL;
797 cpu_data_ptr->cpu_idle_pop = 0x0ULL;
798 cpu_data_ptr->cpu_reset_type = 0x0UL;
799 cpu_data_ptr->cpu_reset_handler = 0x0UL;
800 cpu_data_ptr->cpu_reset_assist = 0x0UL;
801 cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
802 cpu_data_ptr->cpu_phys_id = 0x0UL;
803 cpu_data_ptr->cpu_l2_access_penalty = 0;
804 cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
805 cpu_data_ptr->cpu_cluster_id = 0;
806 cpu_data_ptr->cpu_l2_id = 0;
807 cpu_data_ptr->cpu_l2_size = 0;
808 cpu_data_ptr->cpu_l3_id = 0;
809 cpu_data_ptr->cpu_l3_size = 0;
810
811 cpu_data_ptr->cpu_signal = SIGPdisabled;
812
813 cpu_data_ptr->cpu_get_fiq_handler = NULL;
814 cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
815 cpu_data_ptr->cpu_tbd_hardware_val = NULL;
816 cpu_data_ptr->cpu_get_decrementer_func = NULL;
817 cpu_data_ptr->cpu_set_decrementer_func = NULL;
818 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
819 cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
820 cpu_data_ptr->cpu_xcall_p0 = NULL;
821 cpu_data_ptr->cpu_xcall_p1 = NULL;
822 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
823 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
824
825 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
826 cpu_data_ptr->coresight_base[i] = 0;
827 }
828
829 #if !XNU_MONITOR
830 pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
831
832 pmap_cpu_data_ptr->cpu_nested_pmap = (struct pmap *) NULL;
833 pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
834 pmap_cpu_data_ptr->pv_free.list = NULL;
835 pmap_cpu_data_ptr->pv_free.count = 0;
836 pmap_cpu_data_ptr->pv_free_spill_marker = NULL;
837
838 bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
839 #endif
840 cpu_data_ptr->halt_status = CPU_NOT_HALTED;
841 #if __ARM_KERNEL_PROTECT__
842 cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table;
843 #endif /* __ARM_KERNEL_PROTECT__ */
844
845 #if defined(HAS_APPLE_PAC)
846 cpu_data_ptr->rop_key = 0;
847 cpu_data_ptr->jop_key = ml_default_jop_pid();
848 #endif
849 }
850
851 kern_return_t
cpu_data_register(cpu_data_t * cpu_data_ptr)852 cpu_data_register(cpu_data_t *cpu_data_ptr)
853 {
854 int cpu = cpu_data_ptr->cpu_number;
855
856 #if KASAN
857 for (int i = 0; i < CPUWINDOWS_MAX; i++) {
858 kasan_notify_address_nopoison(pmap_cpu_windows_copy_addr(cpu, i), PAGE_SIZE);
859 }
860 #endif
861
862 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
863 CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
864 CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
865 return KERN_SUCCESS;
866 }
867
868 #if defined(KERNEL_INTEGRITY_CTRR)
869 /* Hibernation needs to reset this state, so data and text are in the hib segment;
870 * this allows them be accessed and executed early.
871 */
872 LCK_GRP_DECLARE(ctrr_cpu_start_lock_grp, "ctrr_cpu_start_lock");
873 LCK_SPIN_DECLARE(ctrr_cpu_start_lck, &ctrr_cpu_start_lock_grp);
874 enum ctrr_cluster_states ctrr_cluster_locked[MAX_CPU_CLUSTERS] MARK_AS_HIBERNATE_DATA;
875
876 MARK_AS_HIBERNATE_TEXT
877 void
init_ctrr_cluster_states(void)878 init_ctrr_cluster_states(void)
879 {
880 for (int i = 0; i < MAX_CPU_CLUSTERS; i++) {
881 ctrr_cluster_locked[i] = CTRR_UNLOCKED;
882 }
883 }
884 #endif
885
886 kern_return_t
cpu_start(int cpu)887 cpu_start(int cpu)
888 {
889 cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
890 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
891
892 if (processor_should_kprintf(processor, true)) {
893 kprintf("cpu_start() cpu: %d\n", cpu);
894 }
895
896 if (cpu == cpu_number()) {
897 cpu_machine_init();
898 configure_coresight_registers(cpu_data_ptr);
899 } else {
900 thread_t first_thread;
901
902 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
903
904 #if !XNU_MONITOR
905 cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL;
906 #endif
907
908 if (processor->startup_thread != THREAD_NULL) {
909 first_thread = processor->startup_thread;
910 } else {
911 first_thread = processor->idle_thread;
912 }
913 cpu_data_ptr->cpu_active_thread = first_thread;
914 first_thread->machine.CpuDatap = cpu_data_ptr;
915 first_thread->machine.pcpu_data_base =
916 (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);
917
918 configure_coresight_registers(cpu_data_ptr);
919
920 flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
921 flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
922 #if defined(KERNEL_INTEGRITY_CTRR)
923
924 /* First CPU being started within a cluster goes ahead to lock CTRR for cluster;
925 * other CPUs block until cluster is locked. */
926 lck_spin_lock(&ctrr_cpu_start_lck);
927 switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) {
928 case CTRR_UNLOCKED:
929 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING;
930 lck_spin_unlock(&ctrr_cpu_start_lck);
931 break;
932 case CTRR_LOCKING:
933 assert_wait(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id], THREAD_UNINT);
934 lck_spin_unlock(&ctrr_cpu_start_lck);
935 thread_block(THREAD_CONTINUE_NULL);
936 assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING);
937 break;
938 default: // CTRR_LOCKED
939 lck_spin_unlock(&ctrr_cpu_start_lck);
940 break;
941 }
942 #endif
943 (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
944 }
945
946 return KERN_SUCCESS;
947 }
948
949
950 void
cpu_timebase_init(boolean_t from_boot)951 cpu_timebase_init(boolean_t from_boot)
952 {
953 cpu_data_t *cdp = getCpuDatap();
954
955 if (cdp->cpu_get_fiq_handler == NULL) {
956 cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
957 cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
958 cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
959 cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
960 cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
961 }
962
963 if (!from_boot && (cdp == &BootCpuData)) {
964 /*
965 * When we wake from sleep, we have no guarantee about the state
966 * of the hardware timebase. It may have kept ticking across sleep, or
967 * it may have reset.
968 *
969 * To deal with this, we calculate an offset to the clock that will
970 * produce a timebase value wake_abstime at the point the boot
971 * CPU calls cpu_timebase_init on wake.
972 *
973 * This ensures that mach_absolute_time() stops ticking across sleep.
974 */
975 rtclock_base_abstime = wake_abstime - ml_get_hwclock();
976 } else if (from_boot) {
977 /* On initial boot, initialize time_since_reset to CNTPCT_EL0. */
978 ml_set_reset_time(ml_get_hwclock());
979 }
980
981 cdp->cpu_decrementer = 0x7FFFFFFFUL;
982 cdp->cpu_timebase = 0x0UL;
983 cdp->cpu_base_timebase = rtclock_base_abstime;
984 }
985
986 int
cpu_cluster_id(void)987 cpu_cluster_id(void)
988 {
989 return getCpuDatap()->cpu_cluster_id;
990 }
991
992 __attribute__((noreturn))
993 void
ml_arm_sleep(void)994 ml_arm_sleep(void)
995 {
996 cpu_data_t *cpu_data_ptr = getCpuDatap();
997
998 if (cpu_data_ptr == &BootCpuData) {
999 cpu_data_t *target_cdp;
1000 int cpu;
1001 int max_cpu;
1002
1003 max_cpu = ml_get_max_cpu_number();
1004 for (cpu = 0; cpu <= max_cpu; cpu++) {
1005 target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1006
1007 if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) {
1008 continue;
1009 }
1010
1011 while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
1012 ;
1013 }
1014 }
1015
1016 /*
1017 * Now that the other cores have entered the sleep path, set
1018 * the abstime value we'll use when we resume.
1019 */
1020 wake_abstime = ml_get_timebase();
1021 ml_set_reset_time(UINT64_MAX);
1022 } else {
1023 CleanPoU_Dcache();
1024 }
1025
1026 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
1027
1028 if (cpu_data_ptr == &BootCpuData) {
1029 #if WITH_CLASSIC_S2R
1030 // Classic suspend to RAM writes the suspend signature into the
1031 // sleep token buffer so that iBoot knows that it's on the warm
1032 // boot (wake) path (as opposed to the cold boot path). Newer SoC
1033 // do not go through SecureROM/iBoot on the warm boot path. The
1034 // reconfig engine script brings the CPU out of reset at the kernel's
1035 // reset vector which points to the warm boot initialization code.
1036 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1037 platform_cache_shutdown();
1038 bcopy((const void *)suspend_signature, (void *)sleepTokenBuffer, sizeof(SleepToken));
1039 } else {
1040 panic("No sleep token buffer");
1041 }
1042 #endif
1043
1044 #if __ARM_GLOBAL_SLEEP_BIT__
1045 /* Allow other CPUs to go to sleep. */
1046 arm64_stall_sleep = FALSE;
1047 __builtin_arm_dmb(DMB_ISH);
1048 #endif
1049
1050 /* Architectural debug state: <rdar://problem/12390433>:
1051 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1052 * tell debugger to not prevent power gating .
1053 */
1054 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1055 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1056 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1057 }
1058
1059 #if HIBERNATION
1060 uint32_t mode = hibernate_write_image();
1061 if (mode == kIOHibernatePostWriteHalt) {
1062 HIBLOG("powering off after writing hibernation image\n");
1063 int halt_result = -1;
1064 if (PE_halt_restart) {
1065 /**
1066 * Drain serial FIFOs now as the normal call further down won't
1067 * be hit when the CPU halts here for hibernation. Here, it'll
1068 * make sure the preceding HIBLOG is flushed as well.
1069 */
1070 serial_go_to_sleep();
1071 halt_result = (*PE_halt_restart)(kPEHaltCPU);
1072 }
1073 panic("can't shutdown: PE_halt_restart returned %d", halt_result);
1074 }
1075 #endif /* HIBERNATION */
1076
1077 serial_go_to_sleep();
1078
1079 #if MONOTONIC
1080 mt_sleep();
1081 #endif /* MONOTONIC */
1082 /* ARM64-specific preparation */
1083 arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBase));
1084 } else {
1085 #if __ARM_GLOBAL_SLEEP_BIT__
1086 /*
1087 * With the exception of the CPU revisions listed above, our ARM64 CPUs have a
1088 * global register to manage entering deep sleep, as opposed to a per-CPU
1089 * register. We cannot update this register until all CPUs are ready to enter
1090 * deep sleep, because if a CPU executes WFI outside of the deep sleep context
1091 * (by idling), it will hang (due to the side effects of enabling deep sleep),
1092 * which can hang the sleep process or cause memory corruption on wake.
1093 *
1094 * To avoid these issues, we'll stall on this global value, which CPU0 will
1095 * manage.
1096 */
1097 while (arm64_stall_sleep) {
1098 __builtin_arm_wfe();
1099 }
1100 #endif
1101 CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
1102
1103 /* Architectural debug state: <rdar://problem/12390433>:
1104 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1105 * tell debugger to not prevent power gating .
1106 */
1107 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1108 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1109 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1110 }
1111
1112 /* ARM64-specific preparation */
1113 arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBase));
1114 }
1115 }
1116
1117 void
cpu_machine_idle_init(boolean_t from_boot)1118 cpu_machine_idle_init(boolean_t from_boot)
1119 {
1120 static vm_address_t resume_idle_cpu_paddr = (vm_address_t)NULL;
1121 cpu_data_t *cpu_data_ptr = getCpuDatap();
1122
1123 if (from_boot) {
1124 uint32_t production = 1;
1125 DTEntry entry;
1126
1127 unsigned long jtag = 0;
1128
1129 if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
1130 if (jtag != 0) {
1131 idle_enable = FALSE;
1132 } else {
1133 idle_enable = TRUE;
1134 }
1135 } else {
1136 idle_enable = TRUE;
1137 }
1138
1139 #if DEVELOPMENT || DEBUG
1140 uint32_t wfe_mode = 0;
1141 if (PE_parse_boot_argn("wfe_mode", &wfe_mode, sizeof(wfe_mode))) {
1142 idle_proximate_timer_wfe = ((wfe_mode & 1) == 1);
1143 idle_proximate_io_wfe_masked = ((wfe_mode & 2) == 2);
1144 extern uint32_t idle_proximate_io_wfe_unmasked;
1145 idle_proximate_io_wfe_unmasked = ((wfe_mode & 4) == 4);
1146 }
1147 #endif
1148
1149 // bits 7..0 give the wfi type
1150 switch (wfi & 0xff) {
1151 case 0:
1152 // disable wfi
1153 wfi = 0;
1154 break;
1155
1156 #if DEVELOPMENT || DEBUG
1157 case 2:
1158 // wfi overhead simulation
1159 // 31..16 - wfi delay is us
1160 // 15..8 - flags
1161 // 7..0 - 2
1162 wfi = 2;
1163 wfi_flags = (wfi >> 8) & 0xFF;
1164 nanoseconds_to_absolutetime(((wfi >> 16) & 0xFFFF) * NSEC_PER_MSEC, &wfi_delay);
1165 break;
1166 #endif /* DEVELOPMENT || DEBUG */
1167
1168 case 1:
1169 default:
1170 // do nothing
1171 break;
1172 }
1173
1174 ResetHandlerData.assist_reset_handler = 0;
1175 ResetHandlerData.cpu_data_entries = ml_static_vtop((vm_offset_t)CpuDataEntries);
1176
1177 #ifdef MONITOR
1178 monitor_call(MONITOR_SET_ENTRY, (uintptr_t)ml_static_vtop((vm_offset_t)&LowResetVectorBase), 0, 0);
1179 #elif !defined(NO_MONITOR)
1180 #error MONITOR undefined, WFI power gating may not operate correctly
1181 #endif /* MONITOR */
1182
1183 // Determine if we are on production or debug chip
1184 if (kSuccess == SecureDTLookupEntry(NULL, "/chosen", &entry)) {
1185 unsigned int size;
1186 void const *prop;
1187
1188 if (kSuccess == SecureDTGetProperty(entry, "effective-production-status-ap", &prop, &size)) {
1189 if (size == 4) {
1190 bcopy(prop, &production, size);
1191 }
1192 }
1193 }
1194 if (!production) {
1195 #if defined(APPLE_ARM64_ARCH_FAMILY)
1196 // Enable coresight debug registers on debug-fused chips
1197 coresight_debug_enabled = TRUE;
1198 #endif
1199 }
1200
1201 start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);
1202 resume_idle_cpu_paddr = ml_static_vtop((vm_offset_t)&resume_idle_cpu);
1203 }
1204
1205 #if WITH_CLASSIC_S2R
1206 if (cpu_data_ptr == &BootCpuData) {
1207 static addr64_t SleepToken_low_paddr = (addr64_t)NULL;
1208 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1209 SleepToken_low_paddr = ml_vtophys(sleepTokenBuffer);
1210 } else {
1211 panic("No sleep token buffer");
1212 }
1213
1214 bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)running_signature),
1215 SleepToken_low_paddr, sizeof(SleepToken));
1216 flush_dcache((vm_offset_t)SleepToken, sizeof(SleepToken), TRUE);
1217 }
1218 ;
1219 #endif
1220
1221 cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
1222 clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
1223 }
1224
1225 _Atomic uint32_t cpu_idle_count = 0;
1226
1227 void
machine_track_platform_idle(boolean_t entry)1228 machine_track_platform_idle(boolean_t entry)
1229 {
1230 if (entry) {
1231 os_atomic_inc(&cpu_idle_count, relaxed);
1232 } else {
1233 os_atomic_dec(&cpu_idle_count, relaxed);
1234 }
1235 }
1236
1237 #if WITH_CLASSIC_S2R
1238 void
sleep_token_buffer_init(void)1239 sleep_token_buffer_init(void)
1240 {
1241 cpu_data_t *cpu_data_ptr = getCpuDatap();
1242 DTEntry entry;
1243 size_t size;
1244 void const * const *prop;
1245
1246 if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) {
1247 /* Find the stpage node in the device tree */
1248 if (kSuccess != SecureDTLookupEntry(0, "stram", &entry)) {
1249 return;
1250 }
1251
1252 if (kSuccess != SecureDTGetProperty(entry, "reg", (const void **)&prop, (unsigned int *)&size)) {
1253 return;
1254 }
1255
1256 /* Map the page into the kernel space */
1257 sleepTokenBuffer = ml_io_map(((vm_offset_t const *)prop)[0], ((vm_size_t const *)prop)[1]);
1258 }
1259 }
1260 #endif
1261