1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm64/cpu.c
30 *
31 * cpu specific routines
32 */
33
34 #include <pexpert/arm64/board_config.h>
35 #include <kern/kalloc.h>
36 #include <kern/machine.h>
37 #include <kern/cpu_number.h>
38 #include <kern/percpu.h>
39 #include <kern/thread.h>
40 #include <kern/timer_queue.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <arm64/proc_reg.h>
50 #include <mach/processor_info.h>
51 #include <vm/pmap.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58
59 #include <machine/atomic.h>
60
61 #include <san/kasan.h>
62
63 #if KPC
64 #include <kern/kpc.h>
65 #endif
66
67 #if MONOTONIC
68 #include <kern/monotonic.h>
69 #endif /* MONOTONIC */
70
71 #if KPERF
72 #include <kperf/kptimer.h>
73 #endif /* KPERF */
74
75 #if HIBERNATION
76 #include <IOKit/IOPlatformExpert.h>
77 #include <IOKit/IOHibernatePrivate.h>
78 #endif /* HIBERNATION */
79
80
81 #include <libkern/section_keywords.h>
82
83 extern boolean_t idle_enable;
84 extern uint64_t wake_abstime;
85
86 #if WITH_CLASSIC_S2R
87 void sleep_token_buffer_init(void);
88 #endif
89
90
91 extern uintptr_t resume_idle_cpu;
92 extern uintptr_t start_cpu;
93
94 #if __ARM_KERNEL_PROTECT__
95 extern void exc_vectors_table;
96 #endif /* __ARM_KERNEL_PROTECT__ */
97
98 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep, unsigned int cpu, uint64_t entry_pa);
99 extern void arm64_force_wfi_clock_gate(void);
100 #if defined(APPLETYPHOON)
101 // <rdar://problem/15827409>
102 extern void typhoon_prepare_for_wfi(void);
103 extern void typhoon_return_from_wfi(void);
104 #endif
105
106 #if HAS_RETENTION_STATE
107 extern void arm64_retention_wfi(void);
108 #endif
109
110 vm_address_t start_cpu_paddr;
111
112 sysreg_restore_t sysreg_restore __attribute__((section("__DATA, __const"))) = {
113 .tcr_el1 = TCR_EL1_BOOT,
114 };
115
116 // wfi - wfi mode
117 // 0 : disabled
118 // 1 : normal
119 // 2 : overhead simulation (delay & flags)
120 TUNABLE(unsigned int, wfi, "wfi", 1);
121 #if DEVELOPMENT || DEBUG
122
123 // wfi_flags
124 // 1 << 0 : flush L1s
125 // 1 << 1 : flush TLBs
126 static int wfi_flags = 0;
127
128 // wfi_delay - delay ticks after wfi exit
129 static uint64_t wfi_delay = 0;
130
131 #endif /* DEVELOPMENT || DEBUG */
132
133 #define CPUPM_IDLE_WFE 0x5310300
134 #define CPUPM_IDLE_TIMER_WFE 0x5310304
135
136 /* When recommended, issue WFE with [FI]IRQ unmasked in the idle
137 * loop. The default.
138 */
139 uint32_t idle_proximate_io_wfe_unmasked = 1;
140 #if DEVELOPMENT || DEBUG
141 uint32_t idle_proximate_timer_wfe = 1;
142 uint32_t idle_proximate_io_wfe_masked = 0;
143 #else
144 /* Issue WFE in lieu of WFI when awaiting a proximate timer. */
145 static uint32_t idle_proximate_timer_wfe = 1;
146 /* When recommended, issue WFE with [FI]IRQ masked in the idle loop.
147 * Non-default, retained for experimentation.
148 */
149 static uint32_t idle_proximate_io_wfe_masked = 0;
150 #endif
151
152 #if __ARM_GLOBAL_SLEEP_BIT__
153 volatile boolean_t arm64_stall_sleep = TRUE;
154 #endif
155
156 #if WITH_CLASSIC_S2R
157 /*
158 * These must be aligned to avoid issues with calling bcopy_phys on them before
159 * we are done with pmap initialization.
160 */
161 static const uint8_t __attribute__ ((aligned(8))) suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
162 static const uint8_t __attribute__ ((aligned(8))) running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};
163 #endif
164
165 #if WITH_CLASSIC_S2R
166 static vm_offset_t sleepTokenBuffer = (vm_offset_t)NULL;
167 #endif
168 static boolean_t coresight_debug_enabled = FALSE;
169
170 #if defined(CONFIG_XNUPOST)
171 void arm64_ipi_test_callback(void *);
172 void arm64_immediate_ipi_test_callback(void *);
173
174 void
arm64_ipi_test_callback(void * parm)175 arm64_ipi_test_callback(void *parm)
176 {
177 volatile uint64_t *ipi_test_data = parm;
178 cpu_data_t *cpu_data;
179
180 cpu_data = getCpuDatap();
181
182 *ipi_test_data = cpu_data->cpu_number;
183 }
184
185 void
arm64_immediate_ipi_test_callback(void * parm)186 arm64_immediate_ipi_test_callback(void *parm)
187 {
188 volatile uint64_t *ipi_test_data = parm;
189 cpu_data_t *cpu_data;
190
191 cpu_data = getCpuDatap();
192
193 *ipi_test_data = cpu_data->cpu_number + MAX_CPUS;
194 }
195
196 uint64_t arm64_ipi_test_data[MAX_CPUS * 2];
197
198 MACHINE_TIMEOUT(arm64_ipi_test_timeout, "arm64-ipi-test", 100, MACHINE_TIMEOUT_UNIT_MSEC, NULL);
199
200 void
arm64_ipi_test()201 arm64_ipi_test()
202 {
203 volatile uint64_t *ipi_test_data, *immediate_ipi_test_data;
204 uint64_t timeout_ms = os_atomic_load(&arm64_ipi_test_timeout, relaxed);
205 uint64_t then, now, delta;
206 int current_cpu_number = getCpuDatap()->cpu_number;
207
208 /*
209 * probably the only way to have this on most systems is with the
210 * cpus=1 boot-arg, but nonetheless, if we only have 1 CPU active,
211 * IPI is not available
212 */
213 if (real_ncpus == 1) {
214 return;
215 }
216
217 const unsigned int max_cpu_id = ml_get_max_cpu_number();
218 for (unsigned int i = 0; i <= max_cpu_id; ++i) {
219 ipi_test_data = &arm64_ipi_test_data[i];
220 immediate_ipi_test_data = &arm64_ipi_test_data[i + MAX_CPUS];
221 *ipi_test_data = ~i;
222 kern_return_t error = cpu_xcall((int)i, (void *)arm64_ipi_test_callback, (void *)(uintptr_t)ipi_test_data);
223 if (error != KERN_SUCCESS) {
224 panic("CPU %d was unable to IPI CPU %u: error %d", current_cpu_number, i, error);
225 }
226
227 while ((error = cpu_immediate_xcall((int)i, (void *)arm64_immediate_ipi_test_callback,
228 (void *)(uintptr_t)immediate_ipi_test_data)) == KERN_ALREADY_WAITING) {
229 now = mach_absolute_time();
230 absolutetime_to_nanoseconds(now - then, &delta);
231 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
232 panic("CPU %d was unable to immediate-IPI CPU %u within %lldms", current_cpu_number, i, timeout_ms);
233 }
234 }
235
236 if (error != KERN_SUCCESS) {
237 panic("CPU %d was unable to immediate-IPI CPU %u: error %d", current_cpu_number, i, error);
238 }
239
240 then = mach_absolute_time();
241
242 while ((*ipi_test_data != i) || (*immediate_ipi_test_data != (i + MAX_CPUS))) {
243 now = mach_absolute_time();
244 absolutetime_to_nanoseconds(now - then, &delta);
245 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
246 panic("CPU %d tried to IPI CPU %d but didn't get correct responses within %lldms, responses: %llx, %llx",
247 current_cpu_number, i, timeout_ms, *ipi_test_data, *immediate_ipi_test_data);
248 }
249 }
250 }
251 }
252 #endif /* defined(CONFIG_XNUPOST) */
253
254 static void
configure_coresight_registers(cpu_data_t * cdp)255 configure_coresight_registers(cpu_data_t *cdp)
256 {
257 int i;
258
259 assert(cdp);
260 vm_offset_t coresight_regs = ml_get_topology_info()->cpus[cdp->cpu_number].coresight_regs;
261
262 /*
263 * ARMv8 coresight registers are optional. If the device tree did not
264 * provide either cpu_regmap_paddr (from the legacy "reg-private" EDT property)
265 * or coresight_regs (from the new "coresight-reg" property), assume that
266 * coresight registers are not supported.
267 */
268 if (cdp->cpu_regmap_paddr || coresight_regs) {
269 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
270 if (i == CORESIGHT_CTI) {
271 continue;
272 }
273 /* Skip debug-only registers on production chips */
274 if (((i == CORESIGHT_ED) || (i == CORESIGHT_UTT)) && !coresight_debug_enabled) {
275 continue;
276 }
277
278 if (!cdp->coresight_base[i]) {
279 if (coresight_regs) {
280 cdp->coresight_base[i] = coresight_regs + CORESIGHT_OFFSET(i);
281 } else {
282 uint64_t addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i);
283 cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE);
284 }
285
286 /*
287 * At this point, failing to io map the
288 * registers is considered as an error.
289 */
290 if (!cdp->coresight_base[i]) {
291 panic("unable to ml_io_map coresight regions");
292 }
293 }
294 /* Unlock EDLAR, CTILAR, PMLAR */
295 if (i != CORESIGHT_UTT) {
296 *(volatile uint32_t *)(cdp->coresight_base[i] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
297 }
298 }
299 }
300 }
301
302
303 /*
304 * Routine: cpu_bootstrap
305 * Function:
306 */
307 void
cpu_bootstrap(void)308 cpu_bootstrap(void)
309 {
310 }
311
312 /*
313 * Routine: cpu_sleep
314 * Function:
315 */
316 void
cpu_sleep(void)317 cpu_sleep(void)
318 {
319 cpu_data_t *cpu_data_ptr = getCpuDatap();
320
321 cpu_data_ptr->cpu_active_thread = current_thread();
322 cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr;
323 cpu_data_ptr->cpu_flags |= SleepState;
324
325 if (cpu_data_ptr->cpu_user_debug != NULL) {
326 arm_debug_set(NULL);
327 }
328
329 #if KPC
330 kpc_idle();
331 #endif /* KPC */
332 #if MONOTONIC
333 mt_cpu_down(cpu_data_ptr);
334 #endif /* MONOTONIC */
335 #if KPERF
336 kptimer_stop_curcpu();
337 #endif /* KPERF */
338
339 CleanPoC_Dcache();
340
341 #if USE_APPLEARMSMP
342 if (ml_is_quiescing()) {
343 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
344 } else {
345 bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id);
346 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
347 // hang CPU on spurious wakeup
348 cpu_data_ptr->cpu_reset_handler = (uintptr_t)0;
349 __builtin_arm_dsb(DSB_ISH);
350 CleanPoU_Dcache();
351 arm64_prepare_for_sleep(deep_sleep, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBase));
352 }
353 #else
354 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
355 #endif
356 /*NOTREACHED*/
357 }
358
359 /*
360 * Routine: cpu_interrupt_is_pending
361 * Function: Returns a bool signifying a non-zero ISR_EL1,
362 * indicating a pending IRQ, FIQ or external abort.
363 */
364
365 bool
cpu_interrupt_is_pending(void)366 cpu_interrupt_is_pending(void)
367 {
368 uint64_t isr_value;
369 isr_value = __builtin_arm_rsr64("ISR_EL1");
370 return isr_value != 0;
371 }
372
373 static bool
cpu_proximate_timer(void)374 cpu_proximate_timer(void)
375 {
376 return !SetIdlePop();
377 }
378
379 #ifdef ARM64_BOARD_CONFIG_T6000
380 int wfe_allowed = 0;
381 #else
382 int wfe_allowed = 1;
383 #endif /* ARM64_BOARD_CONFIG_T6000 */
384
385 #if DEVELOPMENT || DEBUG
386 #define WFE_STAT(x) \
387 do { \
388 (x); \
389 } while(0)
390 #else
391 #define WFE_STAT(x) do {} while(0)
392 #endif /* DEVELOPMENT || DEBUG */
393
394 bool
wfe_to_deadline_or_interrupt(uint32_t cid,uint64_t wfe_deadline,cpu_data_t * cdp,bool unmask)395 wfe_to_deadline_or_interrupt(uint32_t cid, uint64_t wfe_deadline, cpu_data_t *cdp, bool unmask)
396 {
397 bool ipending = false;
398 uint64_t irqc = 0, nirqc = 0;
399
400 /* The ARMv8 architecture permits a processor dwelling in WFE
401 * with F/IRQ masked to ignore a pending interrupt, i.e.
402 * not classify it as an 'event'. This is potentially
403 * problematic with AICv2's IRQ distribution model, as
404 * a transient interrupt masked interval can cause an SIQ
405 * query rejection, possibly routing the interrupt to
406 * another core/cluster in a powergated state.
407 * Hence, optionally unmask IRQs+FIQs across WFE.
408 */
409 if (unmask) {
410 /* Latch SW IRQ+FIQ counter prior to unmasking
411 * interrupts.
412 */
413 irqc = nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
414 /* Unmask IRQ+FIQ. Mirrors mask used by machine_idle()
415 * with ASYNCF omission. Consider that this could
416 * delay recognition of an async abort, including
417 * those triggered by ISRs
418 */
419 __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF));
420 }
421
422 while ((ipending = (cpu_interrupt_is_pending())) == false) {
423 if (unmask) {
424 /* If WFE was issued with IRQs unmasked, an
425 * interrupt may have been processed.
426 * Consult the SW IRQ counter to determine
427 * whether the 'idle loop' must be
428 * re-evaluated.
429 */
430 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
431 if (nirqc != irqc) {
432 break;
433 }
434 }
435
436 if (__probable(wfe_allowed)) {
437 /*
438 * If IRQs are unmasked, there's a small window
439 * where an 'extra' WFE may be issued after
440 * the consultation of the SW interrupt counter
441 * and new interrupt arrival. Hence this WFE
442 * relies on the [FI]RQ interrupt handler
443 * epilogue issuing a 'SEVL', to post an
444 * event which causes the next WFE on the same
445 * PE to retire immediately.
446 */
447
448 __builtin_arm_wfe();
449 }
450
451 WFE_STAT(cdp->wfe_count++);
452 if (wfe_deadline != ~0ULL) {
453 WFE_STAT(cdp->wfe_deadline_checks++);
454 /* Check if the WFE recommendation has expired.
455 * We do not recompute the deadline here.
456 */
457 if ((ml_cluster_wfe_timeout(cid) == 0) ||
458 mach_absolute_time() >= wfe_deadline) {
459 WFE_STAT(cdp->wfe_terminations++);
460 break;
461 }
462 }
463 }
464
465 if (unmask) {
466 /* Re-mask IRQ+FIQ
467 * Mirrors mask used by machine_idle(), with ASYNCF
468 * omission
469 */
470 __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));
471 /* Refetch SW interrupt counter with IRQs masked
472 * It is important that this routine accurately flags
473 * any observed interrupts via its return value,
474 * inaccuracy may lead to an erroneous WFI fallback.
475 */
476 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
477 }
478
479 return ipending || (nirqc != irqc);
480 }
481
482 /*
483 * Routine: cpu_idle
484 * Function:
485 */
486 void __attribute__((noreturn))
cpu_idle(void)487 cpu_idle(void)
488 {
489 cpu_data_t *cpu_data_ptr = getCpuDatap();
490 uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop;
491 bool idle_disallowed = false;
492
493 if (__improbable((!idle_enable))) {
494 idle_disallowed = true;
495 } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) {
496 idle_disallowed = true;
497 }
498
499 if (__improbable(idle_disallowed)) {
500 Idle_load_context();
501 }
502
503 bool ipending = false;
504 uint32_t cid = cpu_data_ptr->cpu_cluster_id;
505
506 if (idle_proximate_io_wfe_masked == 1) {
507 uint64_t wfe_deadline = 0;
508 /* Check for an active perf. controller generated
509 * WFE recommendation for this cluster.
510 */
511 uint64_t wfe_ttd = 0;
512 if ((wfe_ttd = ml_cluster_wfe_timeout(cid)) != 0) {
513 wfe_deadline = mach_absolute_time() + wfe_ttd;
514 }
515
516 if (wfe_deadline != 0) {
517 /* Poll issuing event-bounded WFEs until an interrupt
518 * arrives or the WFE recommendation expires
519 */
520 #if DEVELOPMENT || DEBUG
521 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, wfe_ttd, cid);
522 #endif
523 ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr, false);
524 #if DEVELOPMENT || DEBUG
525 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, wfe_deadline, 0);
526 #endif
527 if (ipending == true) {
528 /* Back to machine_idle() */
529 Idle_load_context();
530 }
531 }
532 }
533
534 if (__improbable(cpu_proximate_timer())) {
535 if (idle_proximate_timer_wfe == 1) {
536 /* Poll issuing WFEs until the expected
537 * timer FIQ arrives.
538 */
539 #if DEVELOPMENT || DEBUG
540 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, ~0ULL, 0);
541 #endif
542 ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr, false);
543 #if DEVELOPMENT || DEBUG
544 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, ~0ULL, 0);
545 #endif
546 assert(ipending == true);
547 }
548 Idle_load_context();
549 }
550
551 lastPop = cpu_data_ptr->rtcPop;
552
553 cpu_data_ptr->cpu_active_thread = current_thread();
554
555 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
556 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
557 }
558
559 if (cpu_data_ptr->idle_timer_notify != NULL) {
560 if (new_idle_timeout_ticks == 0x0ULL) {
561 /* turn off the idle timer */
562 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
563 } else {
564 /* set the new idle timeout */
565 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
566 }
567 timer_resync_deadlines();
568 if (cpu_data_ptr->rtcPop != lastPop) {
569 SetIdlePop();
570 }
571 }
572
573 #if KPC
574 kpc_idle();
575 #endif
576 #if MONOTONIC
577 mt_cpu_idle(cpu_data_ptr);
578 #endif /* MONOTONIC */
579
580 if (wfi) {
581 #if !defined(APPLE_ARM64_ARCH_FAMILY)
582 platform_cache_idle_enter();
583 #endif
584
585 #if DEVELOPMENT || DEBUG
586 // When simulating wfi overhead,
587 // force wfi to clock gating only
588 if (wfi == 2) {
589 arm64_force_wfi_clock_gate();
590 }
591 #endif /* DEVELOPMENT || DEBUG */
592
593 #if defined(APPLETYPHOON)
594 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
595 typhoon_prepare_for_wfi();
596 #endif
597 __builtin_arm_dsb(DSB_SY);
598 #if HAS_RETENTION_STATE
599 arm64_retention_wfi();
600 #else
601 __builtin_arm_wfi();
602 #endif
603
604 #if defined(APPLETYPHOON)
605 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
606 typhoon_return_from_wfi();
607 #endif
608
609 #if DEVELOPMENT || DEBUG
610 // Handle wfi overhead simulation
611 if (wfi == 2) {
612 uint64_t deadline;
613
614 // Calculate wfi delay deadline
615 clock_absolutetime_interval_to_deadline(wfi_delay, &deadline);
616
617 // Flush L1 caches
618 if ((wfi_flags & 1) != 0) {
619 InvalidatePoU_Icache();
620 FlushPoC_Dcache();
621 }
622
623 // Flush TLBs
624 if ((wfi_flags & 2) != 0) {
625 flush_core_tlb();
626 }
627
628 // Wait for the ballance of the wfi delay
629 clock_delay_until(deadline);
630 }
631 #endif /* DEVELOPMENT || DEBUG */
632 #if !defined(APPLE_ARM64_ARCH_FAMILY)
633 platform_cache_idle_exit();
634 #endif
635 }
636
637 ClearIdlePop(TRUE);
638
639 cpu_idle_exit(FALSE);
640 }
641
642 /*
643 * Routine: cpu_idle_exit
644 * Function:
645 */
646 void
cpu_idle_exit(boolean_t from_reset)647 cpu_idle_exit(boolean_t from_reset)
648 {
649 uint64_t new_idle_timeout_ticks = 0x0ULL;
650 cpu_data_t *cpu_data_ptr = getCpuDatap();
651
652 assert(exception_stack_pointer() != 0);
653
654 /* Back from WFI, unlock OSLAR and EDLAR. */
655 if (from_reset) {
656 configure_coresight_registers(cpu_data_ptr);
657 }
658
659 #if KPC
660 kpc_idle_exit();
661 #endif
662
663 #if MONOTONIC
664 mt_cpu_run(cpu_data_ptr);
665 #endif /* MONOTONIC */
666
667 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
668 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
669 }
670
671 if (cpu_data_ptr->idle_timer_notify != NULL) {
672 if (new_idle_timeout_ticks == 0x0ULL) {
673 /* turn off the idle timer */
674 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
675 } else {
676 /* set the new idle timeout */
677 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
678 }
679 timer_resync_deadlines();
680 }
681
682 Idle_load_context();
683 }
684
685 void
cpu_init(void)686 cpu_init(void)
687 {
688 cpu_data_t *cdp = getCpuDatap();
689 arm_cpu_info_t *cpu_info_p;
690
691 assert(exception_stack_pointer() != 0);
692
693 if (cdp->cpu_type != CPU_TYPE_ARM64) {
694 cdp->cpu_type = CPU_TYPE_ARM64;
695
696 timer_call_queue_init(&cdp->rtclock_timer.queue);
697 cdp->rtclock_timer.deadline = EndOfAllTime;
698
699 if (cdp == &BootCpuData) {
700 do_cpuid();
701 do_mvfpid();
702 } else {
703 /*
704 * We initialize non-boot CPUs here; the boot CPU is
705 * dealt with as part of pmap_bootstrap.
706 */
707 pmap_cpu_data_init();
708 }
709
710 do_cacheid();
711
712 /* ARM_SMP: Assuming identical cpu */
713 do_debugid();
714
715 cpu_info_p = cpuid_info();
716
717 /* switch based on CPU's reported architecture */
718 switch (cpu_info_p->arm_info.arm_arch) {
719 case CPU_ARCH_ARMv8:
720 cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8;
721 break;
722 case CPU_ARCH_ARMv8E:
723 cdp->cpu_subtype = CPU_SUBTYPE_ARM64E;
724 break;
725 default:
726 //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL;
727 /* this panic doesn't work this early in startup */
728 panic("Unknown CPU subtype...");
729 break;
730 }
731
732 cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
733 }
734 cdp->cpu_stat.irq_ex_cnt_wake = 0;
735 cdp->cpu_stat.ipi_cnt_wake = 0;
736 #if MONOTONIC
737 cdp->cpu_stat.pmi_cnt_wake = 0;
738 #endif /* MONOTONIC */
739 cdp->cpu_running = TRUE;
740 cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
741 cdp->cpu_sleep_token = 0x0UL;
742 #if KPC
743 kpc_idle_exit();
744 #endif /* KPC */
745 #if MONOTONIC
746 mt_cpu_up(cdp);
747 #endif /* MONOTONIC */
748 }
749
750 void
cpu_stack_alloc(cpu_data_t * cpu_data_ptr)751 cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
752 {
753 vm_offset_t irq_stack = 0;
754 vm_offset_t exc_stack = 0;
755
756 kmem_alloc(kernel_map, &irq_stack,
757 INTSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
758 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
759 VM_KERN_MEMORY_STACK);
760
761 cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
762 cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top;
763
764 kmem_alloc(kernel_map, &exc_stack,
765 EXCEPSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
766 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
767 VM_KERN_MEMORY_STACK);
768
769 cpu_data_ptr->excepstack_top = exc_stack + PAGE_SIZE + EXCEPSTACK_SIZE;
770 cpu_data_ptr->excepstackptr = cpu_data_ptr->excepstack_top;
771 }
772
773 void
cpu_data_free(cpu_data_t * cpu_data_ptr)774 cpu_data_free(cpu_data_t *cpu_data_ptr)
775 {
776 if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) {
777 return;
778 }
779
780 int cpu_number = cpu_data_ptr->cpu_number;
781
782 if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
783 CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
784 CpuDataEntries[cpu_number].cpu_data_paddr = 0;
785 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
786 }
787 kmem_free(kernel_map,
788 cpu_data_ptr->intstack_top - INTSTACK_SIZE - PAGE_SIZE,
789 INTSTACK_SIZE + 2 * PAGE_SIZE);
790 kmem_free(kernel_map,
791 cpu_data_ptr->excepstack_top - EXCEPSTACK_SIZE - PAGE_SIZE,
792 EXCEPSTACK_SIZE + 2 * PAGE_SIZE);
793 }
794
795 void
cpu_data_init(cpu_data_t * cpu_data_ptr)796 cpu_data_init(cpu_data_t *cpu_data_ptr)
797 {
798 uint32_t i;
799
800 cpu_data_ptr->cpu_flags = 0;
801 cpu_data_ptr->cpu_int_state = 0;
802 cpu_data_ptr->cpu_pending_ast = AST_NONE;
803 cpu_data_ptr->cpu_cache_dispatch = NULL;
804 cpu_data_ptr->rtcPop = EndOfAllTime;
805 cpu_data_ptr->rtclock_datap = &RTClockData;
806 cpu_data_ptr->cpu_user_debug = NULL;
807
808
809 cpu_data_ptr->cpu_base_timebase = 0;
810 cpu_data_ptr->cpu_idle_notify = NULL;
811 cpu_data_ptr->cpu_idle_latency = 0x0ULL;
812 cpu_data_ptr->cpu_idle_pop = 0x0ULL;
813 cpu_data_ptr->cpu_reset_type = 0x0UL;
814 cpu_data_ptr->cpu_reset_handler = 0x0UL;
815 cpu_data_ptr->cpu_reset_assist = 0x0UL;
816 cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
817 cpu_data_ptr->cpu_phys_id = 0x0UL;
818 cpu_data_ptr->cpu_l2_access_penalty = 0;
819 cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
820 cpu_data_ptr->cpu_cluster_id = 0;
821 cpu_data_ptr->cpu_l2_id = 0;
822 cpu_data_ptr->cpu_l2_size = 0;
823 cpu_data_ptr->cpu_l3_id = 0;
824 cpu_data_ptr->cpu_l3_size = 0;
825
826 cpu_data_ptr->cpu_signal = SIGPdisabled;
827
828 cpu_data_ptr->cpu_get_fiq_handler = NULL;
829 cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
830 cpu_data_ptr->cpu_tbd_hardware_val = NULL;
831 cpu_data_ptr->cpu_get_decrementer_func = NULL;
832 cpu_data_ptr->cpu_set_decrementer_func = NULL;
833 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
834 cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
835 cpu_data_ptr->cpu_xcall_p0 = NULL;
836 cpu_data_ptr->cpu_xcall_p1 = NULL;
837 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
838 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
839
840 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
841 cpu_data_ptr->coresight_base[i] = 0;
842 }
843
844 #if !XNU_MONITOR
845 pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
846
847 pmap_cpu_data_ptr->cpu_nested_pmap = (struct pmap *) NULL;
848 pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
849 pmap_cpu_data_ptr->pv_free.list = NULL;
850 pmap_cpu_data_ptr->pv_free.count = 0;
851 pmap_cpu_data_ptr->pv_free_spill_marker = NULL;
852
853 bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
854 #endif
855 cpu_data_ptr->halt_status = CPU_NOT_HALTED;
856 #if __ARM_KERNEL_PROTECT__
857 cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table;
858 #endif /* __ARM_KERNEL_PROTECT__ */
859
860 #if defined(HAS_APPLE_PAC)
861 cpu_data_ptr->rop_key = 0;
862 cpu_data_ptr->jop_key = ml_default_jop_pid();
863 #endif
864 }
865
866 kern_return_t
cpu_data_register(cpu_data_t * cpu_data_ptr)867 cpu_data_register(cpu_data_t *cpu_data_ptr)
868 {
869 int cpu = cpu_data_ptr->cpu_number;
870
871 #if KASAN
872 for (int i = 0; i < CPUWINDOWS_MAX; i++) {
873 kasan_notify_address_nopoison(pmap_cpu_windows_copy_addr(cpu, i), PAGE_SIZE);
874 }
875 #endif
876
877 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
878 CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
879 CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
880 return KERN_SUCCESS;
881 }
882
883 #if defined(KERNEL_INTEGRITY_CTRR)
884 /* Hibernation needs to reset this state, so data and text are in the hib segment;
885 * this allows them be accessed and executed early.
886 */
887 LCK_GRP_DECLARE(ctrr_cpu_start_lock_grp, "ctrr_cpu_start_lock");
888 LCK_SPIN_DECLARE(ctrr_cpu_start_lck, &ctrr_cpu_start_lock_grp);
889 enum ctrr_cluster_states ctrr_cluster_locked[MAX_CPU_CLUSTERS] MARK_AS_HIBERNATE_DATA;
890
891 MARK_AS_HIBERNATE_TEXT
892 void
init_ctrr_cluster_states(void)893 init_ctrr_cluster_states(void)
894 {
895 for (int i = 0; i < MAX_CPU_CLUSTERS; i++) {
896 ctrr_cluster_locked[i] = CTRR_UNLOCKED;
897 }
898 }
899 #endif
900
901 kern_return_t
cpu_start(int cpu)902 cpu_start(int cpu)
903 {
904 cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
905
906 kprintf("cpu_start() cpu: %d\n", cpu);
907
908 if (cpu == cpu_number()) {
909 cpu_machine_init();
910 configure_coresight_registers(cpu_data_ptr);
911 } else {
912 thread_t first_thread;
913 processor_t processor;
914
915 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
916
917 #if !XNU_MONITOR
918 cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL;
919 #endif
920
921 processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
922 if (processor->startup_thread != THREAD_NULL) {
923 first_thread = processor->startup_thread;
924 } else {
925 first_thread = processor->idle_thread;
926 }
927 cpu_data_ptr->cpu_active_thread = first_thread;
928 first_thread->machine.CpuDatap = cpu_data_ptr;
929 first_thread->machine.pcpu_data_base =
930 (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);
931
932 configure_coresight_registers(cpu_data_ptr);
933
934 flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
935 flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
936 #if defined(KERNEL_INTEGRITY_CTRR)
937
938 /* First CPU being started within a cluster goes ahead to lock CTRR for cluster;
939 * other CPUs block until cluster is locked. */
940 lck_spin_lock(&ctrr_cpu_start_lck);
941 switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) {
942 case CTRR_UNLOCKED:
943 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING;
944 lck_spin_unlock(&ctrr_cpu_start_lck);
945 break;
946 case CTRR_LOCKING:
947 assert_wait(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id], THREAD_UNINT);
948 lck_spin_unlock(&ctrr_cpu_start_lck);
949 thread_block(THREAD_CONTINUE_NULL);
950 assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING);
951 break;
952 default: // CTRR_LOCKED
953 lck_spin_unlock(&ctrr_cpu_start_lck);
954 break;
955 }
956 #endif
957 (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
958 }
959
960 return KERN_SUCCESS;
961 }
962
963
964 void
cpu_timebase_init(boolean_t from_boot)965 cpu_timebase_init(boolean_t from_boot)
966 {
967 cpu_data_t *cdp = getCpuDatap();
968
969 if (cdp->cpu_get_fiq_handler == NULL) {
970 cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
971 cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
972 cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
973 cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
974 cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
975 }
976
977 if (!from_boot && (cdp == &BootCpuData)) {
978 /*
979 * When we wake from sleep, we have no guarantee about the state
980 * of the hardware timebase. It may have kept ticking across sleep, or
981 * it may have reset.
982 *
983 * To deal with this, we calculate an offset to the clock that will
984 * produce a timebase value wake_abstime at the point the boot
985 * CPU calls cpu_timebase_init on wake.
986 *
987 * This ensures that mach_absolute_time() stops ticking across sleep.
988 */
989 rtclock_base_abstime = wake_abstime - ml_get_hwclock();
990 } else if (from_boot) {
991 /* On initial boot, initialize time_since_reset to CNTPCT_EL0. */
992 ml_set_reset_time(ml_get_hwclock());
993 }
994
995 cdp->cpu_decrementer = 0x7FFFFFFFUL;
996 cdp->cpu_timebase = 0x0UL;
997 cdp->cpu_base_timebase = rtclock_base_abstime;
998 }
999
1000 int
cpu_cluster_id(void)1001 cpu_cluster_id(void)
1002 {
1003 return getCpuDatap()->cpu_cluster_id;
1004 }
1005
1006 __attribute__((noreturn))
1007 void
ml_arm_sleep(void)1008 ml_arm_sleep(void)
1009 {
1010 cpu_data_t *cpu_data_ptr = getCpuDatap();
1011
1012 if (cpu_data_ptr == &BootCpuData) {
1013 cpu_data_t *target_cdp;
1014 int cpu;
1015 int max_cpu;
1016
1017 max_cpu = ml_get_max_cpu_number();
1018 for (cpu = 0; cpu <= max_cpu; cpu++) {
1019 target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1020
1021 if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) {
1022 continue;
1023 }
1024
1025 while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
1026 ;
1027 }
1028 }
1029
1030 /*
1031 * Now that the other cores have entered the sleep path, set
1032 * the abstime value we'll use when we resume.
1033 */
1034 wake_abstime = ml_get_timebase();
1035 ml_set_reset_time(UINT64_MAX);
1036 } else {
1037 CleanPoU_Dcache();
1038 }
1039
1040 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
1041
1042 if (cpu_data_ptr == &BootCpuData) {
1043 #if WITH_CLASSIC_S2R
1044 // Classic suspend to RAM writes the suspend signature into the
1045 // sleep token buffer so that iBoot knows that it's on the warm
1046 // boot (wake) path (as opposed to the cold boot path). Newer SoC
1047 // do not go through SecureROM/iBoot on the warm boot path. The
1048 // reconfig engine script brings the CPU out of reset at the kernel's
1049 // reset vector which points to the warm boot initialization code.
1050 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1051 platform_cache_shutdown();
1052 bcopy((const void *)suspend_signature, (void *)sleepTokenBuffer, sizeof(SleepToken));
1053 } else {
1054 panic("No sleep token buffer");
1055 }
1056 #endif
1057
1058 #if __ARM_GLOBAL_SLEEP_BIT__
1059 /* Allow other CPUs to go to sleep. */
1060 arm64_stall_sleep = FALSE;
1061 __builtin_arm_dmb(DMB_ISH);
1062 #endif
1063
1064 /* Architectural debug state: <rdar://problem/12390433>:
1065 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1066 * tell debugger to not prevent power gating .
1067 */
1068 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1069 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1070 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1071 }
1072
1073 #if HIBERNATION
1074 uint32_t mode = hibernate_write_image();
1075 if (mode == kIOHibernatePostWriteHalt) {
1076 HIBLOG("powering off after writing hibernation image\n");
1077 int halt_result = -1;
1078 if (PE_halt_restart) {
1079 /**
1080 * Drain serial FIFOs now as the normal call further down won't
1081 * be hit when the CPU halts here for hibernation. Here, it'll
1082 * make sure the preceding HIBLOG is flushed as well.
1083 */
1084 serial_go_to_sleep();
1085 halt_result = (*PE_halt_restart)(kPEHaltCPU);
1086 }
1087 panic("can't shutdown: PE_halt_restart returned %d", halt_result);
1088 }
1089 #endif /* HIBERNATION */
1090
1091 serial_go_to_sleep();
1092
1093 #if MONOTONIC
1094 mt_sleep();
1095 #endif /* MONOTONIC */
1096 /* ARM64-specific preparation */
1097 arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBase));
1098 } else {
1099 #if __ARM_GLOBAL_SLEEP_BIT__
1100 /*
1101 * With the exception of the CPU revisions listed above, our ARM64 CPUs have a
1102 * global register to manage entering deep sleep, as opposed to a per-CPU
1103 * register. We cannot update this register until all CPUs are ready to enter
1104 * deep sleep, because if a CPU executes WFI outside of the deep sleep context
1105 * (by idling), it will hang (due to the side effects of enabling deep sleep),
1106 * which can hang the sleep process or cause memory corruption on wake.
1107 *
1108 * To avoid these issues, we'll stall on this global value, which CPU0 will
1109 * manage.
1110 */
1111 while (arm64_stall_sleep) {
1112 __builtin_arm_wfe();
1113 }
1114 #endif
1115 CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
1116
1117 /* Architectural debug state: <rdar://problem/12390433>:
1118 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1119 * tell debugger to not prevent power gating .
1120 */
1121 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1122 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1123 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1124 }
1125
1126 /* ARM64-specific preparation */
1127 arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBase));
1128 }
1129 }
1130
1131 void
cpu_machine_idle_init(boolean_t from_boot)1132 cpu_machine_idle_init(boolean_t from_boot)
1133 {
1134 static vm_address_t resume_idle_cpu_paddr = (vm_address_t)NULL;
1135 cpu_data_t *cpu_data_ptr = getCpuDatap();
1136
1137 if (from_boot) {
1138 uint32_t production = 1;
1139 DTEntry entry;
1140
1141 unsigned long jtag = 0;
1142
1143 if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
1144 if (jtag != 0) {
1145 idle_enable = FALSE;
1146 } else {
1147 idle_enable = TRUE;
1148 }
1149 } else {
1150 idle_enable = TRUE;
1151 }
1152
1153 #if DEVELOPMENT || DEBUG
1154 uint32_t wfe_mode = 0;
1155 if (PE_parse_boot_argn("wfe_mode", &wfe_mode, sizeof(wfe_mode))) {
1156 idle_proximate_timer_wfe = ((wfe_mode & 1) == 1);
1157 idle_proximate_io_wfe_masked = ((wfe_mode & 2) == 2);
1158 extern uint32_t idle_proximate_io_wfe_unmasked;
1159 idle_proximate_io_wfe_unmasked = ((wfe_mode & 4) == 4);
1160 }
1161 #endif
1162
1163 // bits 7..0 give the wfi type
1164 switch (wfi & 0xff) {
1165 case 0:
1166 // disable wfi
1167 wfi = 0;
1168 break;
1169
1170 #if DEVELOPMENT || DEBUG
1171 case 2:
1172 // wfi overhead simulation
1173 // 31..16 - wfi delay is us
1174 // 15..8 - flags
1175 // 7..0 - 2
1176 wfi = 2;
1177 wfi_flags = (wfi >> 8) & 0xFF;
1178 nanoseconds_to_absolutetime(((wfi >> 16) & 0xFFFF) * NSEC_PER_MSEC, &wfi_delay);
1179 break;
1180 #endif /* DEVELOPMENT || DEBUG */
1181
1182 case 1:
1183 default:
1184 // do nothing
1185 break;
1186 }
1187
1188 ResetHandlerData.assist_reset_handler = 0;
1189 ResetHandlerData.cpu_data_entries = ml_static_vtop((vm_offset_t)CpuDataEntries);
1190
1191 #ifdef MONITOR
1192 monitor_call(MONITOR_SET_ENTRY, (uintptr_t)ml_static_vtop((vm_offset_t)&LowResetVectorBase), 0, 0);
1193 #elif !defined(NO_MONITOR)
1194 #error MONITOR undefined, WFI power gating may not operate correctly
1195 #endif /* MONITOR */
1196
1197 // Determine if we are on production or debug chip
1198 if (kSuccess == SecureDTLookupEntry(NULL, "/chosen", &entry)) {
1199 unsigned int size;
1200 void const *prop;
1201
1202 if (kSuccess == SecureDTGetProperty(entry, "effective-production-status-ap", &prop, &size)) {
1203 if (size == 4) {
1204 bcopy(prop, &production, size);
1205 }
1206 }
1207 }
1208 if (!production) {
1209 #if defined(APPLE_ARM64_ARCH_FAMILY)
1210 // Enable coresight debug registers on debug-fused chips
1211 coresight_debug_enabled = TRUE;
1212 #endif
1213 }
1214
1215 start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);
1216 resume_idle_cpu_paddr = ml_static_vtop((vm_offset_t)&resume_idle_cpu);
1217 }
1218
1219 #if WITH_CLASSIC_S2R
1220 if (cpu_data_ptr == &BootCpuData) {
1221 static addr64_t SleepToken_low_paddr = (addr64_t)NULL;
1222 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1223 SleepToken_low_paddr = ml_vtophys(sleepTokenBuffer);
1224 } else {
1225 panic("No sleep token buffer");
1226 }
1227
1228 bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)running_signature),
1229 SleepToken_low_paddr, sizeof(SleepToken));
1230 flush_dcache((vm_offset_t)SleepToken, sizeof(SleepToken), TRUE);
1231 }
1232 ;
1233 #endif
1234
1235 cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
1236 clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
1237 }
1238
1239 _Atomic uint32_t cpu_idle_count = 0;
1240
1241 void
machine_track_platform_idle(boolean_t entry)1242 machine_track_platform_idle(boolean_t entry)
1243 {
1244 if (entry) {
1245 os_atomic_inc(&cpu_idle_count, relaxed);
1246 } else {
1247 os_atomic_dec(&cpu_idle_count, relaxed);
1248 }
1249 }
1250
1251 #if WITH_CLASSIC_S2R
1252 void
sleep_token_buffer_init(void)1253 sleep_token_buffer_init(void)
1254 {
1255 cpu_data_t *cpu_data_ptr = getCpuDatap();
1256 DTEntry entry;
1257 size_t size;
1258 void const * const *prop;
1259
1260 if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) {
1261 /* Find the stpage node in the device tree */
1262 if (kSuccess != SecureDTLookupEntry(0, "stram", &entry)) {
1263 return;
1264 }
1265
1266 if (kSuccess != SecureDTGetProperty(entry, "reg", (const void **)&prop, (unsigned int *)&size)) {
1267 return;
1268 }
1269
1270 /* Map the page into the kernel space */
1271 sleepTokenBuffer = ml_io_map(((vm_offset_t const *)prop)[0], ((vm_size_t const *)prop)[1]);
1272 }
1273 }
1274 #endif
1275