1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm64/cpu.c
30 *
31 * cpu specific routines
32 */
33
34 #include <pexpert/arm64/board_config.h>
35 #include <kern/kalloc.h>
36 #include <kern/machine.h>
37 #include <kern/cpu_number.h>
38 #include <kern/percpu.h>
39 #include <kern/thread.h>
40 #include <kern/timer_queue.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <arm64/proc_reg.h>
50 #include <mach/processor_info.h>
51 #include <vm/pmap.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58
59 #include <machine/atomic.h>
60
61 #include <san/kasan.h>
62
63 #if KPC
64 #include <kern/kpc.h>
65 #endif
66
67 #if MONOTONIC
68 #include <kern/monotonic.h>
69 #endif /* MONOTONIC */
70
71 #if KPERF
72 #include <kperf/kptimer.h>
73 #endif /* KPERF */
74
75 #if HIBERNATION
76 #include <IOKit/IOPlatformExpert.h>
77 #include <IOKit/IOHibernatePrivate.h>
78 #endif /* HIBERNATION */
79
80
81 #include <libkern/section_keywords.h>
82
83 extern boolean_t idle_enable;
84 extern uint64_t wake_abstime;
85
86 #if WITH_CLASSIC_S2R
87 void sleep_token_buffer_init(void);
88 #endif
89
90
91 extern uintptr_t resume_idle_cpu;
92 extern uintptr_t start_cpu;
93
94 #if __ARM_KERNEL_PROTECT__
95 extern void exc_vectors_table;
96 #endif /* __ARM_KERNEL_PROTECT__ */
97
98 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep);
99 extern void arm64_force_wfi_clock_gate(void);
100 #if defined(APPLETYPHOON)
101 // <rdar://problem/15827409>
102 extern void typhoon_prepare_for_wfi(void);
103 extern void typhoon_return_from_wfi(void);
104 #endif
105
106 #if HAS_RETENTION_STATE
107 extern void arm64_retention_wfi(void);
108 #endif
109
110 vm_address_t start_cpu_paddr;
111
112 sysreg_restore_t sysreg_restore __attribute__((section("__DATA, __const"))) = {
113 .tcr_el1 = TCR_EL1_BOOT,
114 };
115
116 // wfi - wfi mode
117 // 0 : disabled
118 // 1 : normal
119 // 2 : overhead simulation (delay & flags)
120 TUNABLE(unsigned int, wfi, "wfi", 1);
121 #if DEVELOPMENT || DEBUG
122
123 // wfi_flags
124 // 1 << 0 : flush L1s
125 // 1 << 1 : flush TLBs
126 static int wfi_flags = 0;
127
128 // wfi_delay - delay ticks after wfi exit
129 static uint64_t wfi_delay = 0;
130
131 #endif /* DEVELOPMENT || DEBUG */
132
133 #define CPUPM_IDLE_WFE 0x5310300
134 #define CPUPM_IDLE_TIMER_WFE 0x5310304
135
136 /* When recommended, issue WFE with [FI]IRQ unmasked in the idle
137 * loop. The default.
138 */
139 uint32_t idle_proximate_io_wfe_unmasked = 1;
140 #if DEVELOPMENT || DEBUG
141 uint32_t idle_proximate_timer_wfe = 1;
142 uint32_t idle_proximate_io_wfe_masked = 0;
143 #else
144 /* Issue WFE in lieu of WFI when awaiting a proximate timer. */
145 static uint32_t idle_proximate_timer_wfe = 1;
146 /* When recommended, issue WFE with [FI]IRQ masked in the idle loop.
147 * Non-default, retained for experimentation.
148 */
149 static uint32_t idle_proximate_io_wfe_masked = 0;
150 #endif
151
152 #if __ARM_GLOBAL_SLEEP_BIT__
153 volatile boolean_t arm64_stall_sleep = TRUE;
154 #endif
155
156 #if WITH_CLASSIC_S2R
157 /*
158 * These must be aligned to avoid issues with calling bcopy_phys on them before
159 * we are done with pmap initialization.
160 */
161 static const uint8_t __attribute__ ((aligned(8))) suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
162 static const uint8_t __attribute__ ((aligned(8))) running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};
163 #endif
164
165 #if WITH_CLASSIC_S2R
166 static vm_offset_t sleepTokenBuffer = (vm_offset_t)NULL;
167 #endif
168 static boolean_t coresight_debug_enabled = FALSE;
169
170 #if defined(CONFIG_XNUPOST)
171 void arm64_ipi_test_callback(void *);
172 void arm64_immediate_ipi_test_callback(void *);
173
174 void
arm64_ipi_test_callback(void * parm)175 arm64_ipi_test_callback(void *parm)
176 {
177 volatile uint64_t *ipi_test_data = parm;
178 cpu_data_t *cpu_data;
179
180 cpu_data = getCpuDatap();
181
182 *ipi_test_data = cpu_data->cpu_number;
183 }
184
185 void
arm64_immediate_ipi_test_callback(void * parm)186 arm64_immediate_ipi_test_callback(void *parm)
187 {
188 volatile uint64_t *ipi_test_data = parm;
189 cpu_data_t *cpu_data;
190
191 cpu_data = getCpuDatap();
192
193 *ipi_test_data = cpu_data->cpu_number + MAX_CPUS;
194 }
195
196 uint64_t arm64_ipi_test_data[MAX_CPUS * 2];
197
198 MACHINE_TIMEOUT(arm64_ipi_test_timeout, "arm64-ipi-test", 100, MACHINE_TIMEOUT_UNIT_MSEC, NULL);
199
200 void
arm64_ipi_test()201 arm64_ipi_test()
202 {
203 volatile uint64_t *ipi_test_data, *immediate_ipi_test_data;
204 uint64_t timeout_ms = os_atomic_load(&arm64_ipi_test_timeout, relaxed);
205 uint64_t then, now, delta;
206 int current_cpu_number = getCpuDatap()->cpu_number;
207
208 /*
209 * probably the only way to have this on most systems is with the
210 * cpus=1 boot-arg, but nonetheless, if we only have 1 CPU active,
211 * IPI is not available
212 */
213 if (real_ncpus == 1) {
214 return;
215 }
216
217 const unsigned int max_cpu_id = ml_get_max_cpu_number();
218 for (unsigned int i = 0; i <= max_cpu_id; ++i) {
219 ipi_test_data = &arm64_ipi_test_data[i];
220 immediate_ipi_test_data = &arm64_ipi_test_data[i + MAX_CPUS];
221 *ipi_test_data = ~i;
222 kern_return_t error = cpu_xcall((int)i, (void *)arm64_ipi_test_callback, (void *)(uintptr_t)ipi_test_data);
223 if (error != KERN_SUCCESS) {
224 panic("CPU %d was unable to IPI CPU %u: error %d", current_cpu_number, i, error);
225 }
226
227 while ((error = cpu_immediate_xcall((int)i, (void *)arm64_immediate_ipi_test_callback,
228 (void *)(uintptr_t)immediate_ipi_test_data)) == KERN_ALREADY_WAITING) {
229 now = mach_absolute_time();
230 absolutetime_to_nanoseconds(now - then, &delta);
231 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
232 panic("CPU %d was unable to immediate-IPI CPU %u within %lldms", current_cpu_number, i, timeout_ms);
233 }
234 }
235
236 if (error != KERN_SUCCESS) {
237 panic("CPU %d was unable to immediate-IPI CPU %u: error %d", current_cpu_number, i, error);
238 }
239
240 then = mach_absolute_time();
241
242 while ((*ipi_test_data != i) || (*immediate_ipi_test_data != (i + MAX_CPUS))) {
243 now = mach_absolute_time();
244 absolutetime_to_nanoseconds(now - then, &delta);
245 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
246 panic("CPU %d tried to IPI CPU %d but didn't get correct responses within %lldms, responses: %llx, %llx",
247 current_cpu_number, i, timeout_ms, *ipi_test_data, *immediate_ipi_test_data);
248 }
249 }
250 }
251 }
252 #endif /* defined(CONFIG_XNUPOST) */
253
254 static void
configure_coresight_registers(cpu_data_t * cdp)255 configure_coresight_registers(cpu_data_t *cdp)
256 {
257 int i;
258
259 assert(cdp);
260 vm_offset_t coresight_regs = ml_get_topology_info()->cpus[cdp->cpu_number].coresight_regs;
261
262 /*
263 * ARMv8 coresight registers are optional. If the device tree did not
264 * provide either cpu_regmap_paddr (from the legacy "reg-private" EDT property)
265 * or coresight_regs (from the new "coresight-reg" property), assume that
266 * coresight registers are not supported.
267 */
268 if (cdp->cpu_regmap_paddr || coresight_regs) {
269 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
270 if (i == CORESIGHT_CTI) {
271 continue;
272 }
273 /* Skip debug-only registers on production chips */
274 if (((i == CORESIGHT_ED) || (i == CORESIGHT_UTT)) && !coresight_debug_enabled) {
275 continue;
276 }
277
278 if (!cdp->coresight_base[i]) {
279 if (coresight_regs) {
280 cdp->coresight_base[i] = coresight_regs + CORESIGHT_OFFSET(i);
281 } else {
282 uint64_t addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i);
283 cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE);
284 }
285
286 /*
287 * At this point, failing to io map the
288 * registers is considered as an error.
289 */
290 if (!cdp->coresight_base[i]) {
291 panic("unable to ml_io_map coresight regions");
292 }
293 }
294 /* Unlock EDLAR, CTILAR, PMLAR */
295 if (i != CORESIGHT_UTT) {
296 *(volatile uint32_t *)(cdp->coresight_base[i] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
297 }
298 }
299 }
300 }
301
302
303 /*
304 * Routine: cpu_bootstrap
305 * Function:
306 */
307 void
cpu_bootstrap(void)308 cpu_bootstrap(void)
309 {
310 }
311
312 /*
313 * Routine: cpu_sleep
314 * Function:
315 */
316 void
cpu_sleep(void)317 cpu_sleep(void)
318 {
319 cpu_data_t *cpu_data_ptr = getCpuDatap();
320
321 cpu_data_ptr->cpu_active_thread = current_thread();
322 cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr;
323 cpu_data_ptr->cpu_flags |= SleepState;
324 cpu_data_ptr->cpu_user_debug = NULL;
325 #if KPC
326 kpc_idle();
327 #endif /* KPC */
328 #if MONOTONIC
329 mt_cpu_down(cpu_data_ptr);
330 #endif /* MONOTONIC */
331 #if KPERF
332 kptimer_stop_curcpu();
333 #endif /* KPERF */
334
335 CleanPoC_Dcache();
336
337 #if USE_APPLEARMSMP
338 if (ml_is_quiescing()) {
339 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
340 } else {
341 bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id);
342 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
343 // hang CPU on spurious wakeup
344 cpu_data_ptr->cpu_reset_handler = (uintptr_t)0;
345 __builtin_arm_dsb(DSB_ISH);
346 CleanPoU_Dcache();
347 arm64_prepare_for_sleep(deep_sleep);
348 }
349 #else
350 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
351 #endif
352 /*NOTREACHED*/
353 }
354
355 /*
356 * Routine: cpu_interrupt_is_pending
357 * Function: Returns a bool signifying a non-zero ISR_EL1,
358 * indicating a pending IRQ, FIQ or external abort.
359 */
360
361 bool
cpu_interrupt_is_pending(void)362 cpu_interrupt_is_pending(void)
363 {
364 uint64_t isr_value;
365 isr_value = __builtin_arm_rsr64("ISR_EL1");
366 return isr_value != 0;
367 }
368
369 static bool
cpu_proximate_timer(void)370 cpu_proximate_timer(void)
371 {
372 return !SetIdlePop();
373 }
374
375 #ifdef ARM64_BOARD_CONFIG_T6000
376 int wfe_allowed = 0;
377 #else
378 int wfe_allowed = 1;
379 #endif /* ARM64_BOARD_CONFIG_T6000 */
380
381 #if DEVELOPMENT || DEBUG
382 #define WFE_STAT(x) \
383 do { \
384 (x); \
385 } while(0)
386 #else
387 #define WFE_STAT(x) do {} while(0)
388 #endif /* DEVELOPMENT || DEBUG */
389
390 bool
wfe_to_deadline_or_interrupt(uint32_t cid,uint64_t wfe_deadline,cpu_data_t * cdp,bool unmask)391 wfe_to_deadline_or_interrupt(uint32_t cid, uint64_t wfe_deadline, cpu_data_t *cdp, bool unmask)
392 {
393 bool ipending = false;
394 uint64_t irqc = 0, nirqc = 0;
395
396 /* The ARMv8 architecture permits a processor dwelling in WFE
397 * with F/IRQ masked to ignore a pending interrupt, i.e.
398 * not classify it as an 'event'. This is potentially
399 * problematic with AICv2's IRQ distribution model, as
400 * a transient interrupt masked interval can cause an SIQ
401 * query rejection, possibly routing the interrupt to
402 * another core/cluster in a powergated state.
403 * Hence, optionally unmask IRQs+FIQs across WFE.
404 */
405 if (unmask) {
406 /* Latch SW IRQ+FIQ counter prior to unmasking
407 * interrupts.
408 */
409 irqc = nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
410 /* Unmask IRQ+FIQ. Mirrors mask used by machine_idle()
411 * with ASYNCF omission. Consider that this could
412 * delay recognition of an async abort, including
413 * those triggered by ISRs
414 */
415 __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF));
416 }
417
418 while ((ipending = (cpu_interrupt_is_pending())) == false) {
419 if (unmask) {
420 /* If WFE was issued with IRQs unmasked, an
421 * interrupt may have been processed.
422 * Consult the SW IRQ counter to determine
423 * whether the 'idle loop' must be
424 * re-evaluated.
425 */
426 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
427 if (nirqc != irqc) {
428 break;
429 }
430 }
431
432 if (__probable(wfe_allowed)) {
433 /*
434 * If IRQs are unmasked, there's a small window
435 * where an 'extra' WFE may be issued after
436 * the consultation of the SW interrupt counter
437 * and new interrupt arrival. Hence this WFE
438 * relies on the [FI]RQ interrupt handler
439 * epilogue issuing a 'SEVL', to post an
440 * event which causes the next WFE on the same
441 * PE to retire immediately.
442 */
443
444 __builtin_arm_wfe();
445 }
446
447 WFE_STAT(cdp->wfe_count++);
448 if (wfe_deadline != ~0ULL) {
449 WFE_STAT(cdp->wfe_deadline_checks++);
450 /* Check if the WFE recommendation has expired.
451 * We do not recompute the deadline here.
452 */
453 if ((ml_cluster_wfe_timeout(cid) == 0) ||
454 mach_absolute_time() >= wfe_deadline) {
455 WFE_STAT(cdp->wfe_terminations++);
456 break;
457 }
458 }
459 }
460
461 if (unmask) {
462 /* Re-mask IRQ+FIQ
463 * Mirrors mask used by machine_idle(), with ASYNCF
464 * omission
465 */
466 __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));
467 /* Refetch SW interrupt counter with IRQs masked
468 * It is important that this routine accurately flags
469 * any observed interrupts via its return value,
470 * inaccuracy may lead to an erroneous WFI fallback.
471 */
472 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
473 }
474
475 return ipending || (nirqc != irqc);
476 }
477
478 /*
479 * Routine: cpu_idle
480 * Function:
481 */
482 void __attribute__((noreturn))
cpu_idle(void)483 cpu_idle(void)
484 {
485 cpu_data_t *cpu_data_ptr = getCpuDatap();
486 uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop;
487 bool idle_disallowed = false;
488
489 if (__improbable((!idle_enable))) {
490 idle_disallowed = true;
491 } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) {
492 idle_disallowed = true;
493 }
494
495 if (__improbable(idle_disallowed)) {
496 Idle_load_context();
497 }
498
499 bool ipending = false;
500 uint32_t cid = cpu_data_ptr->cpu_cluster_id;
501
502 if (idle_proximate_io_wfe_masked == 1) {
503 uint64_t wfe_deadline = 0;
504 /* Check for an active perf. controller generated
505 * WFE recommendation for this cluster.
506 */
507 uint64_t wfe_ttd = 0;
508 if ((wfe_ttd = ml_cluster_wfe_timeout(cid)) != 0) {
509 wfe_deadline = mach_absolute_time() + wfe_ttd;
510 }
511
512 if (wfe_deadline != 0) {
513 /* Poll issuing event-bounded WFEs until an interrupt
514 * arrives or the WFE recommendation expires
515 */
516 #if DEVELOPMENT || DEBUG
517 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, wfe_ttd, cid);
518 #endif
519 ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr, false);
520 #if DEVELOPMENT || DEBUG
521 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, wfe_deadline, 0);
522 #endif
523 if (ipending == true) {
524 /* Back to machine_idle() */
525 Idle_load_context();
526 }
527 }
528 }
529
530 if (__improbable(cpu_proximate_timer())) {
531 if (idle_proximate_timer_wfe == 1) {
532 /* Poll issuing WFEs until the expected
533 * timer FIQ arrives.
534 */
535 #if DEVELOPMENT || DEBUG
536 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, ~0ULL, 0);
537 #endif
538 ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr, false);
539 #if DEVELOPMENT || DEBUG
540 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, ~0ULL, 0);
541 #endif
542 assert(ipending == true);
543 }
544 Idle_load_context();
545 }
546
547 lastPop = cpu_data_ptr->rtcPop;
548
549 cpu_data_ptr->cpu_active_thread = current_thread();
550
551 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
552 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
553 }
554
555 if (cpu_data_ptr->idle_timer_notify != NULL) {
556 if (new_idle_timeout_ticks == 0x0ULL) {
557 /* turn off the idle timer */
558 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
559 } else {
560 /* set the new idle timeout */
561 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
562 }
563 timer_resync_deadlines();
564 if (cpu_data_ptr->rtcPop != lastPop) {
565 SetIdlePop();
566 }
567 }
568
569 #if KPC
570 kpc_idle();
571 #endif
572 #if MONOTONIC
573 mt_cpu_idle(cpu_data_ptr);
574 #endif /* MONOTONIC */
575
576 if (wfi) {
577 #if !defined(APPLE_ARM64_ARCH_FAMILY)
578 platform_cache_idle_enter();
579 #endif
580
581 #if DEVELOPMENT || DEBUG
582 // When simulating wfi overhead,
583 // force wfi to clock gating only
584 if (wfi == 2) {
585 arm64_force_wfi_clock_gate();
586 }
587 #endif /* DEVELOPMENT || DEBUG */
588
589 #if defined(APPLETYPHOON)
590 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
591 typhoon_prepare_for_wfi();
592 #endif
593 __builtin_arm_dsb(DSB_SY);
594 #if HAS_RETENTION_STATE
595 arm64_retention_wfi();
596 #else
597 __builtin_arm_wfi();
598 #endif
599
600 #if defined(APPLETYPHOON)
601 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
602 typhoon_return_from_wfi();
603 #endif
604
605 #if DEVELOPMENT || DEBUG
606 // Handle wfi overhead simulation
607 if (wfi == 2) {
608 uint64_t deadline;
609
610 // Calculate wfi delay deadline
611 clock_absolutetime_interval_to_deadline(wfi_delay, &deadline);
612
613 // Flush L1 caches
614 if ((wfi_flags & 1) != 0) {
615 InvalidatePoU_Icache();
616 FlushPoC_Dcache();
617 }
618
619 // Flush TLBs
620 if ((wfi_flags & 2) != 0) {
621 flush_core_tlb();
622 }
623
624 // Wait for the ballance of the wfi delay
625 clock_delay_until(deadline);
626 }
627 #endif /* DEVELOPMENT || DEBUG */
628 #if !defined(APPLE_ARM64_ARCH_FAMILY)
629 platform_cache_idle_exit();
630 #endif
631 }
632
633 ClearIdlePop(TRUE);
634
635 cpu_idle_exit(FALSE);
636 }
637
638 /*
639 * Routine: cpu_idle_exit
640 * Function:
641 */
642 void
cpu_idle_exit(boolean_t from_reset)643 cpu_idle_exit(boolean_t from_reset)
644 {
645 uint64_t new_idle_timeout_ticks = 0x0ULL;
646 cpu_data_t *cpu_data_ptr = getCpuDatap();
647
648 assert(exception_stack_pointer() != 0);
649
650 /* Back from WFI, unlock OSLAR and EDLAR. */
651 if (from_reset) {
652 configure_coresight_registers(cpu_data_ptr);
653 }
654
655 #if KPC
656 kpc_idle_exit();
657 #endif
658
659 #if MONOTONIC
660 mt_cpu_run(cpu_data_ptr);
661 #endif /* MONOTONIC */
662
663 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
664 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
665 }
666
667 if (cpu_data_ptr->idle_timer_notify != NULL) {
668 if (new_idle_timeout_ticks == 0x0ULL) {
669 /* turn off the idle timer */
670 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
671 } else {
672 /* set the new idle timeout */
673 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
674 }
675 timer_resync_deadlines();
676 }
677
678 Idle_load_context();
679 }
680
681 void
cpu_init(void)682 cpu_init(void)
683 {
684 cpu_data_t *cdp = getCpuDatap();
685 arm_cpu_info_t *cpu_info_p;
686
687 assert(exception_stack_pointer() != 0);
688
689 if (cdp->cpu_type != CPU_TYPE_ARM64) {
690 cdp->cpu_type = CPU_TYPE_ARM64;
691
692 timer_call_queue_init(&cdp->rtclock_timer.queue);
693 cdp->rtclock_timer.deadline = EndOfAllTime;
694
695 if (cdp == &BootCpuData) {
696 do_cpuid();
697 do_mvfpid();
698 } else {
699 /*
700 * We initialize non-boot CPUs here; the boot CPU is
701 * dealt with as part of pmap_bootstrap.
702 */
703 pmap_cpu_data_init();
704 }
705
706 do_cacheid();
707
708 /* ARM_SMP: Assuming identical cpu */
709 do_debugid();
710
711 cpu_info_p = cpuid_info();
712
713 /* switch based on CPU's reported architecture */
714 switch (cpu_info_p->arm_info.arm_arch) {
715 case CPU_ARCH_ARMv8:
716 cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8;
717 break;
718 case CPU_ARCH_ARMv8E:
719 cdp->cpu_subtype = CPU_SUBTYPE_ARM64E;
720 break;
721 default:
722 //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL;
723 /* this panic doesn't work this early in startup */
724 panic("Unknown CPU subtype...");
725 break;
726 }
727
728 cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
729 }
730 cdp->cpu_stat.irq_ex_cnt_wake = 0;
731 cdp->cpu_stat.ipi_cnt_wake = 0;
732 #if MONOTONIC
733 cdp->cpu_stat.pmi_cnt_wake = 0;
734 #endif /* MONOTONIC */
735 cdp->cpu_running = TRUE;
736 cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
737 cdp->cpu_sleep_token = 0x0UL;
738 #if KPC
739 kpc_idle_exit();
740 #endif /* KPC */
741 #if MONOTONIC
742 mt_cpu_up(cdp);
743 #endif /* MONOTONIC */
744 }
745
746 void
cpu_stack_alloc(cpu_data_t * cpu_data_ptr)747 cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
748 {
749 vm_offset_t irq_stack = 0;
750 vm_offset_t exc_stack = 0;
751
752 kern_return_t kr = kernel_memory_allocate(kernel_map, &irq_stack,
753 INTSTACK_SIZE + (2 * PAGE_SIZE),
754 PAGE_MASK,
755 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
756 VM_KERN_MEMORY_STACK);
757 if (kr != KERN_SUCCESS) {
758 panic("Unable to allocate cpu interrupt stack");
759 }
760
761 cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
762 cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top;
763
764 kr = kernel_memory_allocate(kernel_map, &exc_stack,
765 EXCEPSTACK_SIZE + (2 * PAGE_SIZE),
766 PAGE_MASK,
767 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
768 VM_KERN_MEMORY_STACK);
769 if (kr != KERN_SUCCESS) {
770 panic("Unable to allocate cpu exception stack");
771 }
772
773 cpu_data_ptr->excepstack_top = exc_stack + PAGE_SIZE + EXCEPSTACK_SIZE;
774 cpu_data_ptr->excepstackptr = cpu_data_ptr->excepstack_top;
775 }
776
777 void
cpu_data_free(cpu_data_t * cpu_data_ptr)778 cpu_data_free(cpu_data_t *cpu_data_ptr)
779 {
780 if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) {
781 return;
782 }
783
784 int cpu_number = cpu_data_ptr->cpu_number;
785
786 if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) {
787 CpuDataEntries[cpu_number].cpu_data_vaddr = NULL;
788 CpuDataEntries[cpu_number].cpu_data_paddr = 0;
789 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu array are visible
790 }
791 (kfree)((void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE);
792 (kfree)((void *)(cpu_data_ptr->excepstack_top - EXCEPSTACK_SIZE), EXCEPSTACK_SIZE);
793 }
794
795 void
cpu_data_init(cpu_data_t * cpu_data_ptr)796 cpu_data_init(cpu_data_t *cpu_data_ptr)
797 {
798 uint32_t i;
799
800 cpu_data_ptr->cpu_flags = 0;
801 cpu_data_ptr->cpu_int_state = 0;
802 cpu_data_ptr->cpu_pending_ast = AST_NONE;
803 cpu_data_ptr->cpu_cache_dispatch = NULL;
804 cpu_data_ptr->rtcPop = EndOfAllTime;
805 cpu_data_ptr->rtclock_datap = &RTClockData;
806 cpu_data_ptr->cpu_user_debug = NULL;
807
808
809 cpu_data_ptr->cpu_base_timebase = 0;
810 cpu_data_ptr->cpu_idle_notify = NULL;
811 cpu_data_ptr->cpu_idle_latency = 0x0ULL;
812 cpu_data_ptr->cpu_idle_pop = 0x0ULL;
813 cpu_data_ptr->cpu_reset_type = 0x0UL;
814 cpu_data_ptr->cpu_reset_handler = 0x0UL;
815 cpu_data_ptr->cpu_reset_assist = 0x0UL;
816 cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
817 cpu_data_ptr->cpu_phys_id = 0x0UL;
818 cpu_data_ptr->cpu_l2_access_penalty = 0;
819 cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
820 cpu_data_ptr->cpu_cluster_id = 0;
821 cpu_data_ptr->cpu_l2_id = 0;
822 cpu_data_ptr->cpu_l2_size = 0;
823 cpu_data_ptr->cpu_l3_id = 0;
824 cpu_data_ptr->cpu_l3_size = 0;
825
826 cpu_data_ptr->cpu_signal = SIGPdisabled;
827
828 cpu_data_ptr->cpu_get_fiq_handler = NULL;
829 cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
830 cpu_data_ptr->cpu_tbd_hardware_val = NULL;
831 cpu_data_ptr->cpu_get_decrementer_func = NULL;
832 cpu_data_ptr->cpu_set_decrementer_func = NULL;
833 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
834 cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
835 cpu_data_ptr->cpu_xcall_p0 = NULL;
836 cpu_data_ptr->cpu_xcall_p1 = NULL;
837 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
838 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
839
840 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
841 cpu_data_ptr->coresight_base[i] = 0;
842 }
843
844 #if !XNU_MONITOR
845 pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
846
847 pmap_cpu_data_ptr->cpu_nested_pmap = (struct pmap *) NULL;
848 pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
849 pmap_cpu_data_ptr->pv_free.list = NULL;
850 pmap_cpu_data_ptr->pv_free.count = 0;
851 pmap_cpu_data_ptr->pv_free_spill_marker = NULL;
852
853 bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
854 #endif
855 cpu_data_ptr->halt_status = CPU_NOT_HALTED;
856 #if __ARM_KERNEL_PROTECT__
857 cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table;
858 #endif /* __ARM_KERNEL_PROTECT__ */
859
860 #if defined(HAS_APPLE_PAC)
861 cpu_data_ptr->rop_key = 0;
862 cpu_data_ptr->jop_key = ml_default_jop_pid();
863 #endif
864 }
865
866 kern_return_t
cpu_data_register(cpu_data_t * cpu_data_ptr)867 cpu_data_register(cpu_data_t *cpu_data_ptr)
868 {
869 int cpu = cpu_data_ptr->cpu_number;
870
871 #if KASAN
872 for (int i = 0; i < CPUWINDOWS_MAX; i++) {
873 kasan_notify_address_nopoison(pmap_cpu_windows_copy_addr(cpu, i), PAGE_SIZE);
874 }
875 #endif
876
877 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
878 CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
879 CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
880 return KERN_SUCCESS;
881 }
882
883 #if defined(KERNEL_INTEGRITY_CTRR)
884 /* Hibernation needs to reset this state, so data and text are in the hib segment;
885 * this allows them be accessed and executed early.
886 */
887 LCK_GRP_DECLARE(ctrr_cpu_start_lock_grp, "ctrr_cpu_start_lock");
888 LCK_SPIN_DECLARE(ctrr_cpu_start_lck, &ctrr_cpu_start_lock_grp);
889 enum ctrr_cluster_states ctrr_cluster_locked[MAX_CPU_CLUSTERS] MARK_AS_HIBERNATE_DATA;
890
891 MARK_AS_HIBERNATE_TEXT
892 void
init_ctrr_cluster_states(void)893 init_ctrr_cluster_states(void)
894 {
895 for (int i = 0; i < MAX_CPU_CLUSTERS; i++) {
896 ctrr_cluster_locked[i] = CTRR_UNLOCKED;
897 }
898 }
899 #endif
900
901 kern_return_t
cpu_start(int cpu)902 cpu_start(int cpu)
903 {
904 cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
905
906 kprintf("cpu_start() cpu: %d\n", cpu);
907
908 if (cpu == cpu_number()) {
909 cpu_machine_init();
910 configure_coresight_registers(cpu_data_ptr);
911 } else {
912 thread_t first_thread;
913 processor_t processor;
914
915 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
916
917 #if !XNU_MONITOR
918 cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL;
919 #endif
920
921 processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
922 if (processor->startup_thread != THREAD_NULL) {
923 first_thread = processor->startup_thread;
924 } else {
925 first_thread = processor->idle_thread;
926 }
927 cpu_data_ptr->cpu_active_thread = first_thread;
928 first_thread->machine.CpuDatap = cpu_data_ptr;
929 first_thread->machine.pcpu_data_base =
930 (vm_address_t)cpu_data_ptr - __PERCPU_ADDR(cpu_data);
931
932 configure_coresight_registers(cpu_data_ptr);
933
934 flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
935 flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
936 #if defined(KERNEL_INTEGRITY_CTRR)
937
938 /* First CPU being started within a cluster goes ahead to lock CTRR for cluster;
939 * other CPUs block until cluster is locked. */
940 lck_spin_lock(&ctrr_cpu_start_lck);
941 switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) {
942 case CTRR_UNLOCKED:
943 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING;
944 lck_spin_unlock(&ctrr_cpu_start_lck);
945 break;
946 case CTRR_LOCKING:
947 assert_wait(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id], THREAD_UNINT);
948 lck_spin_unlock(&ctrr_cpu_start_lck);
949 thread_block(THREAD_CONTINUE_NULL);
950 assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING);
951 break;
952 default: // CTRR_LOCKED
953 lck_spin_unlock(&ctrr_cpu_start_lck);
954 break;
955 }
956 #endif
957 (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
958 }
959
960 return KERN_SUCCESS;
961 }
962
963
964 void
cpu_timebase_init(boolean_t from_boot)965 cpu_timebase_init(boolean_t from_boot)
966 {
967 cpu_data_t *cdp = getCpuDatap();
968
969 if (cdp->cpu_get_fiq_handler == NULL) {
970 cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
971 cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
972 cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
973 cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
974 cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
975 }
976
977 if (!from_boot && (cdp == &BootCpuData)) {
978 /*
979 * When we wake from sleep, we have no guarantee about the state
980 * of the hardware timebase. It may have kept ticking across sleep, or
981 * it may have reset.
982 *
983 * To deal with this, we calculate an offset to the clock that will
984 * produce a timebase value wake_abstime at the point the boot
985 * CPU calls cpu_timebase_init on wake.
986 *
987 * This ensures that mach_absolute_time() stops ticking across sleep.
988 */
989 rtclock_base_abstime = wake_abstime - ml_get_hwclock();
990 } else if (from_boot) {
991 /* On initial boot, initialize time_since_reset to CNTPCT_EL0. */
992 ml_set_reset_time(ml_get_hwclock());
993 }
994
995 cdp->cpu_decrementer = 0x7FFFFFFFUL;
996 cdp->cpu_timebase = 0x0UL;
997 cdp->cpu_base_timebase = rtclock_base_abstime;
998 }
999
1000 int
cpu_cluster_id(void)1001 cpu_cluster_id(void)
1002 {
1003 return getCpuDatap()->cpu_cluster_id;
1004 }
1005
1006 __attribute__((noreturn))
1007 void
ml_arm_sleep(void)1008 ml_arm_sleep(void)
1009 {
1010 cpu_data_t *cpu_data_ptr = getCpuDatap();
1011
1012 if (cpu_data_ptr == &BootCpuData) {
1013 cpu_data_t *target_cdp;
1014 int cpu;
1015 int max_cpu;
1016
1017 max_cpu = ml_get_max_cpu_number();
1018 for (cpu = 0; cpu <= max_cpu; cpu++) {
1019 target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1020
1021 if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) {
1022 continue;
1023 }
1024
1025 while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
1026 ;
1027 }
1028 }
1029
1030 /*
1031 * Now that the other cores have entered the sleep path, set
1032 * the abstime value we'll use when we resume.
1033 */
1034 wake_abstime = ml_get_timebase();
1035 ml_set_reset_time(UINT64_MAX);
1036 } else {
1037 CleanPoU_Dcache();
1038 }
1039
1040 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
1041
1042 if (cpu_data_ptr == &BootCpuData) {
1043 #if WITH_CLASSIC_S2R
1044 // Classic suspend to RAM writes the suspend signature into the
1045 // sleep token buffer so that iBoot knows that it's on the warm
1046 // boot (wake) path (as opposed to the cold boot path). Newer SoC
1047 // do not go through SecureROM/iBoot on the warm boot path. The
1048 // reconfig engine script brings the CPU out of reset at the kernel's
1049 // reset vector which points to the warm boot initialization code.
1050 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1051 platform_cache_shutdown();
1052 bcopy((const void *)suspend_signature, (void *)sleepTokenBuffer, sizeof(SleepToken));
1053 } else {
1054 panic("No sleep token buffer");
1055 }
1056 #endif
1057
1058 #if __ARM_GLOBAL_SLEEP_BIT__
1059 /* Allow other CPUs to go to sleep. */
1060 arm64_stall_sleep = FALSE;
1061 __builtin_arm_dmb(DMB_ISH);
1062 #endif
1063
1064 /* Architectural debug state: <rdar://problem/12390433>:
1065 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1066 * tell debugger to not prevent power gating .
1067 */
1068 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1069 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1070 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1071 }
1072
1073 #if HIBERNATION
1074 uint32_t mode = hibernate_write_image();
1075 if (mode == kIOHibernatePostWriteHalt) {
1076 HIBLOG("powering off after writing hibernation image\n");
1077 int halt_result = -1;
1078 if (PE_halt_restart) {
1079 /**
1080 * Drain serial FIFOs now as the normal call further down won't
1081 * be hit when the CPU halts here for hibernation. Here, it'll
1082 * make sure the preceding HIBLOG is flushed as well.
1083 */
1084 serial_go_to_sleep();
1085 halt_result = (*PE_halt_restart)(kPEHaltCPU);
1086 }
1087 panic("can't shutdown: PE_halt_restart returned %d", halt_result);
1088 }
1089 #endif /* HIBERNATION */
1090
1091 serial_go_to_sleep();
1092
1093 #if MONOTONIC
1094 mt_sleep();
1095 #endif /* MONOTONIC */
1096 /* ARM64-specific preparation */
1097 arm64_prepare_for_sleep(true);
1098 } else {
1099 #if __ARM_GLOBAL_SLEEP_BIT__
1100 /*
1101 * With the exception of the CPU revisions listed above, our ARM64 CPUs have a
1102 * global register to manage entering deep sleep, as opposed to a per-CPU
1103 * register. We cannot update this register until all CPUs are ready to enter
1104 * deep sleep, because if a CPU executes WFI outside of the deep sleep context
1105 * (by idling), it will hang (due to the side effects of enabling deep sleep),
1106 * which can hang the sleep process or cause memory corruption on wake.
1107 *
1108 * To avoid these issues, we'll stall on this global value, which CPU0 will
1109 * manage.
1110 */
1111 while (arm64_stall_sleep) {
1112 __builtin_arm_wfe();
1113 }
1114 #endif
1115 CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
1116
1117 /* Architectural debug state: <rdar://problem/12390433>:
1118 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1119 * tell debugger to not prevent power gating .
1120 */
1121 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1122 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1123 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1124 }
1125
1126 /* ARM64-specific preparation */
1127 arm64_prepare_for_sleep(true);
1128 }
1129 }
1130
1131 void
cpu_machine_idle_init(boolean_t from_boot)1132 cpu_machine_idle_init(boolean_t from_boot)
1133 {
1134 static vm_address_t resume_idle_cpu_paddr = (vm_address_t)NULL;
1135 cpu_data_t *cpu_data_ptr = getCpuDatap();
1136
1137 if (from_boot) {
1138 uint32_t production = 1;
1139 DTEntry entry;
1140
1141 unsigned long jtag = 0;
1142
1143 if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
1144 if (jtag != 0) {
1145 idle_enable = FALSE;
1146 } else {
1147 idle_enable = TRUE;
1148 }
1149 } else {
1150 idle_enable = TRUE;
1151 }
1152
1153 #if DEVELOPMENT || DEBUG
1154 uint32_t wfe_mode = 0;
1155 if (PE_parse_boot_argn("wfe_mode", &wfe_mode, sizeof(wfe_mode))) {
1156 idle_proximate_timer_wfe = ((wfe_mode & 1) == 1);
1157 idle_proximate_io_wfe_masked = ((wfe_mode & 2) == 2);
1158 extern uint32_t idle_proximate_io_wfe_unmasked;
1159 idle_proximate_io_wfe_unmasked = ((wfe_mode & 4) == 4);
1160 }
1161 #endif
1162
1163 // bits 7..0 give the wfi type
1164 switch (wfi & 0xff) {
1165 case 0:
1166 // disable wfi
1167 wfi = 0;
1168 break;
1169
1170 #if DEVELOPMENT || DEBUG
1171 case 2:
1172 // wfi overhead simulation
1173 // 31..16 - wfi delay is us
1174 // 15..8 - flags
1175 // 7..0 - 2
1176 wfi = 2;
1177 wfi_flags = (wfi >> 8) & 0xFF;
1178 nanoseconds_to_absolutetime(((wfi >> 16) & 0xFFFF) * NSEC_PER_MSEC, &wfi_delay);
1179 break;
1180 #endif /* DEVELOPMENT || DEBUG */
1181
1182 case 1:
1183 default:
1184 // do nothing
1185 break;
1186 }
1187
1188 ResetHandlerData.assist_reset_handler = 0;
1189 ResetHandlerData.cpu_data_entries = ml_static_vtop((vm_offset_t)CpuDataEntries);
1190
1191 #ifdef MONITOR
1192 monitor_call(MONITOR_SET_ENTRY, (uintptr_t)ml_static_vtop((vm_offset_t)&LowResetVectorBase), 0, 0);
1193 #elif !defined(NO_MONITOR)
1194 #error MONITOR undefined, WFI power gating may not operate correctly
1195 #endif /* MONITOR */
1196
1197 // Determine if we are on production or debug chip
1198 if (kSuccess == SecureDTLookupEntry(NULL, "/chosen", &entry)) {
1199 unsigned int size;
1200 void const *prop;
1201
1202 if (kSuccess == SecureDTGetProperty(entry, "effective-production-status-ap", &prop, &size)) {
1203 if (size == 4) {
1204 bcopy(prop, &production, size);
1205 }
1206 }
1207 }
1208 if (!production) {
1209 #if defined(APPLE_ARM64_ARCH_FAMILY)
1210 // Enable coresight debug registers on debug-fused chips
1211 coresight_debug_enabled = TRUE;
1212 #endif
1213 }
1214
1215 start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);
1216 resume_idle_cpu_paddr = ml_static_vtop((vm_offset_t)&resume_idle_cpu);
1217 }
1218
1219 #if WITH_CLASSIC_S2R
1220 if (cpu_data_ptr == &BootCpuData) {
1221 static addr64_t SleepToken_low_paddr = (addr64_t)NULL;
1222 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1223 SleepToken_low_paddr = ml_vtophys(sleepTokenBuffer);
1224 } else {
1225 panic("No sleep token buffer");
1226 }
1227
1228 bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)running_signature),
1229 SleepToken_low_paddr, sizeof(SleepToken));
1230 flush_dcache((vm_offset_t)SleepToken, sizeof(SleepToken), TRUE);
1231 }
1232 ;
1233 #endif
1234
1235 cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
1236 clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
1237 }
1238
1239 _Atomic uint32_t cpu_idle_count = 0;
1240
1241 void
machine_track_platform_idle(boolean_t entry)1242 machine_track_platform_idle(boolean_t entry)
1243 {
1244 if (entry) {
1245 os_atomic_inc(&cpu_idle_count, relaxed);
1246 } else {
1247 os_atomic_dec(&cpu_idle_count, relaxed);
1248 }
1249 }
1250
1251 #if WITH_CLASSIC_S2R
1252 void
sleep_token_buffer_init(void)1253 sleep_token_buffer_init(void)
1254 {
1255 cpu_data_t *cpu_data_ptr = getCpuDatap();
1256 DTEntry entry;
1257 size_t size;
1258 void const * const *prop;
1259
1260 if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) {
1261 /* Find the stpage node in the device tree */
1262 if (kSuccess != SecureDTLookupEntry(0, "stram", &entry)) {
1263 return;
1264 }
1265
1266 if (kSuccess != SecureDTGetProperty(entry, "reg", (const void **)&prop, (unsigned int *)&size)) {
1267 return;
1268 }
1269
1270 /* Map the page into the kernel space */
1271 sleepTokenBuffer = ml_io_map(((vm_offset_t const *)prop)[0], ((vm_size_t const *)prop)[1]);
1272 }
1273 }
1274 #endif
1275