1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 */
4
5 #include <stdint.h>
6 #include <arm64/proc_reg.h>
7 #include <kern/clock.h>
8 #include <mach/mach_time.h>
9 #include <machine/atomic.h>
10 #include <machine/machine_routines.h>
11 #include <pexpert/device_tree.h>
12 #include <pexpert/arm64/board_config.h>
13
14
15 #if HAS_GIC_V3
16 #define GICR_WAKE_TIMEOUT_NS (1000000000ULL) // timeout for redistributor wakeup (default 1s)
17 MACHINE_TIMEOUT(gicr_wake_timeout_ns, "gicr-wake-timeout", GICR_WAKE_TIMEOUT_NS, MACHINE_TIMEOUT_UNIT_NSEC, NULL);
18
19 static vm_offset_t gicd_base;
20 static vm_offset_t gicr_base;
21 static vm_offset_t gicr_size;
22
23 static uint32_t
_gic_read32(vm_offset_t addr)24 _gic_read32(vm_offset_t addr)
25 {
26 return *((volatile uint32_t *) addr);
27 }
28
29 static uint64_t
_gic_read64(vm_offset_t addr)30 _gic_read64(vm_offset_t addr)
31 {
32 return *((volatile uint64_t *) addr);
33 }
34
35 static void
_gic_write32(vm_offset_t addr,uint32_t value)36 _gic_write32(vm_offset_t addr, uint32_t value)
37 {
38 *((volatile uint32_t *) addr) = value;
39 }
40
41 #define gicd_read32(offset) (_gic_read32(gicd_base + (offset)))
42 #define gicd_write32(offset, data) (_gic_write32(gicd_base + (offset), (data)))
43 #define gicr_read32(offset) (_gic_read32(gicr_pe_base + (offset)))
44 #define gicr_write32(offset, data) (_gic_write32(gicr_pe_base + (offset), (data)))
45 #define gicr_read64(offset) (_gic_read64(gicr_pe_base + (offset)))
46
47 static vm_offset_t
find_gicr_pe_base()48 find_gicr_pe_base()
49 {
50 // We only care about aff1 and aff0
51 uint32_t phys_id = __builtin_arm_rsr64("MPIDR_EL1") & (MPIDR_AFF1_MASK | MPIDR_AFF0_MASK);
52
53 for (vm_offset_t offset = 0; offset < gicr_size; offset += GICR_PE_SIZE) {
54 vm_offset_t gicr_pe_base = gicr_base + offset;
55 uint64_t gicr_typer = gicr_read64(GICR_TYPER);
56 uint32_t aff_value = (uint32_t) (gicr_typer >> GICR_TYPER_AFFINITY_VALUE_SHIFT) & (MPIDR_AFF1_MASK | MPIDR_AFF0_MASK);
57
58 if (phys_id == aff_value) {
59 return gicr_pe_base;
60 }
61
62 if (gicr_typer & GICR_TYPER_LAST) {
63 break;
64 }
65 }
66
67 panic("%s: cannot find GICR base for core %u", __func__, ml_get_cpu_number(phys_id));
68 }
69
70 void
pe_init_fiq()71 pe_init_fiq()
72 {
73 int error;
74 DTEntry entry;
75
76 // gicd_base is 0x0 only before it's initialized by the boot processor.
77 // Avoid calling SecureDT* routines on secondary processors to avoid
78 // race conditions because they are not thread-safe.
79 if (!gicd_base) {
80 // Find GIC DT node
81 error = SecureDTLookupEntry(NULL, "/arm-io/gic", &entry);
82 if (error != kSuccess) {
83 panic("%s: cannot find GIC node in DT", __func__);
84 }
85
86 // Find "reg" property
87 void const *prop;
88 unsigned int prop_size;
89 error = SecureDTGetProperty(entry, "reg", &prop, &prop_size);
90 if (error != kSuccess) {
91 panic("%s: cannot find GIC MMIO regions in DT", __func__);
92 }
93
94 // Need at least GICD base, GICD size, GICR base and GICR size
95 if (prop_size < 4 * sizeof(uint64_t)) {
96 panic("%s: incorrect reg property size in GIC DT node; expecting 32 bytes but got %u bytes", __func__, prop_size);
97 }
98
99 vm_offset_t soc_base_phys = pe_arm_get_soc_base_phys();
100
101 uint64_t const gicd_base_prop = ((uint64_t const *) prop)[0];
102 uint64_t const gicd_size_prop = ((uint64_t const *) prop)[1];
103 uint64_t const gicr_base_prop = ((uint64_t const *) prop)[2];
104 uint64_t const gicr_size_prop = ((uint64_t const *) prop)[3];
105
106 // Find GICD base address
107 gicd_base = ml_io_map(soc_base_phys + gicd_base_prop, gicd_size_prop);
108
109 if (!gicd_base) {
110 panic("%s: cannot map GICD region", __func__);
111 }
112
113 // Find GICR base address
114 gicr_base = ml_io_map(soc_base_phys + gicr_base_prop, gicr_size_prop);
115
116 if (!gicr_base) {
117 panic("%s: cannot map GICR region", __func__);
118 }
119
120 gicr_size = gicr_size_prop;
121 }
122
123 // Find the redistributor for this processor
124 vm_offset_t gicr_pe_base = find_gicr_pe_base();
125
126 // Mark this PE to be awake
127 uint32_t gicr_waker = gicr_read32(GICR_WAKER);
128 if (gicr_waker & GICR_WAKER_CHILDRENASLEEP) {
129 gicr_waker &= ~GICR_WAKER_PROCESSORSLEEP;
130
131 gicr_write32(GICR_WAKER, gicr_waker);
132
133 uint64_t gicr_wake_deadline;
134 nanoseconds_to_deadline(os_atomic_load(&gicr_wake_timeout_ns, relaxed), &gicr_wake_deadline);
135 while (gicr_read32(GICR_WAKER) & GICR_WAKER_CHILDRENASLEEP) {
136 // Spin
137 if (gicr_wake_timeout_ns > 0 && mach_absolute_time() > gicr_wake_deadline) {
138 panic("%s: core %u timed out waiting for redistributor to wake up",
139 __func__, ml_get_cpu_number_local());
140 }
141 }
142 }
143
144 // Configure timers and legacy FIQ to be group 0
145 gicr_write32(GICR_IGROUPR0, 0x81FFFFFF);
146
147 // Enable PPI 27
148 gicr_write32(GICR_ISENABLER0, (1 << 27));
149
150 // Enable system register access
151 uint64_t icc_sre = __builtin_arm_rsr64("ICC_SRE_EL1");
152 icc_sre |= ICC_SRE_SRE;
153 __builtin_arm_wsr64("ICC_SRE_EL1", icc_sre);
154 __builtin_arm_isb(ISB_SY);
155
156 // Set priority masks and binary point for group 0
157 __builtin_arm_wsr64("ICC_BPR0_EL1", 0);
158 __builtin_arm_wsr64("ICC_PMR_EL1", 0xFF);
159
160 // Set EOI mode of this processor
161 uint64_t icc_ctlr = __builtin_arm_rsr64("ICC_CTLR_EL1");
162 icc_ctlr &= ~ICC_CTLR_EOIMODE;
163 __builtin_arm_wsr64("ICC_CTLR_EL1", icc_ctlr);
164
165 // Enable the forwarding of the vtimer interrupt
166 uint32_t gicd_ctlr = gicd_read32(GICD_CTLR);
167 gicd_ctlr |= GICD_CTLR_ENABLEGRP0;
168 gicd_write32(GICD_CTLR, gicd_ctlr);
169 __builtin_arm_wsr64("ICC_IGRPEN0_EL1", 1);
170 __builtin_arm_isb(ISB_SY);
171 }
172 #else
173 void
pe_init_fiq()174 pe_init_fiq()
175 {
176 }
177 #endif
178