1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 */
4
5 #include <stdint.h>
6 #include <arm/proc_reg.h>
7 #include <kern/clock.h>
8 #include <mach/mach_time.h>
9 #include <machine/atomic.h>
10 #include <machine/machine_routines.h>
11 #include <pexpert/device_tree.h>
12 #if defined(__arm__)
13 #include <pexpert/arm/board_config.h>
14 #elif defined(__arm64__)
15 #include <pexpert/arm64/board_config.h>
16 #endif
17
18
19 #if HAS_GIC_V3
20 #define GICR_WAKE_TIMEOUT_NS (1000000000ULL) // timeout for redistributor wakeup (default 1s)
21 MACHINE_TIMEOUT(gicr_wake_timeout_ns, "gicr-wake-timeout", GICR_WAKE_TIMEOUT_NS, MACHINE_TIMEOUT_UNIT_NSEC, NULL);
22
23 static vm_offset_t gicd_base;
24 static vm_offset_t gicr_base;
25 static vm_offset_t gicr_size;
26
27 static uint32_t
_gic_read32(vm_offset_t addr)28 _gic_read32(vm_offset_t addr)
29 {
30 return *((volatile uint32_t *) addr);
31 }
32
33 static uint64_t
_gic_read64(vm_offset_t addr)34 _gic_read64(vm_offset_t addr)
35 {
36 return *((volatile uint64_t *) addr);
37 }
38
39 static void
_gic_write32(vm_offset_t addr,uint32_t value)40 _gic_write32(vm_offset_t addr, uint32_t value)
41 {
42 *((volatile uint32_t *) addr) = value;
43 }
44
45 #define gicd_read32(offset) (_gic_read32(gicd_base + (offset)))
46 #define gicd_write32(offset, data) (_gic_write32(gicd_base + (offset), (data)))
47 #define gicr_read32(offset) (_gic_read32(gicr_pe_base + (offset)))
48 #define gicr_write32(offset, data) (_gic_write32(gicr_pe_base + (offset), (data)))
49 #define gicr_read64(offset) (_gic_read64(gicr_pe_base + (offset)))
50
51 static vm_offset_t
find_gicr_pe_base()52 find_gicr_pe_base()
53 {
54 // We only care about aff1 and aff0
55 uint32_t phys_id = __builtin_arm_rsr64("MPIDR_EL1") & (MPIDR_AFF1_MASK | MPIDR_AFF0_MASK);
56
57 for (vm_offset_t offset = 0; offset < gicr_size; offset += GICR_PE_SIZE) {
58 vm_offset_t gicr_pe_base = gicr_base + offset;
59 uint64_t gicr_typer = gicr_read64(GICR_TYPER);
60 uint32_t aff_value = (uint32_t) (gicr_typer >> GICR_TYPER_AFFINITY_VALUE_SHIFT) & (MPIDR_AFF1_MASK | MPIDR_AFF0_MASK);
61
62 if (phys_id == aff_value) {
63 return gicr_pe_base;
64 }
65
66 if (gicr_typer & GICR_TYPER_LAST) {
67 break;
68 }
69 }
70
71 panic("%s: cannot find GICR base for core %u", __func__, ml_get_cpu_number(phys_id));
72 }
73
74 void
pe_init_fiq()75 pe_init_fiq()
76 {
77 int error;
78 DTEntry entry;
79
80 // gicd_base is 0x0 only before it's initialized by the boot processor.
81 // Avoid calling SecureDT* routines on secondary processors to avoid
82 // race conditions because they are not thread-safe.
83 if (!gicd_base) {
84 // Find GIC DT node
85 error = SecureDTLookupEntry(NULL, "/arm-io/gic", &entry);
86 if (error != kSuccess) {
87 panic("%s: cannot find GIC node in DT", __func__);
88 }
89
90 // Find "reg" property
91 void const *prop;
92 unsigned int prop_size;
93 error = SecureDTGetProperty(entry, "reg", &prop, &prop_size);
94 if (error != kSuccess) {
95 panic("%s: cannot find GIC MMIO regions in DT", __func__);
96 }
97
98 // Need at least GICD base, GICD size, GICR base and GICR size
99 if (prop_size < 4 * sizeof(uint64_t)) {
100 panic("%s: incorrect reg property size in GIC DT node; expecting 32 bytes but got %u bytes", __func__, prop_size);
101 }
102
103 vm_offset_t soc_base_phys = pe_arm_get_soc_base_phys();
104
105 uint64_t const gicd_base_prop = ((uint64_t const *) prop)[0];
106 uint64_t const gicd_size_prop = ((uint64_t const *) prop)[1];
107 uint64_t const gicr_base_prop = ((uint64_t const *) prop)[2];
108 uint64_t const gicr_size_prop = ((uint64_t const *) prop)[3];
109
110 // Find GICD base address
111 gicd_base = ml_io_map(soc_base_phys + gicd_base_prop, gicd_size_prop);
112
113 if (!gicd_base) {
114 panic("%s: cannot map GICD region", __func__);
115 }
116
117 // Find GICR base address
118 gicr_base = ml_io_map(soc_base_phys + gicr_base_prop, gicr_size_prop);
119
120 if (!gicr_base) {
121 panic("%s: cannot map GICR region", __func__);
122 }
123
124 gicr_size = gicr_size_prop;
125 }
126
127 // Find the redistributor for this processor
128 vm_offset_t gicr_pe_base = find_gicr_pe_base();
129
130 // Mark this PE to be awake
131 uint32_t gicr_waker = gicr_read32(GICR_WAKER);
132 if (gicr_waker & GICR_WAKER_CHILDRENASLEEP) {
133 gicr_waker &= ~GICR_WAKER_PROCESSORSLEEP;
134
135 gicr_write32(GICR_WAKER, gicr_waker);
136
137 uint64_t gicr_wake_deadline;
138 nanoseconds_to_deadline(os_atomic_load(&gicr_wake_timeout_ns, relaxed), &gicr_wake_deadline);
139 while (gicr_read32(GICR_WAKER) & GICR_WAKER_CHILDRENASLEEP) {
140 // Spin
141 if (gicr_wake_timeout_ns > 0 && mach_absolute_time() > gicr_wake_deadline) {
142 panic("%s: core %u timed out waiting for redistributor to wake up",
143 __func__, ml_get_cpu_number_local());
144 }
145 }
146 }
147
148 // Configure timers and legacy FIQ to be group 0
149 gicr_write32(GICR_IGROUPR0, 0x81FFFFFF);
150
151 // Enable PPI 27
152 gicr_write32(GICR_ISENABLER0, (1 << 27));
153
154 // Enable system register access
155 uint64_t icc_sre = __builtin_arm_rsr64("ICC_SRE_EL1");
156 icc_sre |= ICC_SRE_SRE;
157 __builtin_arm_wsr64("ICC_SRE_EL1", icc_sre);
158 __builtin_arm_isb(ISB_SY);
159
160 // Set priority masks and binary point for group 0
161 __builtin_arm_wsr64("ICC_BPR0_EL1", 0);
162 __builtin_arm_wsr64("ICC_PMR_EL1", 0xFF);
163
164 // Set EOI mode of this processor
165 uint64_t icc_ctlr = __builtin_arm_rsr64("ICC_CTLR_EL1");
166 icc_ctlr &= ~ICC_CTLR_EOIMODE;
167 __builtin_arm_wsr64("ICC_CTLR_EL1", icc_ctlr);
168
169 // Enable the forwarding of the vtimer interrupt
170 uint32_t gicd_ctlr = gicd_read32(GICD_CTLR);
171 gicd_ctlr |= GICD_CTLR_ENABLEGRP0;
172 gicd_write32(GICD_CTLR, gicd_ctlr);
173 __builtin_arm_wsr64("ICC_IGRPEN0_EL1", 1);
174 __builtin_arm_isb(ISB_SY);
175 }
176 #else
177 void
pe_init_fiq()178 pe_init_fiq()
179 {
180 }
181 #endif
182