1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <pexpert/pexpert.h>
30 #if __arm64__
31 #include <pexpert/arm64/board_config.h>
32 #include <arm64/hv_hvc.h>
33 #endif /* __arm64__ */
34
35 #include <arm/cpuid_internal.h>
36 #include <arm/cpu_capabilities_public.h>
37 #include <arm/pmap.h>
38 #include <arm64/proc_reg.h>
39 #include <machine/machine_cpuid.h>
40 #include <machine/machine_routines.h>
41 #include <vm/vm_protos.h>
42
43
44 #if defined(APPLEAVALANCHE) && defined(ARM64_BOARD_CONFIG_T6020)
45 #define CHECK_RDAR_145882231 1
46 #endif
47
48 #if defined(CHECK_RDAR_145882231)
49 SECURITY_READ_ONLY_LATE(bool) needs_rdar_145882231 = FALSE;
50 extern char gTargetTypeBuffer[16];
51 #endif /* CHECK_RDAR_145882231 */
52
53 #if __arm64__
54
55 void configure_misc_apple_boot_args(void);
56 void configure_misc_apple_regs(bool is_boot_cpu);
57 void configure_timer_apple_regs(void);
58 void configure_late_apple_regs(bool cold_boot);
59
60 void
configure_misc_apple_boot_args(void)61 configure_misc_apple_boot_args(void)
62 {
63 }
64
65
66 void
configure_misc_apple_regs(bool is_boot_cpu)67 configure_misc_apple_regs(bool is_boot_cpu)
68 {
69 #pragma unused(is_boot_cpu)
70
71 }
72
73 // machine_routines_apple.c gets built on non-Apple platforms but it won't
74 // #include apple_arm64_regs.h so some of the constants referenced below
75 // won't exist in those builds
76 #if APPLE_ARM64_ARCH_FAMILY
77
78 static bool
cpu_needs_throttle_tunable(uint32_t midr_pnum)79 cpu_needs_throttle_tunable(uint32_t midr_pnum)
80 {
81 switch (midr_pnum) {
82 #if defined(APPLEAVALANCHE)
83 /* ACCP only */
84 case MIDR_RHODES_DIE_AVALANCHE:
85 return true;
86 #endif /* APPLEAVALANCHE */
87
88 #if defined(APPLEEVEREST)
89 case MIDR_IBIZA_ACCE:
90 case MIDR_IBIZA_ACCP:
91 return true;
92 case MIDR_LOBOS_ACCE:
93 case MIDR_LOBOS_ACCP:
94 return true;
95 case MIDR_PALMA_ACCE:
96 case MIDR_PALMA_ACCP:
97 return true;
98 case MIDR_COLL_ACCE:
99 case MIDR_COLL_ACCP:
100 return true;
101 #endif /* APPLEEVEREST */
102 default:
103 return false;
104 }
105 }
106
107 /*
108 * configure_late_apple_regs()
109 *
110 * Normal tunables (HID bits) are applied early on, in the APPLY_TUNABLES
111 * asm macro. This C function is intended to handle special cases where that
112 * isn't possible, e.g.
113 * - Tunables that require PIO mappings
114 * - Tunables that need access to the parsed CPU topology info
115 *
116 * Unlike configure_misc_apple_regs(), it is guaranteed to execute after
117 * ml_parse_cpu_topology() / ml_map_cpu_pio() are done,
118 * and after cpu_number() is valid.
119 */
120 void
configure_late_apple_regs(bool cold_boot)121 configure_late_apple_regs(bool cold_boot)
122 {
123 #if defined(CHECK_RDAR_145882231)
124 if (cold_boot) {
125 /* We only want the rdar://145882231 tunable to apply to J236c */
126 if (0 == strncmp(gTargetTypeBuffer, "J236c", sizeof(gTargetTypeBuffer))) {
127 needs_rdar_145882231 = true;
128 }
129 }
130
131 /* rdar://145882231 */
132 if (needs_rdar_145882231 && arm64_is_p_core()) {
133 uint64_t hid16 = __builtin_arm_rsr64("HID16");
134 hid16 |= ARM64_REG_HID16_leqThrottleAggr;
135 __builtin_arm_wsr64("HID16", hid16);
136 hid16 = __builtin_arm_rsr64("HID16");
137 const uint64_t expect = hid16 | ARM64_REG_HID16_leqThrottleAggr;
138 if (expect != hid16) {
139 panic("HID16 not as expected, Got: 0x%llx, Expected: 0x%llx", hid16, expect);
140 }
141 }
142 #endif /* CHECK_RDAR_145882231 */
143
144 const ml_topology_info_t *tinfo = ml_get_topology_info();
145 uint32_t midr_pnum = machine_read_midr() & MIDR_EL1_PNUM_MASK;
146 uint64_t reg_val;
147
148 bool apply_late_pio_regs = cold_boot;
149 #ifdef APPLEEVEREST
150 /*
151 * On H15 CPUs PIO locks are applied early in the non-cold boot
152 * path.
153 */
154 apply_late_pio_regs = 0;
155 #endif
156 if (apply_late_pio_regs) {
157 if (cpu_needs_throttle_tunable(midr_pnum)) {
158 vm_offset_t cpu_impl = tinfo->cpus[cpu_number()].cpu_IMPL_regs;
159 const uint64_t c1pptThrtlRate = 0xb2;
160 reg_val = ml_io_read64(cpu_impl + CORE_THRTL_CFG2_OFFSET);
161 reg_val &= ~CORE_THRTL_CFG2_c1pptThrtlRate_mask;
162 reg_val |= c1pptThrtlRate << CORE_THRTL_CFG2_c1pptThrtlRate_shift;
163 ml_io_write64(cpu_impl + CORE_THRTL_CFG2_OFFSET, reg_val);
164 }
165 }
166
167 #if defined(APPLEAVALANCHE)
168 if (tinfo->max_die_id > 0) {
169 if (midr_pnum == MIDR_RHODES_DIE_AVALANCHE || midr_pnum == MIDR_RHODES_DIE_BLIZZARD) {
170 // rdar://93675127 (Rhodes address match granularity for BIU)
171 reg_val = __builtin_arm_rsr64("HID5");
172 reg_val &= ~ARM64_REG_HID5_BiuBchMatchGran_mask;
173 reg_val |= ARM64_REG_HID5_BiuBchMatchGran_VALUE(0);
174 __builtin_arm_wsr64("HID5", reg_val);
175 }
176 }
177 #endif /* APPLEAVALANCHE */
178
179 #if defined(APPLEEVEREST)
180 #endif /* APPLEEVEREST */
181 }
182 #endif /* APPLE_ARM64_ARCH_FAMILY */
183
184 void
configure_timer_apple_regs(void)185 configure_timer_apple_regs(void)
186 {
187 }
188
189 #endif /* __arm64__ */
190
191 #if HAS_APPLE_PAC
192
193 #if HAS_PARAVIRTUALIZED_PAC
194 static uint64_t vmapple_default_rop_pid;
195 static uint64_t vmapple_default_jop_pid;
196
197 static inline void
vmapple_pac_get_default_keys()198 vmapple_pac_get_default_keys()
199 {
200 static bool initialized = false;
201 if (os_atomic_xchg(&initialized, true, relaxed)) {
202 return;
203 }
204
205 const uint64_t fn = VMAPPLE_PAC_GET_DEFAULT_KEYS;
206 asm volatile (
207 "mov x0, %[fn]" "\n"
208 "hvc #0" "\n"
209 "cbnz x0, ." "\n"
210 "str x2, %[b_key]" "\n"
211 "str x3, %[el0_key]" "\n"
212 : [b_key] "=m"(vmapple_default_rop_pid),
213 [el0_key] "=m"(vmapple_default_jop_pid)
214 : [fn] "r"(fn)
215 : "x0", "x1", "x2", "x3", "x4"
216 );
217 }
218
219 #endif /* HAS_PARAVIRTUALIZED_PAC */
220
221 /**
222 * Returns the default ROP key.
223 */
224 uint64_t
ml_default_rop_pid(void)225 ml_default_rop_pid(void)
226 {
227 #if HAS_PARAVIRTUALIZED_PAC
228 vmapple_pac_get_default_keys();
229 return vmapple_default_rop_pid;
230 #else
231 return 0;
232 #endif /* HAS_PARAVIRTUALIZED_PAC */
233 }
234
235 /**
236 * Returns the default JOP key.
237 */
238 uint64_t
ml_default_jop_pid(void)239 ml_default_jop_pid(void)
240 {
241 #if HAS_PARAVIRTUALIZED_PAC
242 vmapple_pac_get_default_keys();
243 return vmapple_default_jop_pid;
244 #else
245 return 0;
246 #endif /* HAS_PARAVIRTUALIZED_PAC */
247 }
248
249 /**
250 * Returns an appropriate JOP key for non-arm64e userspace processes. The
251 * return value may vary from call to call.
252 */
253 uint64_t
ml_non_arm64e_user_jop_pid(void)254 ml_non_arm64e_user_jop_pid(void)
255 {
256 #if HAS_PARAVIRTUALIZED_PAC
257 return generate_jop_key();
258 #else
259 return 0;
260 #endif /* HAS_PARAVIRTUALIZED_PAC */
261 }
262 #endif /* HAS_APPLE_PAC */
263
264