1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <pexpert/pexpert.h>
30 #if __arm64__
31 #include <pexpert/arm64/board_config.h>
32 #include <arm64/hv_hvc.h>
33 #endif /* __arm64__ */
34
35 #include <arm/cpuid_internal.h>
36 #include <arm/cpu_capabilities_public.h>
37 #include <arm/pmap.h>
38 #include <arm64/proc_reg.h>
39 #include <machine/machine_cpuid.h>
40 #include <machine/machine_routines.h>
41 #include <vm/vm_protos.h>
42
43 #if APPLEVIRTUALPLATFORM
44 #include <pexpert/device_tree.h>
45 #endif
46
47
48 #if defined(APPLEAVALANCHE) && defined(ARM64_BOARD_CONFIG_T6020)
49 #define CHECK_RDAR_145882231 1
50 #endif
51
52 #if defined(CHECK_RDAR_145882231)
53 SECURITY_READ_ONLY_LATE(bool) needs_rdar_145882231 = FALSE;
54 extern char gTargetTypeBuffer[16];
55 #endif /* CHECK_RDAR_145882231 */
56
57 #if __arm64__
58
59 void configure_misc_apple_boot_args(void);
60 void configure_misc_apple_regs(bool is_boot_cpu);
61 void configure_timer_apple_regs(void);
62 void configure_late_apple_regs(bool cold_boot);
63
64 void
configure_misc_apple_boot_args(void)65 configure_misc_apple_boot_args(void)
66 {
67 }
68
69
70 void
configure_misc_apple_regs(bool is_boot_cpu)71 configure_misc_apple_regs(bool is_boot_cpu)
72 {
73 #pragma unused(is_boot_cpu)
74
75 }
76
77 // machine_routines_apple.c gets built on non-Apple platforms but it won't
78 // #include apple_arm64_regs.h so some of the constants referenced below
79 // won't exist in those builds
80 #if APPLE_ARM64_ARCH_FAMILY
81
82 static bool
cpu_needs_throttle_tunable(uint32_t midr_pnum)83 cpu_needs_throttle_tunable(uint32_t midr_pnum)
84 {
85 switch (midr_pnum) {
86 #if defined(APPLEAVALANCHE)
87 /* ACCP only */
88 case MIDR_RHODES_DIE_AVALANCHE:
89 return true;
90 #endif /* APPLEAVALANCHE */
91
92 #if defined(APPLEEVEREST)
93 case MIDR_IBIZA_ACCE:
94 case MIDR_IBIZA_ACCP:
95 return true;
96 case MIDR_LOBOS_ACCE:
97 case MIDR_LOBOS_ACCP:
98 return true;
99 case MIDR_PALMA_ACCE:
100 case MIDR_PALMA_ACCP:
101 return true;
102 case MIDR_COLL_ACCE:
103 case MIDR_COLL_ACCP:
104 return true;
105 #endif /* APPLEEVEREST */
106 default:
107 return false;
108 }
109 }
110
111 /*
112 * configure_late_apple_regs()
113 *
114 * Normal tunables (HID bits) are applied early on, in the APPLY_TUNABLES
115 * asm macro. This C function is intended to handle special cases where that
116 * isn't possible, e.g.
117 * - Tunables that require PIO mappings
118 * - Tunables that need access to the parsed CPU topology info
119 *
120 * Unlike configure_misc_apple_regs(), it is guaranteed to execute after
121 * ml_parse_cpu_topology() / ml_map_cpu_pio() are done,
122 * and after cpu_number() is valid.
123 */
124 void
configure_late_apple_regs(bool cold_boot)125 configure_late_apple_regs(bool cold_boot)
126 {
127 #if defined(CHECK_RDAR_145882231)
128 if (cold_boot) {
129 /* We only want the rdar://145882231 tunable to apply to J236c */
130 if (0 == strncmp(gTargetTypeBuffer, "J236c", sizeof(gTargetTypeBuffer))) {
131 needs_rdar_145882231 = true;
132 }
133 }
134
135 /* rdar://145882231 */
136 if (needs_rdar_145882231 && arm64_is_p_core()) {
137 uint64_t hid16 = __builtin_arm_rsr64("HID16");
138 hid16 |= ARM64_REG_HID16_leqThrottleAggr;
139 __builtin_arm_wsr64("HID16", hid16);
140 hid16 = __builtin_arm_rsr64("HID16");
141 const uint64_t expect = hid16 | ARM64_REG_HID16_leqThrottleAggr;
142 if (expect != hid16) {
143 panic("HID16 not as expected, Got: 0x%llx, Expected: 0x%llx", hid16, expect);
144 }
145 }
146 #endif /* CHECK_RDAR_145882231 */
147
148 const ml_topology_info_t *tinfo = ml_get_topology_info();
149 uint32_t midr_pnum = machine_read_midr() & MIDR_EL1_PNUM_MASK;
150 uint64_t reg_val;
151
152 bool apply_late_pio_regs = cold_boot;
153 #ifdef APPLEEVEREST
154 /*
155 * On H15 CPUs PIO locks are applied early in the non-cold boot
156 * path.
157 */
158 apply_late_pio_regs = 0;
159 #endif
160 if (apply_late_pio_regs) {
161 if (cpu_needs_throttle_tunable(midr_pnum)) {
162 vm_offset_t cpu_impl = tinfo->cpus[cpu_number()].cpu_IMPL_regs;
163 const uint64_t c1pptThrtlRate = 0xb2;
164 reg_val = ml_io_read64(cpu_impl + CORE_THRTL_CFG2_OFFSET);
165 reg_val &= ~CORE_THRTL_CFG2_c1pptThrtlRate_mask;
166 reg_val |= c1pptThrtlRate << CORE_THRTL_CFG2_c1pptThrtlRate_shift;
167 ml_io_write64(cpu_impl + CORE_THRTL_CFG2_OFFSET, reg_val);
168 }
169 }
170
171 #if defined(APPLEAVALANCHE)
172 if (tinfo->max_die_id > 0) {
173 if (midr_pnum == MIDR_RHODES_DIE_AVALANCHE || midr_pnum == MIDR_RHODES_DIE_BLIZZARD) {
174 // rdar://93675127 (Rhodes address match granularity for BIU)
175 reg_val = __builtin_arm_rsr64("HID5");
176 reg_val &= ~ARM64_REG_HID5_BiuBchMatchGran_mask;
177 reg_val |= ARM64_REG_HID5_BiuBchMatchGran_VALUE(0);
178 __builtin_arm_wsr64("HID5", reg_val);
179 }
180 }
181 #endif /* APPLEAVALANCHE */
182
183 #if defined(APPLEEVEREST)
184 #endif /* APPLEEVEREST */
185 }
186 #endif /* APPLE_ARM64_ARCH_FAMILY */
187
188 void
configure_timer_apple_regs(void)189 configure_timer_apple_regs(void)
190 {
191 }
192
193 #endif /* __arm64__ */
194
195 #if HAS_APPLE_PAC
196
197 #if HAS_PARAVIRTUALIZED_PAC
198 static uint64_t vmapple_default_rop_pid;
199 static uint64_t vmapple_default_jop_pid;
200
201 static inline void
vmapple_pac_get_default_keys()202 vmapple_pac_get_default_keys()
203 {
204 static bool initialized = false;
205 if (os_atomic_xchg(&initialized, true, relaxed)) {
206 return;
207 }
208
209 const uint64_t fn = VMAPPLE_PAC_GET_DEFAULT_KEYS;
210 asm volatile (
211 "mov x0, %[fn]" "\n"
212 "hvc #0" "\n"
213 "cbnz x0, ." "\n"
214 "str x2, %[b_key]" "\n"
215 "str x3, %[el0_key]" "\n"
216 : [b_key] "=m"(vmapple_default_rop_pid),
217 [el0_key] "=m"(vmapple_default_jop_pid)
218 : [fn] "r"(fn)
219 : "x0", "x1", "x2", "x3", "x4"
220 );
221 }
222
223 #endif /* HAS_PARAVIRTUALIZED_PAC */
224
225 /**
226 * Returns the default ROP key.
227 */
228 uint64_t
ml_default_rop_pid(void)229 ml_default_rop_pid(void)
230 {
231 #if HAS_PARAVIRTUALIZED_PAC
232 vmapple_pac_get_default_keys();
233 return vmapple_default_rop_pid;
234 #else
235 return 0;
236 #endif /* HAS_PARAVIRTUALIZED_PAC */
237 }
238
239 /**
240 * Returns the default JOP key.
241 */
242 uint64_t
ml_default_jop_pid(void)243 ml_default_jop_pid(void)
244 {
245 #if HAS_PARAVIRTUALIZED_PAC
246 vmapple_pac_get_default_keys();
247 return vmapple_default_jop_pid;
248 #else
249 return 0;
250 #endif /* HAS_PARAVIRTUALIZED_PAC */
251 }
252
253 /**
254 * Returns an appropriate JOP key for non-arm64e userspace processes. The
255 * return value may vary from call to call.
256 */
257 uint64_t
ml_non_arm64e_user_jop_pid(void)258 ml_non_arm64e_user_jop_pid(void)
259 {
260 #if HAS_PARAVIRTUALIZED_PAC
261 return generate_jop_key();
262 #else
263 return 0;
264 #endif /* HAS_PARAVIRTUALIZED_PAC */
265 }
266 #endif /* HAS_APPLE_PAC */
267
268
269