1 /*
2 * Copyright (c) 2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <darwintest.h>
30 #include <machine/cpu_capabilities.h>
31 #include <sys/sysctl.h>
32
33 #include "exc_helpers.h"
34
35 T_GLOBAL_META(
36 T_META_NAMESPACE("xnu.arm"),
37 T_META_RADAR_COMPONENT_NAME("xnu"),
38 T_META_RADAR_COMPONENT_VERSION("arm"),
39 T_META_OWNER("sdooher"),
40 T_META_RUN_CONCURRENTLY(true)
41 );
42
43 static volatile bool cap_usable;
44
45 static size_t
bad_instruction_handler(mach_port_t task __unused,mach_port_t thread __unused,exception_type_t type __unused,mach_exception_data_t codes __unused)46 bad_instruction_handler(mach_port_t task __unused, mach_port_t thread __unused,
47 exception_type_t type __unused, mach_exception_data_t codes __unused)
48 {
49 cap_usable = false;
50 return 4;
51 }
52
53 static void
try_fp16(void)54 try_fp16(void)
55 {
56 asm volatile (
57 "fmov h0, #0" "\n"
58 :
59 :
60 : "v0"
61 );
62 }
63
64 static void
try_atomics(void)65 try_atomics(void)
66 {
67 uint64_t dword;
68 asm volatile (
69 "swp xzr, xzr, [%[dword]]"
70 :
71 : [dword]"r"(&dword)
72 );
73 }
74
75 static void
try_crc32(void)76 try_crc32(void)
77 {
78 asm volatile ( "crc32b wzr, wzr, wzr");
79 }
80
81 static void
try_fhm(void)82 try_fhm(void)
83 {
84 asm volatile (
85 "fmov d0, #0" "\n"
86 "fmlal v0.2s, v0.2h, v0.2h" "\n"
87 :
88 :
89 : "v0"
90 );
91 }
92
93 static void
try_sha512(void)94 try_sha512(void)
95 {
96 asm volatile (
97 "fmov d0, #0" "\n"
98 "fmov d1, #0" "\n"
99 "sha512h q0, q0, v0.2d" "\n"
100 :
101 :
102 : "v0"
103 );
104 }
105
106 static void
try_sha3(void)107 try_sha3(void)
108 {
109 asm volatile (
110 "fmov d0, #0" "\n"
111 "fmov d1, #0" "\n"
112 "eor3 v0.16b, v0.16b, v0.16b, v0.16b" "\n"
113 :
114 :
115 : "v0"
116 );
117 }
118
119 static void
try_sha1(void)120 try_sha1(void)
121 {
122 asm volatile (
123 "fmov s0, #0" "\n"
124 "sha1h s0, s0" "\n"
125 :
126 :
127 : "v0"
128 );
129 }
130
131 static void
try_pmull(void)132 try_pmull(void)
133 {
134 asm volatile (
135 "fmov d0, #0" "\n"
136 "pmull v0.1q, v0.1d, v0.1d" "\n"
137 :
138 :
139 : "v0"
140 );
141 }
142
143 static void
try_aes(void)144 try_aes(void)
145 {
146 asm volatile (
147 "fmov d0, #0" "\n"
148 "fmov d1, #0" "\n"
149 "aesd v0.16B, v0.16B" "\n"
150 :
151 :
152 : "v0"
153 );
154 }
155
156
157 static void
try_sha256(void)158 try_sha256(void)
159 {
160 asm volatile (
161 "fmov d0, #0" "\n"
162 "fmov d1, #0" "\n"
163 "sha256h q0, q0, v0.4s" "\n"
164 :
165 :
166 : "v0"
167 );
168 }
169
170
171 static void
try_compnum(void)172 try_compnum(void)
173 {
174 asm volatile (
175 "fmov d0, #0" "\n"
176 "fcadd v0.2s, v0.2s, v0.2s, #90" "\n"
177 :
178 :
179 : "v0"
180 );
181 }
182
183
184 static void
try_flagm(void)185 try_flagm(void)
186 {
187 asm volatile (
188 "cfinv" "\n"
189 "cfinv" "\n"
190 );
191 }
192
193 static void
try_flagm2(void)194 try_flagm2(void)
195 {
196 asm volatile (
197 "axflag" "\n"
198 "xaflag" "\n"
199 );
200 }
201
202 static void
try_dotprod(void)203 try_dotprod(void)
204 {
205 asm volatile (
206 "udot v0.4S,v1.16B,v2.16B"
207 :
208 :
209 : "v0"
210 );
211 }
212
213 static void
try_rdm(void)214 try_rdm(void)
215 {
216 asm volatile (
217 "sqrdmlah s0, s1, s2"
218 :
219 :
220 : "s0"
221 );
222 }
223
224 static void
try_sb(void)225 try_sb(void)
226 {
227 asm volatile (
228 "sb"
229 );
230 }
231
232 static void
try_frintts(void)233 try_frintts(void)
234 {
235 asm volatile (
236 "frint32x s0, s0"
237 :
238 :
239 : "s0"
240 );
241 }
242
243 static void
try_jscvt(void)244 try_jscvt(void)
245 {
246 asm volatile (
247 "fmov d0, #0" "\n"
248 "fjcvtzs w1, d0" "\n"
249 :
250 :
251 : "w1", "d0"
252 );
253 }
254
255 static void
try_pauth(void)256 try_pauth(void)
257 {
258 asm volatile (
259 "pacga x0, x0, x0"
260 :
261 :
262 : "x0"
263 );
264 }
265
266 static void
try_dpb(void)267 try_dpb(void)
268 {
269 int x;
270 asm volatile (
271 "dc cvap, %0"
272 :
273 : "r" (&x)
274 );
275 }
276
277 static void
try_dpb2(void)278 try_dpb2(void)
279 {
280 int x;
281 asm volatile (
282 "dc cvadp, %0"
283 :
284 : "r" (&x)
285 );
286 }
287
288 static void
try_lrcpc(void)289 try_lrcpc(void)
290 {
291 int x;
292 asm volatile (
293 "ldaprb w0, [%0]"
294 :
295 : "r" (&x)
296 : "w0"
297 );
298 }
299
300 static void
try_lrcpc2(void)301 try_lrcpc2(void)
302 {
303 int x;
304 asm volatile (
305 "ldapurb w0, [%0]"
306 :
307 : "r" (&x)
308 : "w0"
309 );
310 }
311
312
313 static void
try_specres(void)314 try_specres(void)
315 {
316 int x;
317 asm volatile (
318 "cfp rctx, %0"
319 :
320 : "r" (&x)
321 );
322 }
323
324 static void
try_bf16(void)325 try_bf16(void)
326 {
327 asm volatile (
328 "bfdot v0.4S,v1.8H,v2.8H"
329 :
330 :
331 : "v0"
332 );
333 }
334
335 static void
try_i8mm(void)336 try_i8mm(void)
337 {
338 asm volatile (
339 "sudot v0.4S,v1.16B,v2.4B[0]"
340 :
341 :
342 : "v0"
343 );
344 }
345
346
347 static void
try_fpexcp(void)348 try_fpexcp(void)
349 {
350 /* FP Exceptions are supported if all exceptions bit can be set. */
351 const uint64_t flags = (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 15);
352
353 uint64_t old_fpcr = __builtin_arm_rsr64("FPCR");
354 __builtin_arm_wsr64("FPCR", old_fpcr | flags);
355 uint64_t new_fpcr = __builtin_arm_rsr64("FPCR");
356 __builtin_arm_wsr64("FPCR", old_fpcr);
357
358 if ((new_fpcr & flags) != flags) {
359 cap_usable = false;
360 }
361 }
362
363 static void
try_dit(void)364 try_dit(void)
365 {
366 asm volatile (
367 "msr DIT, x0"
368 :
369 :
370 : "x0"
371 );
372 }
373
374 static mach_port_t exc_port;
375
376 static void
test_cpu_capability(const char * cap_name,uint64_t cap_flag,bool has_commpage_entry,const char * cap_sysctl,void (* try_cpu_capability)(void))377 test_cpu_capability(const char *cap_name, uint64_t cap_flag, bool has_commpage_entry, const char *cap_sysctl, void (*try_cpu_capability)(void))
378 {
379 uint64_t caps = _get_cpu_capabilities();
380 bool has_cap_flag = (caps & cap_flag);
381
382 int sysctl_val;
383 bool has_sysctl_flag = 0;
384 if (cap_sysctl != NULL) {
385 size_t sysctl_size = sizeof(sysctl_val);
386 int err = sysctlbyname(cap_sysctl, &sysctl_val, &sysctl_size, NULL, 0);
387 has_sysctl_flag = (err == 0 && sysctl_val > 0);
388 }
389
390 bool has_capability = has_commpage_entry ? has_cap_flag : has_sysctl_flag;
391
392 if (!has_commpage_entry && cap_sysctl == NULL) {
393 T_FAIL("Tested capability must have either sysctl or commpage flag");
394 return;
395 }
396
397 if (has_commpage_entry && cap_sysctl != NULL) {
398 T_EXPECT_EQ(has_cap_flag, has_sysctl_flag, "%s commpage flag matches sysctl flag", cap_name);
399 }
400
401 if (try_cpu_capability != NULL) {
402 cap_usable = true;
403 try_cpu_capability();
404 T_EXPECT_EQ(has_capability, cap_usable, "%s capability matches actual usability", cap_name);
405 }
406 }
407
408 T_DECL(cpu_capabilities, "Verify ARM CPU capabilities") {
409 exc_port = create_exception_port(EXC_MASK_BAD_INSTRUCTION);
410 repeat_exception_handler(exc_port, bad_instruction_handler);
411
412 test_cpu_capability("FP16 (deprecated sysctl)", kHasFeatFP16, true, "hw.optional.neon_fp16", NULL);
413 test_cpu_capability("FP16", kHasFeatFP16, true, "hw.optional.arm.FEAT_FP16", try_fp16);
414 test_cpu_capability("LSE (deprecated sysctl)", kHasFeatLSE, true, "hw.optional.armv8_1_atomics", NULL);
415 test_cpu_capability("LSE", kHasFeatLSE, true, "hw.optional.arm.FEAT_LSE", try_atomics);
416 test_cpu_capability("CRC32", kHasARMv8Crc32, true, "hw.optional.armv8_crc32", try_crc32);
417 test_cpu_capability("FHM (deprecated sysctl)", kHasFeatFHM, true, "hw.optional.armv8_2_fhm", NULL);
418 test_cpu_capability("FHM", kHasFeatFHM, true, "hw.optional.arm.FEAT_FHM", try_fhm);
419 test_cpu_capability("SHA512", kHasFeatSHA512, true, "hw.optional.armv8_2_sha512", try_sha512);
420 test_cpu_capability("SHA3", kHasFeatSHA3, true, "hw.optional.armv8_2_sha3", try_sha3);
421 test_cpu_capability("AES", kHasFeatAES, true, "hw.optional.arm.FEAT_AES", try_aes);
422 test_cpu_capability("SHA1", kHasFeatSHA1, true, "hw.optional.arm.FEAT_SHA1", try_sha1);
423 test_cpu_capability("SHA256", kHasFeatSHA256, true, "hw.optional.arm.FEAT_SHA256", try_sha256);
424 test_cpu_capability("PMULL", kHasFeatPMULL, true, "hw.optional.arm.FEAT_PMULL", try_pmull);
425 test_cpu_capability("FCMA (deprecated sysctl)", kHasFeatFCMA, true, "hw.optional.armv8_3_compnum", NULL);
426 test_cpu_capability("FCMA", kHasFeatFCMA, true, "hw.optional.arm.FEAT_FCMA", try_compnum);
427 test_cpu_capability("FlagM", kHasFEATFlagM, true, "hw.optional.arm.FEAT_FlagM", try_flagm);
428 test_cpu_capability("FlagM2", kHasFEATFlagM2, true, "hw.optional.arm.FEAT_FlagM2", try_flagm2);
429 test_cpu_capability("DotProd", kHasFeatDotProd, true, "hw.optional.arm.FEAT_DotProd", try_dotprod);
430 test_cpu_capability("RDM", kHasFeatRDM, true, "hw.optional.arm.FEAT_RDM", try_rdm);
431 test_cpu_capability("SB", kHasFeatSB, true, "hw.optional.arm.FEAT_SB", try_sb);
432 test_cpu_capability("FRINTTS", kHasFeatFRINTTS, true, "hw.optional.arm.FEAT_FRINTTS", try_frintts);
433 test_cpu_capability("JSCVT", kHasFeatJSCVT, true, "hw.optional.arm.FEAT_JSCVT", try_jscvt);
434 test_cpu_capability("PAuth", kHasFeatPAuth, true, "hw.optional.arm.FEAT_PAuth", try_pauth);
435 test_cpu_capability("DBP", kHasFeatDPB, true, "hw.optional.arm.FEAT_DPB", try_dpb);
436 test_cpu_capability("DBP2", kHasFeatDPB2, true, "hw.optional.arm.FEAT_DPB2", try_dpb2);
437 test_cpu_capability("SPECRES", kHasFeatSPECRES, true, "hw.optional.arm.FEAT_SPECRES", try_specres);
438 test_cpu_capability("LRCPC", kHasFeatLRCPC, true, "hw.optional.arm.FEAT_LRCPC", try_lrcpc);
439 test_cpu_capability("LRCPC2", kHasFeatLRCPC2, true, "hw.optional.arm.FEAT_LRCPC2", try_lrcpc2);
440 test_cpu_capability("DIT", kHasFeatDIT, true, "hw.optional.arm.FEAT_DIT", try_dit);
441 test_cpu_capability("FP16", kHasFP_SyncExceptions, true, "hw.optional.arm.FP_SyncExceptions", try_fpexcp);
442
443 // The following features do not have a commpage entry
444 test_cpu_capability("BF16", 0, false, "hw.optional.arm.FEAT_BF16", try_bf16);
445 test_cpu_capability("I8MM", 0, false, "hw.optional.arm.FEAT_I8MM", try_i8mm);
446
447 // The following features do not add instructions or registers to test for the presence of
448 test_cpu_capability("LSE2", kHasFeatLSE2, true, "hw.optional.arm.FEAT_LSE2", NULL);
449 test_cpu_capability("CSV2", kHasFeatCSV2, true, "hw.optional.arm.FEAT_CSV2", NULL);
450 test_cpu_capability("CSV3", kHasFeatCSV3, true, "hw.optional.arm.FEAT_CSV3", NULL);
451 }
452