1 /*
2 * Copyright (c) 2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <darwintest.h>
30 #include <machine/cpu_capabilities.h>
31 #include <sys/sysctl.h>
32
33 #include "exc_helpers.h"
34
35 T_GLOBAL_META(
36 T_META_NAMESPACE("xnu.arm"),
37 T_META_RADAR_COMPONENT_NAME("xnu"),
38 T_META_RADAR_COMPONENT_VERSION("arm"),
39 T_META_OWNER("sdooher"),
40 T_META_RUN_CONCURRENTLY(true)
41 );
42
43 static volatile bool cap_usable;
44
45 static size_t
bad_instruction_handler(exception_type_t type __unused,mach_exception_data_t codes __unused)46 bad_instruction_handler(exception_type_t type __unused, mach_exception_data_t codes __unused)
47 {
48 cap_usable = false;
49 return 4;
50 }
51
52 static void
try_fp16(void)53 try_fp16(void)
54 {
55 asm volatile (
56 "fmov h0, #0" "\n"
57 :
58 :
59 : "v0"
60 );
61 }
62
63 static void
try_atomics(void)64 try_atomics(void)
65 {
66 uint64_t dword;
67 asm volatile (
68 "swp xzr, xzr, [%[dword]]"
69 :
70 : [dword]"r"(&dword)
71 );
72 }
73
74 static void
try_crc32(void)75 try_crc32(void)
76 {
77 asm volatile ( "crc32b wzr, wzr, wzr");
78 }
79
80 static void
try_fhm(void)81 try_fhm(void)
82 {
83 asm volatile (
84 "fmov d0, #0" "\n"
85 "fmlal v0.2s, v0.2h, v0.2h" "\n"
86 :
87 :
88 : "v0"
89 );
90 }
91
92 static void
try_sha512(void)93 try_sha512(void)
94 {
95 asm volatile (
96 "fmov d0, #0" "\n"
97 "fmov d1, #0" "\n"
98 "sha512h q0, q0, v0.2d" "\n"
99 :
100 :
101 : "v0"
102 );
103 }
104
105 static void
try_sha3(void)106 try_sha3(void)
107 {
108 asm volatile (
109 "fmov d0, #0" "\n"
110 "fmov d1, #0" "\n"
111 "eor3 v0.16b, v0.16b, v0.16b, v0.16b" "\n"
112 :
113 :
114 : "v0"
115 );
116 }
117
118 static void
try_sha1(void)119 try_sha1(void)
120 {
121 asm volatile (
122 "fmov s0, #0" "\n"
123 "sha1h s0, s0" "\n"
124 :
125 :
126 : "v0"
127 );
128 }
129
130 static void
try_pmull(void)131 try_pmull(void)
132 {
133 asm volatile (
134 "fmov d0, #0" "\n"
135 "pmull v0.1q, v0.1d, v0.1d" "\n"
136 :
137 :
138 : "v0"
139 );
140 }
141
142 static void
try_aes(void)143 try_aes(void)
144 {
145 asm volatile (
146 "fmov d0, #0" "\n"
147 "fmov d1, #0" "\n"
148 "aesd v0.16B, v0.16B" "\n"
149 :
150 :
151 : "v0"
152 );
153 }
154
155
156 static void
try_sha256(void)157 try_sha256(void)
158 {
159 asm volatile (
160 "fmov d0, #0" "\n"
161 "fmov d1, #0" "\n"
162 "sha256h q0, q0, v0.4s" "\n"
163 :
164 :
165 : "v0"
166 );
167 }
168
169
170 static void
try_compnum(void)171 try_compnum(void)
172 {
173 asm volatile (
174 "fmov d0, #0" "\n"
175 "fcadd v0.2s, v0.2s, v0.2s, #90" "\n"
176 :
177 :
178 : "v0"
179 );
180 }
181
182
183 static void
try_flagm(void)184 try_flagm(void)
185 {
186 asm volatile (
187 "cfinv" "\n"
188 "cfinv" "\n"
189 );
190 }
191
192 static void
try_flagm2(void)193 try_flagm2(void)
194 {
195 asm volatile (
196 "axflag" "\n"
197 "xaflag" "\n"
198 );
199 }
200
201 static void
try_dotprod(void)202 try_dotprod(void)
203 {
204 asm volatile (
205 "udot v0.4S,v1.16B,v2.16B"
206 :
207 :
208 : "v0"
209 );
210 }
211
212 static void
try_rdm(void)213 try_rdm(void)
214 {
215 asm volatile (
216 "sqrdmlah s0, s1, s2"
217 :
218 :
219 : "s0"
220 );
221 }
222
223 static void
try_sb(void)224 try_sb(void)
225 {
226 asm volatile (
227 "sb"
228 );
229 }
230
231 static void
try_frintts(void)232 try_frintts(void)
233 {
234 asm volatile (
235 "frint32x s0, s0"
236 :
237 :
238 : "s0"
239 );
240 }
241
242 static void
try_jscvt(void)243 try_jscvt(void)
244 {
245 asm volatile (
246 "fmov d0, #0" "\n"
247 "fjcvtzs w1, d0" "\n"
248 :
249 :
250 : "w1", "d0"
251 );
252 }
253
254 static void
try_pauth(void)255 try_pauth(void)
256 {
257 asm volatile (
258 "pacga x0, x0, x0"
259 :
260 :
261 : "x0"
262 );
263 }
264
265 static void
try_dpb(void)266 try_dpb(void)
267 {
268 int x;
269 asm volatile (
270 "dc cvap, %0"
271 :
272 : "r" (&x)
273 );
274 }
275
276 static void
try_dpb2(void)277 try_dpb2(void)
278 {
279 int x;
280 asm volatile (
281 "dc cvadp, %0"
282 :
283 : "r" (&x)
284 );
285 }
286
287 static void
try_lrcpc(void)288 try_lrcpc(void)
289 {
290 int x;
291 asm volatile (
292 "ldaprb w0, [%0]"
293 :
294 : "r" (&x)
295 : "w0"
296 );
297 }
298
299 static void
try_lrcpc2(void)300 try_lrcpc2(void)
301 {
302 int x;
303 asm volatile (
304 "ldapurb w0, [%0]"
305 :
306 : "r" (&x)
307 : "w0"
308 );
309 }
310
311
312 static void
try_specres(void)313 try_specres(void)
314 {
315 int x;
316 asm volatile (
317 "cfp rctx, %0"
318 :
319 : "r" (&x)
320 );
321 }
322
323 static void
try_bf16(void)324 try_bf16(void)
325 {
326 asm volatile (
327 "bfdot v0.4S,v1.8H,v2.8H"
328 :
329 :
330 : "v0"
331 );
332 }
333
334 static void
try_i8mm(void)335 try_i8mm(void)
336 {
337 asm volatile (
338 "sudot v0.4S,v1.16B,v2.4B[0]"
339 :
340 :
341 : "v0"
342 );
343 }
344
345 static mach_port_t exc_port;
346
347 static void
test_cpu_capability(const char * cap_name,uint64_t cap_flag,bool has_commpage_entry,const char * cap_sysctl,void (* try_cpu_capability)(void))348 test_cpu_capability(const char *cap_name, uint64_t cap_flag, bool has_commpage_entry, const char *cap_sysctl, void (*try_cpu_capability)(void))
349 {
350 uint64_t caps = _get_cpu_capabilities();
351 bool has_cap_flag = (caps & cap_flag);
352
353 int sysctl_val;
354 bool has_sysctl_flag = 0;
355 if (cap_sysctl != NULL) {
356 size_t sysctl_size = sizeof(sysctl_val);
357 int err = sysctlbyname(cap_sysctl, &sysctl_val, &sysctl_size, NULL, 0);
358 has_sysctl_flag = (err == 0 && sysctl_val > 0);
359 }
360
361 bool has_capability = has_commpage_entry ? has_cap_flag : has_sysctl_flag;
362
363 if (!has_commpage_entry && cap_sysctl == NULL) {
364 T_FAIL("Tested capability must have either sysctl or commpage flag");
365 return;
366 }
367
368 if (has_commpage_entry && cap_sysctl != NULL) {
369 T_EXPECT_EQ(has_cap_flag, has_sysctl_flag, "%s commpage flag matches sysctl flag", cap_name);
370 }
371
372 if (try_cpu_capability != NULL) {
373 cap_usable = true;
374 try_cpu_capability();
375 T_EXPECT_EQ(has_capability, cap_usable, "%s capability matches actual usability", cap_name);
376 }
377 }
378
379 T_DECL(cpu_capabilities, "Verify ARM CPU capabilities") {
380 exc_port = create_exception_port(EXC_MASK_BAD_INSTRUCTION);
381 repeat_exception_handler(exc_port, bad_instruction_handler);
382
383 test_cpu_capability("FP16 (deprecated sysctl)", kHasFeatFP16, true, "hw.optional.neon_fp16", NULL);
384 test_cpu_capability("FP16", kHasFeatFP16, true, "hw.optional.arm.FEAT_FP16", try_fp16);
385 test_cpu_capability("LSE (deprecated sysctl)", kHasFeatLSE, true, "hw.optional.armv8_1_atomics", NULL);
386 test_cpu_capability("LSE", kHasFeatLSE, true, "hw.optional.arm.FEAT_LSE", try_atomics);
387 test_cpu_capability("CRC32", kHasARMv8Crc32, true, "hw.optional.armv8_crc32", try_crc32);
388 test_cpu_capability("FHM (deprecated sysctl)", kHasFeatFHM, true, "hw.optional.armv8_2_fhm", NULL);
389 test_cpu_capability("FHM", kHasFeatFHM, true, "hw.optional.arm.FEAT_FHM", try_fhm);
390 test_cpu_capability("SHA512", kHasFeatSHA512, true, "hw.optional.armv8_2_sha512", try_sha512);
391 test_cpu_capability("SHA3", kHasFeatSHA3, true, "hw.optional.armv8_2_sha3", try_sha3);
392 test_cpu_capability("AES", kHasFeatAES, true, "hw.optional.arm.FEAT_AES", try_aes);
393 test_cpu_capability("SHA1", kHasFeatSHA1, true, "hw.optional.arm.FEAT_SHA1", try_sha1);
394 test_cpu_capability("SHA256", kHasFeatSHA256, true, "hw.optional.arm.FEAT_SHA256", try_sha256);
395 test_cpu_capability("PMULL", kHasFeatPMULL, true, "hw.optional.arm.FEAT_PMULL", try_pmull);
396 test_cpu_capability("FCMA (deprecatded sysctl)", kHasFeatFCMA, true, "hw.optional.armv8_3_compnum", NULL);
397 test_cpu_capability("FCMA", kHasFeatFCMA, true, "hw.optional.arm.FEAT_FCMA", try_compnum);
398 test_cpu_capability("FlagM", kHasFEATFlagM, true, "hw.optional.arm.FEAT_FlagM", try_flagm);
399 test_cpu_capability("FlagM2", kHasFEATFlagM2, true, "hw.optional.arm.FEAT_FlagM2", try_flagm2);
400 test_cpu_capability("DotProd", kHasFeatDotProd, true, "hw.optional.arm.FEAT_DotProd", try_dotprod);
401 test_cpu_capability("RDM", kHasFeatRDM, true, "hw.optional.arm.FEAT_RDM", try_rdm);
402 test_cpu_capability("SB", kHasFeatSB, true, "hw.optional.arm.FEAT_SB", try_sb);
403 test_cpu_capability("FRINTTS", kHasFeatFRINTTS, true, "hw.optional.arm.FEAT_FRINTTS", try_frintts);
404 test_cpu_capability("JSCVT", kHasFeatJSCVT, true, "hw.optional.arm.FEAT_JSCVT", try_jscvt);
405 test_cpu_capability("PAuth", kHasFeatPAuth, true, "hw.optional.arm.FEAT_PAuth", try_pauth);
406 test_cpu_capability("DBP", kHasFeatDPB, true, "hw.optional.arm.FEAT_DPB", try_dpb);
407 test_cpu_capability("DBP2", kHasFeatDPB2, true, "hw.optional.arm.FEAT_DPB2", try_dpb2);
408 test_cpu_capability("SPECRES", kHasFeatSPECRES, true, "hw.optional.arm.FEAT_SPECRES", try_specres);
409 test_cpu_capability("LRCPC", kHasFeatLRCPC, true, "hw.optional.arm.FEAT_LRCPC", try_lrcpc);
410 test_cpu_capability("LRCPC2", kHasFeatLRCPC2, true, "hw.optional.arm.FEAT_LRCPC2", try_lrcpc2);
411
412 test_cpu_capability("BF16", 0, false, "hw.optional.arm.FEAT_BF16", try_bf16);
413 test_cpu_capability("I8MM", 0, false, "hw.optional.arm.FEAT_I8MM", try_i8mm);
414
415 // The following features do not add instructions
416 test_cpu_capability("LSE2", kHasFeatLSE2, true, "hw.optional.arm.FEAT_LSE2", NULL);
417 test_cpu_capability("CSV2", kHasFeatCSV2, true, "hw.optional.arm.FEAT_CSV2", NULL);
418 test_cpu_capability("CSV3", kHasFeatCSV3, true, "hw.optional.arm.FEAT_CSV3", NULL);
419 }
420