1 /*
2 * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #if !(DEVELOPMENT || DEBUG)
30 #error "Testing is not enabled on RELEASE configurations"
31 #endif
32
33 #if defined(__arm64__)
34 #include <tests/xnupost.h>
35 #include <kern/kalloc.h>
36 #include <kern/clock.h>
37 #include <kern/thread.h>
38 #include <sys/random.h>
39
40 #define VFP_STATE_TEST_N_THREADS 4
41 #define VFP_STATE_TEST_N_REGS 8
42 #define VFP_STATE_TEST_N_ITER 100
43 #define VFP_STATE_TEST_DELAY_USEC 10000
44 #define VFP_STATE_TEST_RMODE_STRIDE_SHIFT 20
45 #define VFP_STATE_TEST_RMODE_STRIDE_MAX 16
46
47 extern kern_return_t vfp_state_test(void);
48
49 const uint64_t vfp_state_test_regs[VFP_STATE_TEST_N_REGS] = {
50 0x6a4cac4427ab5658, 0x51200e9ebbe0c9d1,
51 0xa94d20c2bbe367bc, 0xfee45035460927db,
52 0x64f3f1f7e93d019f, 0x02a625f02b890a40,
53 0xf5e42399d8480de8, 0xc38cdde520908d6b,
54 };
55
56 struct vfp_state_test_args {
57 uint64_t vfp_reg_rand;
58 uint64_t fp_control_mask;
59
60 int result;
61 int *start_barrier;
62 int *end_barrier;
63 };
64
65 static void
wait_threads(int * var,int num)66 wait_threads(
67 int* var,
68 int num)
69 {
70 if (var != NULL) {
71 while (os_atomic_load(var, acquire) != num) {
72 assert_wait((event_t) var, THREAD_UNINT);
73 if (os_atomic_load(var, acquire) != num) {
74 (void) thread_block(THREAD_CONTINUE_NULL);
75 } else {
76 clear_wait(current_thread(), THREAD_AWAKENED);
77 }
78 }
79 }
80 }
81
82 static void
wake_threads(int * var)83 wake_threads(
84 int* var)
85 {
86 if (var) {
87 os_atomic_inc(var, relaxed);
88 thread_wakeup((event_t) var);
89 }
90 }
91
92 static void
vfp_state_test_thread_routine(void * args,__unused wait_result_t wr)93 vfp_state_test_thread_routine(void *args, __unused wait_result_t wr)
94 {
95 struct vfp_state_test_args *vfp_state_test_args = (struct vfp_state_test_args *)args;
96 uint64_t *vfp_regs, *vfp_regs_expected;
97 int retval;
98 uint64_t fp_control, fp_control_expected;
99
100
101 vfp_state_test_args->result = -1;
102
103 /* Allocate memory to store expected and actual VFP register values */
104 vfp_regs = kalloc_data(sizeof(vfp_state_test_regs),
105 Z_WAITOK | Z_NOFAIL);
106
107 vfp_regs_expected = kalloc_data(sizeof(vfp_state_test_regs),
108 Z_WAITOK | Z_NOFAIL);
109
110 /* Preload VFP registers with unique, per-thread patterns */
111 bcopy(vfp_state_test_regs, vfp_regs_expected, sizeof(vfp_state_test_regs));
112 for (int i = 0; i < VFP_STATE_TEST_N_REGS; i++) {
113 vfp_regs_expected[i] ^= vfp_state_test_args->vfp_reg_rand;
114 }
115
116 asm volatile ("ldr d8, [%0, #0] \t\n ldr d9, [%0, #8] \t\n\
117 ldr d10, [%0, #16] \t\n ldr d11, [%0, #24] \t\n\
118 ldr d12, [%0, #32] \t\n ldr d13, [%0, #40] \t\n\
119 ldr d14, [%0, #48] \t\n ldr d15, [%0, #56]" \
120 : : "r"(vfp_regs_expected) : \
121 "memory", "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15");
122
123 asm volatile ("mrs %0, fpcr" : "=r"(fp_control_expected));
124 fp_control_expected |= vfp_state_test_args->fp_control_mask;
125 asm volatile ("msr fpcr, %0" : : "r"(fp_control_expected));
126
127 /* Make sure all threads start at roughly the same time */
128 wake_threads(vfp_state_test_args->start_barrier);
129 wait_threads(vfp_state_test_args->start_barrier, VFP_STATE_TEST_N_THREADS);
130
131 /* Check VFP registers against expected values, and go to sleep */
132 for (int i = 0; i < VFP_STATE_TEST_N_ITER; i++) {
133 bzero(vfp_regs, sizeof(vfp_state_test_regs));
134
135 asm volatile ("str d8, [%0, #0] \t\n str d9, [%0, #8] \t\n\
136 str d10, [%0, #16] \t\n str d11, [%0, #24] \t\n\
137 str d12, [%0, #32] \t\n str d13, [%0, #40] \t\n\
138 str d14, [%0, #48] \t\n str d15, [%0, #56]" \
139 : : "r"(vfp_regs) : "memory");
140 asm volatile ("mrs %0, fpcr" : "=r"(fp_control));
141
142 retval = bcmp(vfp_regs, vfp_regs_expected, sizeof(vfp_state_test_regs));
143 if ((retval != 0) || (fp_control != fp_control_expected)) {
144 goto vfp_state_thread_cmp_failure;
145 }
146
147 delay(VFP_STATE_TEST_DELAY_USEC);
148 }
149
150 vfp_state_test_args->result = 0;
151
152 vfp_state_thread_cmp_failure:
153 kfree_data(vfp_regs_expected, sizeof(vfp_state_test_regs));
154 kfree_data(vfp_regs, sizeof(vfp_state_test_regs));
155
156 /* Signal that the thread has finished, and terminate */
157 wake_threads(vfp_state_test_args->end_barrier);
158 thread_terminate_self();
159 }
160
161 /*
162 * This test spawns N threads that preload unique values into
163 * callee-saved VFP registers and then repeatedly check them
164 * for correctness after waking up from delay()
165 */
166 kern_return_t
vfp_state_test(void)167 vfp_state_test(void)
168 {
169 thread_t vfp_state_thread[VFP_STATE_TEST_N_THREADS];
170 struct vfp_state_test_args vfp_state_test_args[VFP_STATE_TEST_N_THREADS];
171 kern_return_t retval;
172 int start_barrier = 0, end_barrier = 0;
173
174 /* Spawn threads */
175 for (int i = 0; i < VFP_STATE_TEST_N_THREADS; i++) {
176 vfp_state_test_args[i].start_barrier = &start_barrier;
177 vfp_state_test_args[i].end_barrier = &end_barrier;
178 vfp_state_test_args[i].fp_control_mask = (i % VFP_STATE_TEST_RMODE_STRIDE_MAX) << VFP_STATE_TEST_RMODE_STRIDE_SHIFT;
179 read_random(&vfp_state_test_args[i].vfp_reg_rand, sizeof(uint64_t));
180
181 retval = kernel_thread_start((thread_continue_t)vfp_state_test_thread_routine,
182 (void *)&vfp_state_test_args[i],
183 &vfp_state_thread[i]);
184
185 T_EXPECT((retval == KERN_SUCCESS), "thread %d started", i);
186 }
187
188 /* Wait for all threads to finish */
189 wait_threads(&end_barrier, VFP_STATE_TEST_N_THREADS);
190
191 /* Check if all threads completed successfully */
192 for (int i = 0; i < VFP_STATE_TEST_N_THREADS; i++) {
193 T_EXPECT((vfp_state_test_args[i].result == 0), "thread %d finished", i);
194 }
195
196 return KERN_SUCCESS;
197 }
198 #endif /* defined(__arm64__) */
199