1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #if !(DEVELOPMENT || DEBUG)
30 #error "Testing is not enabled on RELEASE configurations"
31 #endif
32
33 #include <tests/xnupost.h>
34 #include <kern/kalloc.h>
35 #include <kern/clock.h>
36 #include <kern/thread.h>
37 #include <sys/random.h>
38
39 #define VFP_STATE_TEST_N_THREADS 4
40 #define VFP_STATE_TEST_N_REGS 8
41 #define VFP_STATE_TEST_N_ITER 100
42 #define VFP_STATE_TEST_DELAY_USEC 10000
43 #if __arm__
44 #define VFP_STATE_TEST_NZCV_SHIFT 28
45 #define VFP_STATE_TEST_NZCV_MAX 16
46 #else
47 #define VFP_STATE_TEST_RMODE_STRIDE_SHIFT 20
48 #define VFP_STATE_TEST_RMODE_STRIDE_MAX 16
49 #endif
50
51 #if __ARM_VFP__
52 extern kern_return_t vfp_state_test(void);
53
54 const uint64_t vfp_state_test_regs[VFP_STATE_TEST_N_REGS] = {
55 0x6a4cac4427ab5658, 0x51200e9ebbe0c9d1,
56 0xa94d20c2bbe367bc, 0xfee45035460927db,
57 0x64f3f1f7e93d019f, 0x02a625f02b890a40,
58 0xf5e42399d8480de8, 0xc38cdde520908d6b,
59 };
60
61 struct vfp_state_test_args {
62 uint64_t vfp_reg_rand;
63 #if __arm__
64 uint32_t fp_control_mask;
65 #else
66 uint64_t fp_control_mask;
67 #endif
68 int result;
69 int *start_barrier;
70 int *end_barrier;
71 };
72
73 static void
wait_threads(int * var,int num)74 wait_threads(
75 int* var,
76 int num)
77 {
78 if (var != NULL) {
79 while (os_atomic_load(var, acquire) != num) {
80 assert_wait((event_t) var, THREAD_UNINT);
81 if (os_atomic_load(var, acquire) != num) {
82 (void) thread_block(THREAD_CONTINUE_NULL);
83 } else {
84 clear_wait(current_thread(), THREAD_AWAKENED);
85 }
86 }
87 }
88 }
89
90 static void
wake_threads(int * var)91 wake_threads(
92 int* var)
93 {
94 if (var) {
95 os_atomic_inc(var, relaxed);
96 thread_wakeup((event_t) var);
97 }
98 }
99
100 static void
vfp_state_test_thread_routine(void * args,__unused wait_result_t wr)101 vfp_state_test_thread_routine(void *args, __unused wait_result_t wr)
102 {
103 struct vfp_state_test_args *vfp_state_test_args = (struct vfp_state_test_args *)args;
104 uint64_t *vfp_regs, *vfp_regs_expected;
105 int retval;
106 #if __arm__
107 uint32_t fp_control, fp_control_expected;
108 #else
109 uint64_t fp_control, fp_control_expected;
110 #endif
111
112 vfp_state_test_args->result = -1;
113
114 /* Allocate memory to store expected and actual VFP register values */
115 vfp_regs = kalloc_data(sizeof(vfp_state_test_regs),
116 Z_WAITOK | Z_NOFAIL);
117
118 vfp_regs_expected = kalloc_data(sizeof(vfp_state_test_regs),
119 Z_WAITOK | Z_NOFAIL);
120
121 /* Preload VFP registers with unique, per-thread patterns */
122 bcopy(vfp_state_test_regs, vfp_regs_expected, sizeof(vfp_state_test_regs));
123 for (int i = 0; i < VFP_STATE_TEST_N_REGS; i++) {
124 vfp_regs_expected[i] ^= vfp_state_test_args->vfp_reg_rand;
125 }
126
127 #if __arm__
128 asm volatile ("vldr d8, [%0, #0] \t\n vldr d9, [%0, #8] \t\n\
129 vldr d10, [%0, #16] \t\n vldr d11, [%0, #24] \t\n\
130 vldr d12, [%0, #32] \t\n vldr d13, [%0, #40] \t\n\
131 vldr d14, [%0, #48] \t\n vldr d15, [%0, #56]" \
132 : : "r"(vfp_regs_expected) : \
133 "memory", "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15");
134
135 /*
136 * Set FPSCR to a known value, so we can validate the save/restore path.
137 * Only touch NZCV flags, since 1) writing them does not have visible side-effects
138 * and 2) they're only set by the CPU as a result of executing an FP comparison,
139 * which do not exist in this function.
140 */
141 asm volatile ("fmrx %0, fpscr" : "=r"(fp_control_expected));
142 fp_control_expected |= vfp_state_test_args->fp_control_mask;
143 asm volatile ("fmxr fpscr, %0" : : "r"(fp_control_expected));
144 #else
145 asm volatile ("ldr d8, [%0, #0] \t\n ldr d9, [%0, #8] \t\n\
146 ldr d10, [%0, #16] \t\n ldr d11, [%0, #24] \t\n\
147 ldr d12, [%0, #32] \t\n ldr d13, [%0, #40] \t\n\
148 ldr d14, [%0, #48] \t\n ldr d15, [%0, #56]" \
149 : : "r"(vfp_regs_expected) : \
150 "memory", "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15");
151
152 asm volatile ("mrs %0, fpcr" : "=r"(fp_control_expected));
153 fp_control_expected |= vfp_state_test_args->fp_control_mask;
154 asm volatile ("msr fpcr, %0" : : "r"(fp_control_expected));
155 #endif
156
157 /* Make sure all threads start at roughly the same time */
158 wake_threads(vfp_state_test_args->start_barrier);
159 wait_threads(vfp_state_test_args->start_barrier, VFP_STATE_TEST_N_THREADS);
160
161 /* Check VFP registers against expected values, and go to sleep */
162 for (int i = 0; i < VFP_STATE_TEST_N_ITER; i++) {
163 bzero(vfp_regs, sizeof(vfp_state_test_regs));
164
165 #if __arm__
166 asm volatile ("vstr d8, [%0, #0] \t\n vstr d9, [%0, #8] \t\n\
167 vstr d10, [%0, #16] \t\n vstr d11, [%0, #24] \t\n\
168 vstr d12, [%0, #32] \t\n vstr d13, [%0, #40] \t\n\
169 vstr d14, [%0, #48] \t\n vstr d15, [%0, #56]" \
170 : : "r"(vfp_regs) : "memory");
171 asm volatile ("fmrx %0, fpscr" : "=r"(fp_control));
172 #else
173 asm volatile ("str d8, [%0, #0] \t\n str d9, [%0, #8] \t\n\
174 str d10, [%0, #16] \t\n str d11, [%0, #24] \t\n\
175 str d12, [%0, #32] \t\n str d13, [%0, #40] \t\n\
176 str d14, [%0, #48] \t\n str d15, [%0, #56]" \
177 : : "r"(vfp_regs) : "memory");
178 asm volatile ("mrs %0, fpcr" : "=r"(fp_control));
179 #endif
180
181 retval = bcmp(vfp_regs, vfp_regs_expected, sizeof(vfp_state_test_regs));
182 if ((retval != 0) || (fp_control != fp_control_expected)) {
183 goto vfp_state_thread_cmp_failure;
184 }
185
186 delay(VFP_STATE_TEST_DELAY_USEC);
187 }
188
189 vfp_state_test_args->result = 0;
190
191 vfp_state_thread_cmp_failure:
192 kfree_data(vfp_regs_expected, sizeof(vfp_state_test_regs));
193 kfree_data(vfp_regs, sizeof(vfp_state_test_regs));
194
195 /* Signal that the thread has finished, and terminate */
196 wake_threads(vfp_state_test_args->end_barrier);
197 thread_terminate_self();
198 }
199
200 /*
201 * This test spawns N threads that preload unique values into
202 * callee-saved VFP registers and then repeatedly check them
203 * for correctness after waking up from delay()
204 */
205 kern_return_t
vfp_state_test(void)206 vfp_state_test(void)
207 {
208 thread_t vfp_state_thread[VFP_STATE_TEST_N_THREADS];
209 struct vfp_state_test_args vfp_state_test_args[VFP_STATE_TEST_N_THREADS];
210 kern_return_t retval;
211 int start_barrier = 0, end_barrier = 0;
212
213 /* Spawn threads */
214 for (int i = 0; i < VFP_STATE_TEST_N_THREADS; i++) {
215 vfp_state_test_args[i].start_barrier = &start_barrier;
216 vfp_state_test_args[i].end_barrier = &end_barrier;
217 #if __arm__
218 vfp_state_test_args[i].fp_control_mask = (i % VFP_STATE_TEST_NZCV_MAX) << VFP_STATE_TEST_NZCV_SHIFT;
219 #else
220 vfp_state_test_args[i].fp_control_mask = (i % VFP_STATE_TEST_RMODE_STRIDE_MAX) << VFP_STATE_TEST_RMODE_STRIDE_SHIFT;
221 #endif
222 read_random(&vfp_state_test_args[i].vfp_reg_rand, sizeof(uint64_t));
223
224 retval = kernel_thread_start((thread_continue_t)vfp_state_test_thread_routine,
225 (void *)&vfp_state_test_args[i],
226 &vfp_state_thread[i]);
227
228 T_EXPECT((retval == KERN_SUCCESS), "thread %d started", i);
229 }
230
231 /* Wait for all threads to finish */
232 wait_threads(&end_barrier, VFP_STATE_TEST_N_THREADS);
233
234 /* Check if all threads completed successfully */
235 for (int i = 0; i < VFP_STATE_TEST_N_THREADS; i++) {
236 T_EXPECT((vfp_state_test_args[i].result == 0), "thread %d finished", i);
237 }
238
239 return KERN_SUCCESS;
240 }
241 #endif /* __ARM_VFP__ */
242