1 /* * Copyright (c) 2020 Apple Inc. All rights reserved.
2 *
3 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
4 *
5 * This file contains Original Code and/or Modifications of Original Code
6 * as defined in and that are subject to the Apple Public Source License
7 * Version 2.0 (the 'License'). You may not use this file except in
8 * compliance with the License. The rights granted to you under the License
9 * may not be used to create, or enable the creation or redistribution of,
10 * unlawful or unlicensed copies of an Apple operating system, or to
11 * circumvent, violate, or enable the circumvention or violation of, any
12 * terms of an Apple operating system software license agreement.
13 *
14 * Please obtain a copy of the License at
15 * http://www.opensource.apple.com/apsl/ and read it before using this file.
16 *
17 * The Original Code and all software distributed under the License are
18 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
19 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
20 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
22 * Please see the License for the specific language governing rights and
23 * limitations under the License.
24 *
25 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
26 */
27
28 /* sysctl interface for testing percpu counters in DEBUG or DEVELOPMENT kernel only. */
29 #if !(DEVELOPMENT || DEBUG)
30 #error "Counter testing is not enabled on RELEASE configurations"
31 #endif
32
33 #include <sys/sysctl.h>
34 #include <kern/counter.h>
35 #include <machine/atomic.h>
36 #include <libkern/libkern.h>
37 #include <machine/machine_routines.h>
38 #include <kern/cpu_data.h>
39
40 #include <os/log.h>
41
42 #ifdef CONFIG_XNUPOST
43 #include <tests/xnupost.h>
44 #endif /* CONFIG_XNUPOST */
45
46 static _Atomic boolean_t scalable_counter_test_running = FALSE;
47 scalable_counter_t test_scalable_counter;
48
49 SCALABLE_COUNTER_DEFINE(test_static_scalable_counter);
50
51 #ifdef CONFIG_XNUPOST
52 kern_return_t counter_tests(void);
53 /*
54 * Sanity test that a counter can be modified before zalloc is initialized.
55 */
56 static void
bump_static_counter(void * arg)57 bump_static_counter(void* arg)
58 {
59 (void) arg;
60 counter_inc(&test_static_scalable_counter);
61 }
62
63 STARTUP_ARG(PMAP_STEAL, STARTUP_RANK_MIDDLE, bump_static_counter, NULL);
64
65 kern_return_t
counter_tests()66 counter_tests()
67 {
68 T_ASSERT_EQ_ULLONG(counter_load(&test_static_scalable_counter), 1, "Counter was incremented");
69 return KERN_SUCCESS;
70 }
71 #endif /* CONFIG_XNUPOST */
72
73 static int
74 sysctl_scalable_counter_test_start SYSCTL_HANDLER_ARGS
75 {
76 #pragma unused(oidp, arg1, arg2)
77 int ret_val = 1;
78 int error = 0;
79 boolean_t exclusive;
80 error = sysctl_io_number(req, ret_val, sizeof(int), &ret_val, NULL);
81 if (error || !req->newptr) {
82 return error;
83 }
84 /* The test doesn't support being run multiple times in parallel. */
85 exclusive = os_atomic_cmpxchg(&scalable_counter_test_running, FALSE, TRUE, seq_cst);
86 if (!exclusive) {
87 os_log(OS_LOG_DEFAULT, "scalable_counter_test: Caught attempt to run the test in parallel.");
88 return EINVAL;
89 }
90 counter_alloc(&test_scalable_counter);
91 return 0;
92 }
93
94 static int
95 sysctl_scalable_counter_test_finish SYSCTL_HANDLER_ARGS
96 {
97 #pragma unused(oidp, arg1, arg2)
98 boolean_t exclusive;
99 int ret_val = 0;
100 int error = 0;
101 error = sysctl_io_number(req, ret_val, sizeof(int), &ret_val, NULL);
102 if (error || !req->newptr) {
103 return error;
104 }
105
106 /* The test doesn't support being run multiple times in parallel. */
107 exclusive = os_atomic_cmpxchg(&scalable_counter_test_running, TRUE, FALSE, seq_cst);
108 if (!exclusive) {
109 /* Finish called without start. */
110 return EINVAL;
111 }
112 return 0;
113 }
114
115 static int
116 sysctl_scalable_counter_add SYSCTL_HANDLER_ARGS
117 {
118 #pragma unused(oidp, arg1, arg2)
119 int64_t value = 0;
120 int error = 0;
121 if (!os_atomic_load(&scalable_counter_test_running, seq_cst)) {
122 /* Must call start */
123 return EINVAL;
124 }
125 error = sysctl_io_number(req, value, sizeof(int64_t), &value, NULL);
126 if (error || !req->newptr) {
127 return error;
128 }
129 counter_add(&test_scalable_counter, value);
130 return 0;
131 }
132
133 static int
134 sysctl_static_scalable_counter_add SYSCTL_HANDLER_ARGS
135 {
136 #pragma unused(oidp, arg1, arg2)
137 int64_t value = 0;
138 int error = 0;
139 if (!os_atomic_load(&scalable_counter_test_running, seq_cst)) {
140 /* Must call start */
141 return EINVAL;
142 }
143 error = sysctl_io_number(req, value, sizeof(int64_t), &value, NULL);
144 if (error || !req->newptr) {
145 return error;
146 }
147 counter_add(&test_static_scalable_counter, value);
148 return 0;
149 }
150
151 static int
152 sysctl_scalable_counter_load SYSCTL_HANDLER_ARGS
153 {
154 #pragma unused(oidp, arg1, arg2)
155 uint64_t value;
156 if (!os_atomic_load(&scalable_counter_test_running, seq_cst)) {
157 /* Must call start */
158 return EINVAL;
159 }
160 value = counter_load(&test_scalable_counter);
161 return SYSCTL_OUT(req, &value, sizeof(value));
162 }
163
164 static int
165 sysctl_scalable_counter_write_benchmark SYSCTL_HANDLER_ARGS
166 {
167 #pragma unused(oidp, arg1, arg2)
168 int error;
169 int64_t iterations;
170 int ret_val = 0;
171 if (!os_atomic_load(&scalable_counter_test_running, seq_cst)) {
172 /* Must call start */
173 return EINVAL;
174 }
175 error = sysctl_io_number(req, ret_val, sizeof(int), &iterations, NULL);
176 if (error || !req->newptr) {
177 return error;
178 }
179 for (int64_t i = 0; i < iterations; i++) {
180 counter_inc(&test_scalable_counter);
181 }
182 return 0;
183 }
184
185 static volatile uint64_t racy_counter;
186
187 static int
188 sysctl_racy_counter_write_benchmark SYSCTL_HANDLER_ARGS
189 {
190 #pragma unused(oidp, arg1, arg2)
191 int error;
192 int64_t iterations;
193 int ret_val = 0;
194 error = sysctl_io_number(req, ret_val, sizeof(int), &iterations, NULL);
195 if (error || !req->newptr) {
196 return error;
197 }
198 for (int64_t i = 0; i < iterations; i++) {
199 racy_counter++;
200 }
201 return 0;
202 }
203
204 static int
205 sysctl_racy_counter_load SYSCTL_HANDLER_ARGS
206 {
207 #pragma unused(oidp, arg1, arg2)
208 uint64_t value = racy_counter;
209 return SYSCTL_OUT(req, &value, sizeof(value));
210 }
211
212 static _Atomic uint64_t atomic_counter;
213
214 static int
215 sysctl_atomic_counter_write_benchmark SYSCTL_HANDLER_ARGS
216 {
217 #pragma unused(oidp, arg1, arg2)
218 int error;
219 int64_t iterations;
220 int ret_val = 0;
221 error = sysctl_io_number(req, ret_val, sizeof(int), &iterations, NULL);
222 if (error || !req->newptr) {
223 return error;
224 }
225 for (int64_t i = 0; i < iterations; i++) {
226 os_atomic_add(&atomic_counter, 1, relaxed);
227 }
228 return 0;
229 }
230
231 static int
232 sysctl_atomic_counter_load SYSCTL_HANDLER_ARGS
233 {
234 #pragma unused(oidp, arg1, arg2)
235 uint64_t value = os_atomic_load_wide(&atomic_counter, relaxed);
236 return SYSCTL_OUT(req, &value, sizeof(value));
237 }
238
239 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_test_start,
240 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
241 0, 0, sysctl_scalable_counter_test_start, "I", "Setup per-cpu counter test");
242
243 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_test_finish,
244 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
245 0, 0, sysctl_scalable_counter_test_finish, "I", "Finish per-cpu counter test");
246
247 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_test_add,
248 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
249 0, 0, sysctl_scalable_counter_add, "I", "Perform an add on the per-cpu counter");
250
251 SYSCTL_PROC(_kern, OID_AUTO, static_scalable_counter_test_add,
252 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
253 0, 0, sysctl_static_scalable_counter_add, "I", "Perform an add on the static per-cpu counter");
254
255 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_test_load,
256 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
257 0, 0, sysctl_scalable_counter_load, "I", "Load the current per-cpu counter value.");
258
259 SYSCTL_SCALABLE_COUNTER(_kern, static_scalable_counter_test_load,
260 test_static_scalable_counter, "Load the current static per-cpu counter value.");
261
262 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_write_benchmark,
263 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
264 0, 0, sysctl_scalable_counter_write_benchmark, "I", "Per-cpu counter write benchmark");
265
266 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_racy_counter_benchmark,
267 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
268 0, 0, sysctl_racy_counter_write_benchmark, "I", "Global counter racy benchmark");
269
270 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_racy_counter_load,
271 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
272 0, 0, sysctl_racy_counter_load, "I", "Global counter racy load");
273
274 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_atomic_counter_write_benchmark,
275 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
276 0, 0, sysctl_atomic_counter_write_benchmark, "I", "Atomic counter write benchmark");
277
278 SYSCTL_PROC(_kern, OID_AUTO, scalable_counter_atomic_counter_load,
279 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
280 0, 0, sysctl_atomic_counter_load, "I", "Atomic counter load");
281