1 // Copyright (c) 2021-2022 Apple Inc. All rights reserved.
2
3 #include <darwintest.h>
4 #include <darwintest_utils.h>
5 #include <dispatch/dispatch.h>
6 #include <mach/semaphore.h>
7 #include <mach/mach.h>
8 #include <mach/task.h>
9 #include <mach-o/dyld.h>
10 #include <stdbool.h>
11 #include <stdlib.h>
12 #include <sys/sysctl.h>
13 #include <unistd.h>
14
15 #include "recount_test_utils.h"
16
17 static bool may_fail_status = false;
18
19 void
set_expects_may_fail(bool may_fail)20 set_expects_may_fail(bool may_fail)
21 {
22 may_fail_status = may_fail;
23 }
24
25 bool
expects_may_fail(void)26 expects_may_fail(void)
27 {
28 return may_fail_status;
29 }
30
31 bool
has_user_system_times(void)32 has_user_system_times(void)
33 {
34 static dispatch_once_t user_system_once;
35 static bool precise_times = false;
36 dispatch_once(&user_system_once, ^{
37 int precise_times_int = 0;
38 size_t precise_times_size = sizeof(precise_times_int);
39 T_SETUPBEGIN;
40 T_QUIET;
41 T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.precise_user_kernel_time",
42 &precise_times_int, &precise_times_size, NULL, 0),
43 "sysctl kern.precise_user_kernel_time");
44 T_SETUPEND;
45 precise_times = precise_times_int != 0;
46 });
47 return precise_times;
48 }
49
50 bool
has_cpi(void)51 has_cpi(void)
52 {
53 static dispatch_once_t cpi_once;
54 static int cpi = 0;
55 dispatch_once(&cpi_once, ^{
56 size_t cpi_size = sizeof(cpi);
57 T_SETUPBEGIN;
58 int ret = sysctlbyname("kern.monotonic.supported", &cpi, &cpi_size,
59 NULL, 0);
60 // ENOENT also means that CPI is unavailable.
61 if (ret != 0 && errno != ENOENT) {
62 T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.monotonic.supported");
63 }
64 T_SETUPEND;
65 });
66 return cpi != 0;
67 }
68
69 bool
has_energy(void)70 has_energy(void)
71 {
72 static dispatch_once_t energy_once;
73 static int energy = false;
74 dispatch_once(&energy_once, ^{
75 size_t energy_size = sizeof(energy);
76 T_SETUPBEGIN;
77 T_QUIET;
78 T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.pervasive_energy",
79 &energy, &energy_size, NULL, 0),
80 "sysctl kern.pervasive_energy");
81 T_SETUPEND;
82 });
83 return energy != 0;
84 }
85
86 unsigned int
perf_level_count(void)87 perf_level_count(void)
88 {
89 static dispatch_once_t count_once;
90 static unsigned int count = 0;
91 dispatch_once(&count_once, ^{
92 T_SETUPBEGIN;
93 T_QUIET;
94 T_ASSERT_POSIX_SUCCESS(sysctlbyname("hw.nperflevels", &count,
95 &(size_t){ sizeof(count) }, NULL, 0),
96 "sysctl hw.nperflevels");
97 T_SETUPEND;
98 });
99 return count;
100 }
101
102 static const char **
_perf_level_names(void)103 _perf_level_names(void)
104 {
105 static char names[2][32] = { 0 };
106 static dispatch_once_t names_once;
107 dispatch_once(&names_once, ^{
108 T_SETUPBEGIN;
109 unsigned int count = perf_level_count();
110 for (unsigned int i = 0; i < count; i++) {
111 char sysctl_name[64] = { 0 };
112 snprintf(sysctl_name, sizeof(sysctl_name), "hw.perflevel%d.name",
113 i);
114 T_QUIET;
115 T_ASSERT_POSIX_SUCCESS(sysctlbyname(sysctl_name, &names[i],
116 &(size_t){ sizeof(names[i]) }, NULL, 0),
117 "sysctl %s", sysctl_name);
118 }
119 T_SETUPEND;
120 });
121 static const char *ret_names[] = {
122 (char *)&names[0],
123 (char *)&names[1],
124 };
125 return ret_names;
126 }
127
128 const char *
perf_level_name(unsigned int perf_level)129 perf_level_name(unsigned int perf_level)
130 {
131 return _perf_level_names()[perf_level];
132 }
133
134 unsigned int
perf_level_index(const char * name)135 perf_level_index(const char *name)
136 {
137 unsigned int count = perf_level_count();
138 const char **names = _perf_level_names();
139 for (unsigned int i = 0; i < count; i++) {
140 if (strcmp(name, names[i]) == 0) {
141 return i;
142 }
143 }
144 T_ASSERT_FAIL("cannot find perf level named %s", name);
145 }
146
147 void
bind_to_cluster(char type)148 bind_to_cluster(char type)
149 {
150 int ret = sysctlbyname("kern.sched_thread_bind_cluster_type", NULL, NULL,
151 &type, sizeof(type));
152 T_QUIET;
153 T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.sched_thread_bind_cluster_type");
154 // Do a little spin to tighten the odds that we get CPU time on the bound
155 // cluster--needed for devices running the AMP scheduler.
156 volatile int count = 0;
157 while (count++ < 100000);
158 }
159
160 char *
sched_policy_name(void)161 sched_policy_name(void)
162 {
163 static char policy_name[64] = { 0 };
164 static dispatch_once_t policy_name_once;
165 dispatch_once(&policy_name_once, ^{
166 T_SETUPBEGIN;
167 T_QUIET;
168 T_ASSERT_POSIX_SUCCESS(sysctlbyname("kern.sched", &policy_name,
169 &(size_t){ sizeof(policy_name) }, NULL, 0), "kern.sched");
170 T_SETUPEND;
171 });
172 return policy_name;
173 }
174
175 static void
_unbind_from_cluster(void)176 _unbind_from_cluster(void)
177 {
178 // Best effort.
179 (void)sysctlbyname("kern.sched_thread_bind_cluster_type", NULL, NULL, NULL,
180 0);
181 }
182
183 void
run_on_all_perf_levels(void)184 run_on_all_perf_levels(void)
185 {
186 if (perf_level_count() == 1) {
187 return;
188 }
189
190 T_SETUPBEGIN;
191 for (unsigned int i = 0; i < perf_level_count(); i++) {
192 bind_to_cluster(perf_level_name(i)[0]);
193 }
194 // Return to the kernel to synchronize timings with the scheduler.
195 (void)getppid();
196 _unbind_from_cluster();
197 T_SETUPEND;
198 }
199
200 static void
_run_on_exclaves(void)201 _run_on_exclaves(void)
202 {
203 int64_t output = 0;
204 size_t output_size = sizeof(output);
205 int64_t input = 0;
206 int ret = sysctlbyname("debug.test.exclaves_hello_exclave_test", &output,
207 &output_size, &input, sizeof(input));
208 T_QUIET;
209 T_ASSERT_POSIX_SUCCESS(ret, "systcl debug.test.exclaves_hello_exclave_test");
210 }
211
212 void
run_in_exclaves_on_all_perf_levels(void)213 run_in_exclaves_on_all_perf_levels(void)
214 {
215 if (perf_level_count() == 1) {
216 _run_on_exclaves();
217 return;
218 }
219
220 T_SETUPBEGIN;
221 for (unsigned int i = 0; i < perf_level_count(); i++) {
222 bind_to_cluster(perf_level_name(i)[0]);
223 _run_on_exclaves();
224 }
225 _unbind_from_cluster();
226 T_SETUPEND;
227 }
228
229 uint64_t
ns_from_mach(uint64_t mach_time)230 ns_from_mach(uint64_t mach_time)
231 {
232 mach_timebase_info_data_t tbi = { 0 };
233 mach_timebase_info(&tbi);
234 return mach_time * tbi.numer / tbi.denom;
235 }
236
237 uint64_t
ns_from_timeval(struct timeval tv)238 ns_from_timeval(struct timeval tv)
239 {
240 return (uint64_t)tv.tv_sec * NSEC_PER_SEC + (uint64_t)tv.tv_usec * 1000;
241 }
242
243 struct timeval
timeval_from_ns(uint64_t ns)244 timeval_from_ns(uint64_t ns)
245 {
246 return (struct timeval){
247 .tv_sec = ns / NSEC_PER_SEC,
248 .tv_usec = (ns % NSEC_PER_SEC) / 1000,
249 };
250 }
251
252 uint64_t
ns_from_time_value(struct time_value tv)253 ns_from_time_value(struct time_value tv)
254 {
255 return (uint64_t)tv.seconds * NSEC_PER_SEC +
256 (uint64_t)tv.microseconds * 1000;
257 }
258
259 struct time_value
time_value_from_ns(uint64_t ns)260 time_value_from_ns(uint64_t ns)
261 {
262 return (struct time_value){
263 .seconds = (integer_t)(ns / NSEC_PER_SEC),
264 .microseconds = (ns % NSEC_PER_SEC) / 1000,
265 };
266 }
267
268 static void *
spin_role(void * arg)269 spin_role(void *arg)
270 {
271 volatile uintptr_t *keep_spinning = arg;
272 while (*keep_spinning) {
273 ;
274 }
275 return NULL;
276 }
277
278 struct wait_start {
279 semaphore_t ws_wait;
280 semaphore_t ws_start;
281 };
282
283 static void *
wait_role(void * arg)284 wait_role(void *arg)
285 {
286 struct wait_start *ws = arg;
287 semaphore_wait_signal(ws->ws_wait, ws->ws_start);
288 return NULL;
289 }
290
291 struct scene *
scene_start(unsigned int n,role_t * roles)292 scene_start(unsigned int n, role_t *roles)
293 {
294 if (n == 0) {
295 return NULL;
296 }
297
298 T_SETUPBEGIN;
299
300 size_t scene_size = sizeof(struct scene) + (n + 1) * sizeof(struct actor);
301 struct scene *scene = malloc(scene_size);
302 T_QUIET; T_WITH_ERRNO;
303 T_ASSERT_NOTNULL(scene, "scene = malloc(%zu)", scene_size);
304
305 bzero(scene, scene_size);
306 unsigned int role_i = 0;
307 unsigned int wait_count = 0;
308 for (unsigned int i = 0; i < n; i++) {
309 role_t role = roles[role_i];
310 if (role == ROLE_NONE) {
311 role_i = 0;
312 role = roles[role_i];
313 }
314 if (role == ROLE_WAIT) {
315 wait_count++;
316 }
317 scene->scn_actors[i].act_role = role;
318 role_i++;
319 }
320
321 struct wait_start ws = { 0 };
322 kern_return_t kr = semaphore_create(mach_task_self(), &ws.ws_wait,
323 SYNC_POLICY_FIFO, 0);
324 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_create(... wait ...)");
325 kr = semaphore_create(mach_task_self(), &ws.ws_start,
326 SYNC_POLICY_FIFO, 0);
327 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_create(... start ...)");
328
329 for (unsigned int i = 0; i < n; i++) {
330 struct actor *act = &scene->scn_actors[i];
331 void *(*action)(void *) = NULL;
332 void *sync = NULL;
333 switch (act->act_role) {
334 case ROLE_SPIN:
335 sync = &scene->scn_spin_sync;
336 action = spin_role;
337 break;
338 case ROLE_WAIT:
339 sync = &ws;
340 action = wait_role;
341 break;
342 default:
343 T_ASSERT_FAIL("unexpected role: %d", act->act_role);
344 }
345 int error = pthread_create(&act->act_thread, NULL, action, sync);
346 T_QUIET; T_ASSERT_POSIX_ZERO(error, "pthread_create");
347 }
348
349 T_SETUPEND;
350 for (unsigned int i = 0; i < wait_count; i++) {
351 semaphore_wait(ws.ws_start);
352 }
353 semaphore_destroy(mach_task_self(), ws.ws_start);
354 scene->scn_wait_sync = (void *)(uintptr_t)ws.ws_wait;
355 return scene;
356 }
357
358 void
scene_end(struct scene * scene)359 scene_end(struct scene *scene)
360 {
361 if (!scene) {
362 return;
363 }
364
365 scene->scn_spin_sync = 0;
366 semaphore_signal_all((semaphore_t)scene->scn_wait_sync);
367 semaphore_destroy(mach_task_self(), (semaphore_t)scene->scn_wait_sync);
368 struct actor *act = scene->scn_actors;
369 while (act->act_role != ROLE_NONE) {
370 int error = pthread_join(act->act_thread, NULL);
371 T_QUIET; T_ASSERT_POSIX_ZERO(error, "pthread_join");
372 act++;
373 }
374 free(scene);
375 }
376
377 pid_t
launch_helper(char * name)378 launch_helper(char *name)
379 {
380 char bin_path[MAXPATHLEN];
381 uint32_t path_size = sizeof(bin_path);
382
383 T_SETUPBEGIN;
384 int ret = _NSGetExecutablePath(bin_path, &path_size);
385 T_QUIET;
386 T_ASSERT_EQ(ret, 0, "_NSGetExecutablePath()");
387 pid_t pid = 0;
388 ret = dt_launch_tool(&pid, (char *[]){ bin_path, name, NULL}, false, NULL,
389 NULL);
390 T_QUIET;
391 T_ASSERT_POSIX_SUCCESS(ret, "dt_launch_tool(... %s, %s ...)", bin_path,
392 name);
393 T_SETUPEND;
394
395 return pid;
396 }
397