1 /* test that the header doesn't implicitly depend on others */
2 #include <sys/work_interval.h>
3
4 #include <stdlib.h>
5 #include <stdio.h>
6 #include <unistd.h>
7 #include <errno.h>
8 #include <err.h>
9 #include <string.h>
10 #include <pthread.h>
11 #include <sys/sysctl.h>
12
13 #include <mach/mach.h>
14 #include <mach/semaphore.h>
15
16 #include <libkern/OSAtomic.h>
17
18 #include <darwintest.h>
19 #include "test_utils.h"
20
21 T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler"),
22 T_META_RADAR_COMPONENT_NAME("xnu"),
23 T_META_RADAR_COMPONENT_VERSION("scheduler"));
24
25
26 static mach_timebase_info_data_t timebase_info;
27
28 static uint64_t
nanos_to_abs(uint64_t nanos)29 nanos_to_abs(uint64_t nanos)
30 {
31 mach_timebase_info(&timebase_info);
32 return nanos * timebase_info.denom / timebase_info.numer;
33 }
34
35 static uint64_t
abs_to_nanos(uint64_t abs)36 abs_to_nanos(uint64_t abs)
37 {
38 return abs * timebase_info.numer / timebase_info.denom;
39 }
40
41 static void
set_realtime(pthread_t thread,uint64_t interval_nanos)42 set_realtime(pthread_t thread, uint64_t interval_nanos)
43 {
44 kern_return_t kr;
45 thread_time_constraint_policy_data_t pol;
46
47 mach_port_t target_thread = pthread_mach_thread_np(thread);
48 T_QUIET; T_ASSERT_GT(target_thread, 0, "pthread_mach_thread_np");
49
50 /* 1s 100ms 10ms */
51 pol.period = (uint32_t)nanos_to_abs(interval_nanos);
52 pol.constraint = (uint32_t)nanos_to_abs(interval_nanos);
53 pol.computation = (uint32_t)nanos_to_abs(interval_nanos - 1000000); // 1 ms of leeway
54
55 pol.preemptible = 0; /* Ignored by OS */
56 kr = thread_policy_set(target_thread, THREAD_TIME_CONSTRAINT_POLICY, (thread_policy_t) &pol,
57 THREAD_TIME_CONSTRAINT_POLICY_COUNT);
58 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_policy_set(THREAD_TIME_CONSTRAINT_POLICY)");
59 }
60
61 static void
create_coreaudio_work_interval(work_interval_t * wi_handle,work_interval_instance_t * wi_instance,mach_port_t * wi_port,bool enable_telemetry,uint32_t create_flags)62 create_coreaudio_work_interval(work_interval_t *wi_handle, work_interval_instance_t *wi_instance,
63 mach_port_t *wi_port, bool enable_telemetry, uint32_t create_flags)
64 {
65 int ret = 0;
66 create_flags |= WORK_INTERVAL_FLAG_GROUP | WORK_INTERVAL_FLAG_JOINABLE | WORK_INTERVAL_TYPE_COREAUDIO;
67 if (enable_telemetry) {
68 create_flags |= WORK_INTERVAL_FLAG_ENABLE_TELEMETRY_DATA;
69 }
70
71 ret = work_interval_create(wi_handle, create_flags);
72 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_create");
73
74 ret = work_interval_copy_port(*wi_handle, wi_port);
75 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_copy_port");
76
77 *wi_instance = work_interval_instance_alloc(*wi_handle);
78 T_QUIET; T_ASSERT_NE(*wi_instance, NULL, "work_interval_instance_alloc");
79 }
80
81 static void
join_coreaudio_work_interval(mach_port_t * wi_port,uint64_t interval_nanos)82 join_coreaudio_work_interval(mach_port_t *wi_port, uint64_t interval_nanos)
83 {
84 int ret = 0;
85
86 set_realtime(pthread_self(), interval_nanos);
87
88 ret = work_interval_join_port(*wi_port);
89 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_join_port");
90 }
91
92 static pthread_mutex_t barrier_lock = PTHREAD_MUTEX_INITIALIZER;
93 static pthread_cond_t barrier_cond = PTHREAD_COND_INITIALIZER;
94 static uint32_t barrier_count[2];
95 static unsigned int active_barrier_ind;
96 static uint32_t total_thread_count;
97 static uint32_t expected_cond_wakeups;
98
99 /*
100 * This implementation of a barrier using pthread_cond_t is
101 * intended to control the number of thread sleeps/wakeups
102 * that can occur, so that the reported wakeup counts from
103 * the work interval data can be validated.
104 * Each call to pthread_mutex_lock can produce 0 or 1 thread
105 * wakeups, and each call to pthread_cond_wait produces 0 or
106 * 1 wakeups.
107 */
108 static void
thread_barrier(void)109 thread_barrier(void)
110 {
111 int ret = 0;
112 ret = pthread_mutex_lock(&barrier_lock);
113 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_mutex_lock");
114
115 barrier_count[active_barrier_ind]--;
116
117 if (barrier_count[active_barrier_ind]) {
118 unsigned int local_active_barrier_ind = active_barrier_ind;
119 while (barrier_count[local_active_barrier_ind]) {
120 expected_cond_wakeups++;
121 ret = pthread_cond_wait(&barrier_cond, &barrier_lock);
122 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_cond_wait");
123 }
124 } else {
125 ret = pthread_cond_broadcast(&barrier_cond);
126 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_cond_broadcast");
127 active_barrier_ind = (active_barrier_ind + 1) % 2;
128 barrier_count[active_barrier_ind] = total_thread_count;
129 }
130
131 ret = pthread_mutex_unlock(&barrier_lock);
132 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_mutex_unlock");
133 }
134
135 struct thread_data {
136 work_interval_t wi_handle;
137 mach_port_t *wi_port;
138 unsigned int num_iterations;
139 uint64_t interval_nanos;
140 };
141
142 static volatile int64_t work_sum;
143
144 /*
145 * This work performed in the work interval is designed to
146 * require CPU compute so that CLPC perf-controls the work
147 * interval as it typically would. It is also designed such that
148 * the threads agree when the work interval work is done
149 * (work_sum higher than a specified threshold), so that the
150 * amount of work performed will be consistent between the
151 * different work interval instances.
152 */
153 static void
contribute_to_work_sum(void)154 contribute_to_work_sum(void)
155 {
156 volatile unsigned int x = 0;
157 do {
158 for (int i = 0; i < 1000; i++) {
159 x = x * x - x - 1;
160 }
161 x %= 10;
162 } while (OSAtomicAdd64(x, &work_sum) < 10000);
163 }
164
165 static void *
coreaudio_workload_fn(void * arg)166 coreaudio_workload_fn(void *arg)
167 {
168 struct thread_data *info = (struct thread_data *)arg;
169
170 join_coreaudio_work_interval(info->wi_port, info->interval_nanos);
171
172 for (unsigned int i = 0; i < info->num_iterations; i++) {
173 thread_barrier();
174 contribute_to_work_sum();
175 }
176
177 int ret = work_interval_leave();
178 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_leave");
179
180 thread_barrier();
181
182 return NULL;
183 }
184
185 static void
start_helper_threads(unsigned int num_threads,pthread_t * threads,struct thread_data * thread_datas,work_interval_t wi_handle,mach_port_t * wi_port,unsigned int num_iterations,uint64_t interval_nanos)186 start_helper_threads(unsigned int num_threads, pthread_t *threads, struct thread_data *thread_datas,
187 work_interval_t wi_handle, mach_port_t *wi_port, unsigned int num_iterations, uint64_t interval_nanos)
188 {
189 int ret = 0;
190 for (unsigned int i = 0; i < num_threads; i++) {
191 thread_datas[i].wi_handle = wi_handle;
192 thread_datas[i].wi_port = wi_port;
193 thread_datas[i].num_iterations = num_iterations;
194 thread_datas[i].interval_nanos = interval_nanos;
195 ret = pthread_create(&threads[i], NULL, coreaudio_workload_fn, &thread_datas[i]);
196 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "pthread_create");
197 }
198 }
199
200 static void
start_work_interval_instance(uint64_t interval_length_abs,work_interval_instance_t wi_instance,work_interval_data_t wi_data)201 start_work_interval_instance(uint64_t interval_length_abs, work_interval_instance_t wi_instance,
202 work_interval_data_t wi_data)
203 {
204 int ret = 0;
205 uint64_t start = mach_absolute_time();
206
207 work_interval_instance_clear(wi_instance);
208 work_interval_instance_set_start(wi_instance, start);
209 work_interval_instance_set_deadline(wi_instance, start + interval_length_abs);
210
211 // Sanity assertions that the work interval creation flags and interval id are as expected
212 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_FLAG_IGNORED, 0, "ignored flag start");
213 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_TYPE_MASK, WORK_INTERVAL_TYPE_COREAUDIO, "coreaudio start");
214 T_QUIET; T_ASSERT_NE(wi_instance->wi_interval_id, 0ULL, "nonzero wi_interval_id");
215
216 ret = work_interval_instance_start(wi_instance);
217 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "work_interval_instance_start");
218
219 if (wi_instance->wi_instance_id == 0ULL) {
220 T_LOG("wi_instance_id is 0, which is an acceptable condition for devices running legacy CLPC");
221 }
222
223 work_interval_instance_get_telemetry_data(wi_instance, wi_data, sizeof(struct work_interval_data));
224 }
225
226 static uint64_t
finish_work_interval_instance(work_interval_instance_t wi_instance,work_interval_data_t wi_data)227 finish_work_interval_instance(work_interval_instance_t wi_instance, work_interval_data_t wi_data)
228 {
229 int ret = 0;
230 uint64_t finish = mach_absolute_time();
231 work_interval_instance_set_finish(wi_instance, finish);
232
233 // Sanity assertions that the work interval creation flags and interval id are as expected
234 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_FLAG_IGNORED, 0, "ignored flag");
235 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_TYPE_MASK, WORK_INTERVAL_TYPE_COREAUDIO, "coreaudio start");
236 T_QUIET; T_ASSERT_NE(wi_instance->wi_interval_id, 0ULL, "nonzero wi_interval_id");
237
238 uint64_t remembered_start = wi_instance->wi_start;
239
240 ret = work_interval_instance_finish(wi_instance);
241 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "work_interval_instance_finish");
242
243 work_interval_instance_get_telemetry_data(wi_instance, wi_data, sizeof(struct work_interval_data));
244
245 return abs_to_nanos(finish - remembered_start);
246 }
247
248 static void
verify_monotonic_work_interval_data(struct work_interval_data * curr_data,struct work_interval_data * prev_data,bool supports_cpi)249 verify_monotonic_work_interval_data(struct work_interval_data *curr_data, struct work_interval_data *prev_data, bool supports_cpi)
250 {
251 if (prev_data != NULL) {
252 T_QUIET; T_ASSERT_GE(curr_data->wid_external_wakeups, prev_data->wid_external_wakeups, "wid_external_wakeups");
253 T_QUIET; T_ASSERT_GE(curr_data->wid_total_wakeups, prev_data->wid_total_wakeups, "wid_external_wakeups");
254 }
255 T_QUIET; T_ASSERT_GE(curr_data->wid_user_time_mach, prev_data == NULL ? 1 : prev_data->wid_user_time_mach, "monotonic wid_user_time_mach");
256 T_QUIET; T_ASSERT_GE(curr_data->wid_system_time_mach, prev_data == NULL ? 1 : prev_data->wid_system_time_mach, "monotonic wid_system_time_mach");
257 if (supports_cpi) {
258 T_QUIET; T_ASSERT_GE(curr_data->wid_cycles, prev_data == NULL ? 1 : prev_data->wid_cycles, "monotonic wid_cycles");
259 T_QUIET; T_ASSERT_GE(curr_data->wid_instructions, prev_data == NULL ? 1 : prev_data->wid_instructions, "monotonic wid_instructions");
260 }
261 }
262
263 static void
verify_zero_work_interval_data(struct work_interval_data * wi_data,bool supports_cpi)264 verify_zero_work_interval_data(struct work_interval_data *wi_data, bool supports_cpi)
265 {
266 T_QUIET; T_ASSERT_EQ(wi_data->wid_external_wakeups, 0, "zero wid_external_wakeups");
267 T_QUIET; T_ASSERT_EQ(wi_data->wid_total_wakeups, 0, "zero wid_total_wakeups");
268 T_QUIET; T_ASSERT_EQ(wi_data->wid_user_time_mach, 0ULL, "zero wid_user_time_mach");
269 T_QUIET; T_ASSERT_EQ(wi_data->wid_system_time_mach, 0ULL, "zero wid_system_time_mach");
270 if (supports_cpi) {
271 T_QUIET; T_ASSERT_EQ(wi_data->wid_cycles, 0ULL, "zero wid_cycles");
272 T_QUIET; T_ASSERT_EQ(wi_data->wid_instructions, 0ULL, "zero wid_instructions");
273 }
274 }
275
276 static void
run_work_interval_data_test(unsigned int num_iterations,uint64_t interval_nanos,unsigned int thread_count,bool enable_telemetry,uint32_t flags)277 run_work_interval_data_test(unsigned int num_iterations, uint64_t interval_nanos, unsigned int thread_count,
278 bool enable_telemetry, uint32_t flags)
279 {
280 T_SETUPBEGIN;
281
282 int ret = 0;
283
284 int supports_cpi = 0;
285 size_t supports_cpi_size = sizeof(supports_cpi);
286 ret = sysctlbyname("kern.monotonic.supported", &supports_cpi, &supports_cpi_size, NULL, 0);
287 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.monotonic.supported");
288
289 work_interval_t wi_handle = NULL;
290 work_interval_instance_t wi_instance = NULL;
291 mach_port_t wi_port = MACH_PORT_NULL;
292
293 create_coreaudio_work_interval(&wi_handle, &wi_instance, &wi_port, enable_telemetry, flags);
294 join_coreaudio_work_interval(&wi_port, interval_nanos);
295
296 total_thread_count = thread_count;
297 expected_cond_wakeups = 0;
298 unsigned int num_helper_threads = thread_count - 1;
299 active_barrier_ind = 0;
300 barrier_count[active_barrier_ind] = thread_count;
301 pthread_t wi_threads[num_helper_threads];
302 struct thread_data wi_thread_datas[num_helper_threads];
303
304 start_helper_threads(num_helper_threads, wi_threads, wi_thread_datas, wi_handle, &wi_port, num_iterations, interval_nanos);
305
306 T_SETUPEND;
307
308 uint64_t interval_length_abs = nanos_to_abs(interval_nanos);
309 uint64_t duration_sum = 0;
310 struct work_interval_data start_data = {0};
311 struct work_interval_data finish_data = {0};
312
313 for (unsigned int i = 0; i < num_iterations; i++) {
314 work_sum = 0;
315
316 usleep(1000);
317
318 start_work_interval_instance(interval_length_abs, wi_instance, &start_data);
319 if (i == 0 && enable_telemetry) {
320 verify_monotonic_work_interval_data(&start_data, NULL, supports_cpi);
321 } else if (!enable_telemetry) {
322 verify_zero_work_interval_data(&start_data, supports_cpi);
323 }
324
325 thread_barrier();
326 contribute_to_work_sum();
327
328 duration_sum += finish_work_interval_instance(wi_instance, &finish_data);
329 if (enable_telemetry) {
330 verify_monotonic_work_interval_data(&finish_data, &start_data, supports_cpi);
331 } else {
332 verify_zero_work_interval_data(&finish_data, supports_cpi);
333 }
334 }
335
336 ret = work_interval_leave();
337 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_leave");
338 thread_barrier();
339
340 if (enable_telemetry) {
341 T_ASSERT_TRUE(true, "Overall wid_external_wakeups: %u\n", finish_data.wid_external_wakeups);
342 // Only the wakeups from usleep() are guaranteed to occur
343 T_ASSERT_GE(finish_data.wid_total_wakeups, num_iterations, "wid_total_wakeups at least accounts for the usleep() wakeups");
344 }
345 T_ASSERT_TRUE(true, "Workload survived %u iterations without failures!!! Avg. work interval duration was %llu ns out of a requested %llu ns", num_iterations, duration_sum / num_iterations, interval_nanos);
346 }
347
348 static const unsigned int DEFAULT_ITERS = 1000;
349 static const uint64_t DEFAULT_INTERVAL_NS = 15000000; // 15 ms
350 static const uint64_t DEFAULT_THREAD_COUNT = 3;
351
352 T_DECL(work_interval_rt_coreaudio_quality_telemetry_data, "receiving accurate telemetry data as a coreaudio work interval",
353 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
354 {
355 run_work_interval_data_test(
356 DEFAULT_ITERS,
357 DEFAULT_INTERVAL_NS,
358 DEFAULT_THREAD_COUNT,
359 true, // enable_telemetry
360 0); // no added flags
361 }
362
363 T_DECL(work_interval_rt_coreaudio_telemetry_disabled, "reading telemetry data should see all zeroes if it isn't enabled",
364 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
365 {
366 run_work_interval_data_test(
367 DEFAULT_ITERS,
368 DEFAULT_INTERVAL_NS,
369 DEFAULT_THREAD_COUNT,
370 false, // enable_telemetry
371 0); // no added flags
372 }
373
374 T_DECL(work_interval_rt_coreaudio_telemetry_data_many_threads, "work interval telemetry data works with many joined threads",
375 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
376 {
377 run_work_interval_data_test(
378 DEFAULT_ITERS,
379 DEFAULT_INTERVAL_NS,
380 20, // threads
381 true, // enable_telemetry
382 0); // no added flags
383 }
384
385 T_DECL(work_interval_rt_coreaudio_telemetry_supported_with_other_flags, "telemetry supported when the other creation flags used by coreaudio are set",
386 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
387 {
388 T_LOG("Coreaudio work interval with auto-join and deferred finish enabled");
389 run_work_interval_data_test(
390 DEFAULT_ITERS,
391 DEFAULT_INTERVAL_NS,
392 DEFAULT_THREAD_COUNT, // threads
393 true, // enable_telemetry
394 WORK_INTERVAL_FLAG_ENABLE_AUTO_JOIN | WORK_INTERVAL_FLAG_ENABLE_DEFERRED_FINISH);
395
396 T_LOG("Coreaudio work interval with auto-join, deferred finish, and unrestricted flags enabled");
397 run_work_interval_data_test(
398 DEFAULT_ITERS,
399 DEFAULT_INTERVAL_NS,
400 DEFAULT_THREAD_COUNT, // threads
401 true, // enable_telemetry
402 WORK_INTERVAL_FLAG_ENABLE_AUTO_JOIN | WORK_INTERVAL_FLAG_ENABLE_DEFERRED_FINISH | WORK_INTERVAL_FLAG_UNRESTRICTED);
403 }
404