1 /* test that the header doesn't implicitly depend on others */
2 #include <sys/work_interval.h>
3
4 #include <stdlib.h>
5 #include <stdio.h>
6 #include <unistd.h>
7 #include <errno.h>
8 #include <err.h>
9 #include <string.h>
10 #include <pthread.h>
11 #include <sys/sysctl.h>
12
13 #include <mach/mach.h>
14 #include <mach/semaphore.h>
15
16 #include <libkern/OSAtomic.h>
17
18 #include <darwintest.h>
19 #include "test_utils.h"
20
21 T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler"),
22 T_META_RADAR_COMPONENT_NAME("xnu"),
23 T_META_RADAR_COMPONENT_VERSION("scheduler"),
24 T_META_TAG_VM_NOT_ELIGIBLE);
25
26
27 static mach_timebase_info_data_t timebase_info;
28
29 static uint64_t
nanos_to_abs(uint64_t nanos)30 nanos_to_abs(uint64_t nanos)
31 {
32 mach_timebase_info(&timebase_info);
33 return nanos * timebase_info.denom / timebase_info.numer;
34 }
35
36 static uint64_t
abs_to_nanos(uint64_t abs)37 abs_to_nanos(uint64_t abs)
38 {
39 return abs * timebase_info.numer / timebase_info.denom;
40 }
41
42 static void
set_realtime(pthread_t thread,uint64_t interval_nanos)43 set_realtime(pthread_t thread, uint64_t interval_nanos)
44 {
45 kern_return_t kr;
46 thread_time_constraint_policy_data_t pol;
47
48 mach_port_t target_thread = pthread_mach_thread_np(thread);
49 T_QUIET; T_ASSERT_GT(target_thread, 0, "pthread_mach_thread_np");
50
51 /* 1s 100ms 10ms */
52 pol.period = (uint32_t)nanos_to_abs(interval_nanos);
53 pol.constraint = (uint32_t)nanos_to_abs(interval_nanos);
54 pol.computation = (uint32_t)nanos_to_abs(interval_nanos - 1000000); // 1 ms of leeway
55
56 pol.preemptible = 0; /* Ignored by OS */
57 kr = thread_policy_set(target_thread, THREAD_TIME_CONSTRAINT_POLICY, (thread_policy_t) &pol,
58 THREAD_TIME_CONSTRAINT_POLICY_COUNT);
59 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_policy_set(THREAD_TIME_CONSTRAINT_POLICY)");
60 }
61
62 static void
create_coreaudio_work_interval(work_interval_t * wi_handle,work_interval_instance_t * wi_instance,mach_port_t * wi_port,bool enable_telemetry,uint32_t create_flags)63 create_coreaudio_work_interval(work_interval_t *wi_handle, work_interval_instance_t *wi_instance,
64 mach_port_t *wi_port, bool enable_telemetry, uint32_t create_flags)
65 {
66 int ret = 0;
67 create_flags |= WORK_INTERVAL_FLAG_GROUP | WORK_INTERVAL_FLAG_JOINABLE | WORK_INTERVAL_TYPE_COREAUDIO;
68 if (enable_telemetry) {
69 create_flags |= WORK_INTERVAL_FLAG_ENABLE_TELEMETRY_DATA;
70 }
71
72 ret = work_interval_create(wi_handle, create_flags);
73 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_create");
74
75 ret = work_interval_copy_port(*wi_handle, wi_port);
76 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_copy_port");
77
78 *wi_instance = work_interval_instance_alloc(*wi_handle);
79 T_QUIET; T_ASSERT_NE(*wi_instance, NULL, "work_interval_instance_alloc");
80 }
81
82 static void
join_coreaudio_work_interval(mach_port_t * wi_port,uint64_t interval_nanos)83 join_coreaudio_work_interval(mach_port_t *wi_port, uint64_t interval_nanos)
84 {
85 int ret = 0;
86
87 set_realtime(pthread_self(), interval_nanos);
88
89 ret = work_interval_join_port(*wi_port);
90 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_join_port");
91 }
92
93 static pthread_mutex_t barrier_lock = PTHREAD_MUTEX_INITIALIZER;
94 static pthread_cond_t barrier_cond = PTHREAD_COND_INITIALIZER;
95 static uint32_t barrier_count[2];
96 static unsigned int active_barrier_ind;
97 static uint32_t total_thread_count;
98 static uint32_t expected_cond_wakeups;
99
100 /*
101 * This implementation of a barrier using pthread_cond_t is
102 * intended to control the number of thread sleeps/wakeups
103 * that can occur, so that the reported wakeup counts from
104 * the work interval data can be validated.
105 * Each call to pthread_mutex_lock can produce 0 or 1 thread
106 * wakeups, and each call to pthread_cond_wait produces 0 or
107 * 1 wakeups.
108 */
109 static void
thread_barrier(void)110 thread_barrier(void)
111 {
112 int ret = 0;
113 ret = pthread_mutex_lock(&barrier_lock);
114 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_mutex_lock");
115
116 barrier_count[active_barrier_ind]--;
117
118 if (barrier_count[active_barrier_ind]) {
119 unsigned int local_active_barrier_ind = active_barrier_ind;
120 while (barrier_count[local_active_barrier_ind]) {
121 expected_cond_wakeups++;
122 ret = pthread_cond_wait(&barrier_cond, &barrier_lock);
123 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_cond_wait");
124 }
125 } else {
126 ret = pthread_cond_broadcast(&barrier_cond);
127 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_cond_broadcast");
128 active_barrier_ind = (active_barrier_ind + 1) % 2;
129 barrier_count[active_barrier_ind] = total_thread_count;
130 }
131
132 ret = pthread_mutex_unlock(&barrier_lock);
133 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "pthread_mutex_unlock");
134 }
135
136 struct thread_data {
137 work_interval_t wi_handle;
138 mach_port_t *wi_port;
139 unsigned int num_iterations;
140 uint64_t interval_nanos;
141 };
142
143 static volatile int64_t work_sum;
144
145 /*
146 * This work performed in the work interval is designed to
147 * require CPU compute so that CLPC perf-controls the work
148 * interval as it typically would. It is also designed such that
149 * the threads agree when the work interval work is done
150 * (work_sum higher than a specified threshold), so that the
151 * amount of work performed will be consistent between the
152 * different work interval instances.
153 */
154 static void
contribute_to_work_sum(void)155 contribute_to_work_sum(void)
156 {
157 volatile unsigned int x = 0;
158 do {
159 for (int i = 0; i < 1000; i++) {
160 x = x * x - x - 1;
161 }
162 x %= 10;
163 } while (OSAtomicAdd64(x, &work_sum) < 10000);
164 }
165
166 static void *
coreaudio_workload_fn(void * arg)167 coreaudio_workload_fn(void *arg)
168 {
169 struct thread_data *info = (struct thread_data *)arg;
170
171 join_coreaudio_work_interval(info->wi_port, info->interval_nanos);
172
173 for (unsigned int i = 0; i < info->num_iterations; i++) {
174 thread_barrier();
175 contribute_to_work_sum();
176 }
177
178 int ret = work_interval_leave();
179 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_leave");
180
181 thread_barrier();
182
183 return NULL;
184 }
185
186 static void
start_helper_threads(unsigned int num_threads,pthread_t * threads,struct thread_data * thread_datas,work_interval_t wi_handle,mach_port_t * wi_port,unsigned int num_iterations,uint64_t interval_nanos)187 start_helper_threads(unsigned int num_threads, pthread_t *threads, struct thread_data *thread_datas,
188 work_interval_t wi_handle, mach_port_t *wi_port, unsigned int num_iterations, uint64_t interval_nanos)
189 {
190 int ret = 0;
191 for (unsigned int i = 0; i < num_threads; i++) {
192 thread_datas[i].wi_handle = wi_handle;
193 thread_datas[i].wi_port = wi_port;
194 thread_datas[i].num_iterations = num_iterations;
195 thread_datas[i].interval_nanos = interval_nanos;
196 ret = pthread_create(&threads[i], NULL, coreaudio_workload_fn, &thread_datas[i]);
197 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "pthread_create");
198 }
199 }
200
201 static bool logged_wi_instance_id_zero = false;
202
203 static void
start_work_interval_instance(uint64_t interval_length_abs,work_interval_instance_t wi_instance,work_interval_data_t wi_data)204 start_work_interval_instance(uint64_t interval_length_abs, work_interval_instance_t wi_instance,
205 work_interval_data_t wi_data)
206 {
207 int ret = 0;
208 uint64_t start = mach_absolute_time();
209
210 work_interval_instance_clear(wi_instance);
211 work_interval_instance_set_start(wi_instance, start);
212 work_interval_instance_set_deadline(wi_instance, start + interval_length_abs);
213
214 // Sanity assertions that the work interval creation flags and interval id are as expected
215 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_FLAG_IGNORED, 0, "ignored flag start");
216 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_TYPE_MASK, WORK_INTERVAL_TYPE_COREAUDIO, "coreaudio start");
217 T_QUIET; T_ASSERT_NE(wi_instance->wi_interval_id, 0ULL, "nonzero wi_interval_id");
218
219 ret = work_interval_instance_start(wi_instance);
220 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "work_interval_instance_start");
221
222 if (wi_instance->wi_instance_id == 0ULL && !logged_wi_instance_id_zero) {
223 T_LOG("Note, wi_instance_id is 0, which is an acceptable condition for devices running legacy CLPC");
224 logged_wi_instance_id_zero = true;
225 }
226
227 work_interval_instance_get_telemetry_data(wi_instance, wi_data, sizeof(struct work_interval_data));
228 }
229
230 static uint64_t
finish_work_interval_instance(work_interval_instance_t wi_instance,work_interval_data_t wi_data)231 finish_work_interval_instance(work_interval_instance_t wi_instance, work_interval_data_t wi_data)
232 {
233 int ret = 0;
234 uint64_t finish = mach_absolute_time();
235 work_interval_instance_set_finish(wi_instance, finish);
236
237 // Sanity assertions that the work interval creation flags and interval id are as expected
238 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_FLAG_IGNORED, 0, "ignored flag");
239 T_QUIET; T_ASSERT_EQ(wi_instance->wi_create_flags & WORK_INTERVAL_TYPE_MASK, WORK_INTERVAL_TYPE_COREAUDIO, "coreaudio start");
240 T_QUIET; T_ASSERT_NE(wi_instance->wi_interval_id, 0ULL, "nonzero wi_interval_id");
241
242 uint64_t remembered_start = wi_instance->wi_start;
243
244 ret = work_interval_instance_finish(wi_instance);
245 T_QUIET; T_ASSERT_POSIX_ZERO(ret, "work_interval_instance_finish");
246
247 work_interval_instance_get_telemetry_data(wi_instance, wi_data, sizeof(struct work_interval_data));
248
249 return abs_to_nanos(finish - remembered_start);
250 }
251
252 static void
verify_monotonic_work_interval_data(struct work_interval_data * curr_data,struct work_interval_data * prev_data,bool supports_cpi)253 verify_monotonic_work_interval_data(struct work_interval_data *curr_data, struct work_interval_data *prev_data, bool supports_cpi)
254 {
255 if (prev_data != NULL) {
256 T_QUIET; T_ASSERT_GE(curr_data->wid_external_wakeups, prev_data->wid_external_wakeups, "wid_external_wakeups");
257 T_QUIET; T_ASSERT_GE(curr_data->wid_total_wakeups, prev_data->wid_total_wakeups, "wid_external_wakeups");
258 }
259 T_QUIET; T_ASSERT_GE(curr_data->wid_user_time_mach, prev_data == NULL ? 1 : prev_data->wid_user_time_mach, "monotonic wid_user_time_mach");
260 T_QUIET; T_ASSERT_GE(curr_data->wid_system_time_mach, prev_data == NULL ? 1 : prev_data->wid_system_time_mach, "monotonic wid_system_time_mach");
261 if (supports_cpi) {
262 T_QUIET; T_ASSERT_GE(curr_data->wid_cycles, prev_data == NULL ? 1 : prev_data->wid_cycles, "monotonic wid_cycles");
263 T_QUIET; T_ASSERT_GE(curr_data->wid_instructions, prev_data == NULL ? 1 : prev_data->wid_instructions, "monotonic wid_instructions");
264 }
265 }
266
267 static void
verify_zero_work_interval_data(struct work_interval_data * wi_data,bool supports_cpi)268 verify_zero_work_interval_data(struct work_interval_data *wi_data, bool supports_cpi)
269 {
270 T_QUIET; T_ASSERT_EQ(wi_data->wid_external_wakeups, 0, "zero wid_external_wakeups");
271 T_QUIET; T_ASSERT_EQ(wi_data->wid_total_wakeups, 0, "zero wid_total_wakeups");
272 T_QUIET; T_ASSERT_EQ(wi_data->wid_user_time_mach, 0ULL, "zero wid_user_time_mach");
273 T_QUIET; T_ASSERT_EQ(wi_data->wid_system_time_mach, 0ULL, "zero wid_system_time_mach");
274 if (supports_cpi) {
275 T_QUIET; T_ASSERT_EQ(wi_data->wid_cycles, 0ULL, "zero wid_cycles");
276 T_QUIET; T_ASSERT_EQ(wi_data->wid_instructions, 0ULL, "zero wid_instructions");
277 }
278 }
279
280 static void
run_work_interval_data_test(unsigned int num_iterations,uint64_t interval_nanos,unsigned int thread_count,bool enable_telemetry,uint32_t flags)281 run_work_interval_data_test(unsigned int num_iterations, uint64_t interval_nanos, unsigned int thread_count,
282 bool enable_telemetry, uint32_t flags)
283 {
284 T_SETUPBEGIN;
285
286 int ret = 0;
287
288 int supports_cpi = 0;
289 size_t supports_cpi_size = sizeof(supports_cpi);
290 ret = sysctlbyname("kern.monotonic.supported", &supports_cpi, &supports_cpi_size, NULL, 0);
291 if (ret < 0 || supports_cpi == 0) {
292 T_LOG("Monotonic stats are unsupported on this platform. Skipping cycles/instructions stats validation");
293 }
294
295 work_interval_t wi_handle = NULL;
296 work_interval_instance_t wi_instance = NULL;
297 mach_port_t wi_port = MACH_PORT_NULL;
298
299 create_coreaudio_work_interval(&wi_handle, &wi_instance, &wi_port, enable_telemetry, flags);
300 join_coreaudio_work_interval(&wi_port, interval_nanos);
301
302 total_thread_count = thread_count;
303 expected_cond_wakeups = 0;
304 unsigned int num_helper_threads = thread_count - 1;
305 active_barrier_ind = 0;
306 barrier_count[active_barrier_ind] = thread_count;
307 pthread_t wi_threads[num_helper_threads];
308 struct thread_data wi_thread_datas[num_helper_threads];
309
310 start_helper_threads(num_helper_threads, wi_threads, wi_thread_datas, wi_handle, &wi_port, num_iterations, interval_nanos);
311
312 T_SETUPEND;
313
314 uint64_t interval_length_abs = nanos_to_abs(interval_nanos);
315 uint64_t duration_sum = 0;
316 struct work_interval_data start_data = {0};
317 struct work_interval_data finish_data = {0};
318
319 for (unsigned int i = 0; i < num_iterations; i++) {
320 work_sum = 0;
321
322 usleep(1000);
323
324 start_work_interval_instance(interval_length_abs, wi_instance, &start_data);
325 if (i == 0 && enable_telemetry) {
326 verify_monotonic_work_interval_data(&start_data, NULL, supports_cpi);
327 } else if (!enable_telemetry) {
328 verify_zero_work_interval_data(&start_data, supports_cpi);
329 }
330
331 thread_barrier();
332 contribute_to_work_sum();
333
334 duration_sum += finish_work_interval_instance(wi_instance, &finish_data);
335 if (enable_telemetry) {
336 verify_monotonic_work_interval_data(&finish_data, &start_data, supports_cpi);
337 } else {
338 verify_zero_work_interval_data(&finish_data, supports_cpi);
339 }
340 }
341
342 ret = work_interval_leave();
343 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "work_interval_leave");
344 thread_barrier();
345
346 if (enable_telemetry) {
347 T_ASSERT_TRUE(true, "Overall wid_external_wakeups: %u\n", finish_data.wid_external_wakeups);
348 // Only the wakeups from usleep() are guaranteed to occur
349 T_ASSERT_GE(finish_data.wid_total_wakeups, num_iterations, "wid_total_wakeups at least accounts for the usleep() wakeups");
350 }
351 T_ASSERT_TRUE(true, "Workload survived %u iterations without failures!!! Avg. work interval duration was %llu ns out of a requested %llu ns", num_iterations, duration_sum / num_iterations, interval_nanos);
352 }
353
354 static const unsigned int DEFAULT_ITERS = 1000;
355 static const uint64_t DEFAULT_INTERVAL_NS = 15000000; // 15 ms
356 static const uint64_t DEFAULT_THREAD_COUNT = 3;
357
358 T_DECL(work_interval_rt_coreaudio_quality_telemetry_data, "receiving accurate telemetry data as a coreaudio work interval",
359 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
360 {
361 run_work_interval_data_test(
362 DEFAULT_ITERS,
363 DEFAULT_INTERVAL_NS,
364 DEFAULT_THREAD_COUNT,
365 true, // enable_telemetry
366 0); // no added flags
367 }
368
369 T_DECL(work_interval_rt_coreaudio_telemetry_disabled, "reading telemetry data should see all zeroes if it isn't enabled",
370 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
371 {
372 run_work_interval_data_test(
373 DEFAULT_ITERS,
374 DEFAULT_INTERVAL_NS,
375 DEFAULT_THREAD_COUNT,
376 false, // enable_telemetry
377 0); // no added flags
378 }
379
380 T_DECL(work_interval_rt_coreaudio_telemetry_data_many_threads, "work interval telemetry data works with many joined threads",
381 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
382 {
383 run_work_interval_data_test(
384 DEFAULT_ITERS,
385 DEFAULT_INTERVAL_NS,
386 20, // threads
387 true, // enable_telemetry
388 0); // no added flags
389 }
390
391 T_DECL(work_interval_rt_coreaudio_telemetry_supported_with_other_flags, "telemetry supported when the other creation flags used by coreaudio are set",
392 T_META_ASROOT(YES), XNU_T_META_SOC_SPECIFIC, T_META_ENABLED(TARGET_CPU_ARM64))
393 {
394 T_LOG("Coreaudio work interval with auto-join and deferred finish enabled");
395 run_work_interval_data_test(
396 DEFAULT_ITERS,
397 DEFAULT_INTERVAL_NS,
398 DEFAULT_THREAD_COUNT, // threads
399 true, // enable_telemetry
400 WORK_INTERVAL_FLAG_ENABLE_AUTO_JOIN | WORK_INTERVAL_FLAG_ENABLE_DEFERRED_FINISH);
401
402 T_LOG("Coreaudio work interval with auto-join, deferred finish, and unrestricted flags enabled");
403 run_work_interval_data_test(
404 DEFAULT_ITERS,
405 DEFAULT_INTERVAL_NS,
406 DEFAULT_THREAD_COUNT, // threads
407 true, // enable_telemetry
408 WORK_INTERVAL_FLAG_ENABLE_AUTO_JOIN | WORK_INTERVAL_FLAG_ENABLE_DEFERRED_FINISH | WORK_INTERVAL_FLAG_UNRESTRICTED);
409 }
410