1 /*
2 * turnstile_multihop: Tests turnstile and multi hop priority propagation.
3 */
4
5 #ifdef T_NAMESPACE
6 #undef T_NAMESPACE
7 #endif
8
9 #include <darwintest.h>
10 #include <darwintest_multiprocess.h>
11
12 #include <dispatch/dispatch.h>
13 #include <pthread.h>
14 #include <launch.h>
15 #include <mach/mach.h>
16 #include <mach/message.h>
17 #include <mach/mach_voucher.h>
18 #include <pthread/workqueue_private.h>
19 #include <voucher/ipc_pthread_priority_types.h>
20 #include <servers/bootstrap.h>
21 #include <stdlib.h>
22 #include <sys/event.h>
23 #include <unistd.h>
24 #include <crt_externs.h>
25 #include <signal.h>
26 #include <sys/types.h>
27 #include <sys/sysctl.h>
28 #include <libkern/OSAtomic.h>
29 #include <sys/wait.h>
30
31 #include "turnstile_multihop_helper.h"
32
33 T_GLOBAL_META(T_META_NAMESPACE("xnu.turnstile_multihop"));
34
35 #define HELPER_TIMEOUT_SECS (3000)
36
37 struct test_msg {
38 mach_msg_header_t header;
39 mach_msg_body_t body;
40 mach_msg_port_descriptor_t port_descriptor;
41 };
42
43 static boolean_t spin_for_ever = false;
44
45 static boolean_t test_noimportance = false;
46
47 #define EXPECTED_MESSAGE_ID 0x100
48
49 static void
50 thread_create_at_qos(qos_class_t qos, void * (*function)(void *));
51 static uint64_t
52 nanoseconds_to_absolutetime(uint64_t nanoseconds);
53 static int
54 sched_create_load_at_qos(qos_class_t qos, void **load_token);
55 static int
56 sched_terminate_load(void *load_token) __unused;
57 static void do_work(int num);
58 static void
59 dispatch_sync_cancel(mach_port_t owner_thread, qos_class_t promote_qos);
60
61 static void *sched_load_thread(void *);
62
63 struct load_token_context {
64 volatile int threads_should_exit;
65 int thread_count;
66 qos_class_t qos;
67 pthread_t *threads;
68 };
69
70 static struct mach_timebase_info sched_mti;
71 static pthread_once_t sched_mti_once_control = PTHREAD_ONCE_INIT;
72
73 static void
sched_mti_init(void)74 sched_mti_init(void)
75 {
76 mach_timebase_info(&sched_mti);
77 }
78 uint64_t
nanoseconds_to_absolutetime(uint64_t nanoseconds)79 nanoseconds_to_absolutetime(uint64_t nanoseconds)
80 {
81 pthread_once(&sched_mti_once_control, sched_mti_init);
82
83 return (uint64_t)(nanoseconds * (((double)sched_mti.denom) / ((double)sched_mti.numer)));
84 }
85
86 static int
sched_create_load_at_qos(qos_class_t qos,void ** load_token)87 sched_create_load_at_qos(qos_class_t qos, void **load_token)
88 {
89 struct load_token_context *context = NULL;
90 int ret;
91 int ncpu;
92 size_t ncpu_size = sizeof(ncpu);
93 int nthreads;
94 int i;
95 pthread_attr_t attr;
96
97 ret = sysctlbyname("hw.ncpu", &ncpu, &ncpu_size, NULL, 0);
98 if (ret == -1) {
99 T_LOG("sysctlbyname(hw.ncpu)");
100 return errno;
101 }
102
103 T_QUIET; T_LOG("%s: Detected %d CPUs\n", __FUNCTION__, ncpu);
104
105 nthreads = ncpu;
106 T_QUIET; T_LOG("%s: Will create %d threads\n", __FUNCTION__, nthreads);
107
108 ret = pthread_attr_init(&attr);
109 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_attr_init");
110
111 if (&pthread_attr_set_qos_class_np) {
112 ret = pthread_attr_set_qos_class_np(&attr, qos, 0);
113 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_attr_set_qos_class_np");
114 }
115
116 context = calloc(1, sizeof(*context));
117 if (context == NULL) {
118 T_QUIET; T_LOG("calloc returned error"); return ENOMEM;
119 }
120
121 context->threads_should_exit = 0;
122 context->thread_count = nthreads;
123 context->qos = qos;
124 context->threads = calloc((unsigned int)nthreads, sizeof(pthread_t));
125
126 OSMemoryBarrier();
127
128 for (i = 0; i < nthreads; i++) {
129 ret = pthread_create(&context->threads[i], &attr, sched_load_thread, context);
130 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_create");
131 T_QUIET; T_LOG("%s: Created thread %d (%p)\n", __FUNCTION__, i, (void *)context->threads[i]);
132 }
133
134 ret = pthread_attr_destroy(&attr);
135 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_attr_destroy");
136
137 *load_token = context;
138
139 return 0;
140 }
141
142 static void *
sched_load_thread(void * arg)143 sched_load_thread(void *arg)
144 {
145 struct load_token_context *context = (struct load_token_context *)arg;
146
147 T_QUIET; T_LOG("%s: Thread started %p\n", __FUNCTION__, (void *)pthread_self());
148
149 while (!context->threads_should_exit) {
150 uint64_t start = mach_absolute_time();
151 uint64_t end = start + nanoseconds_to_absolutetime(900ULL * NSEC_PER_MSEC);
152
153 while ((mach_absolute_time() < end) && !context->threads_should_exit) {
154 ;
155 }
156 }
157
158 T_QUIET; T_LOG("%s: Thread terminating %p\n", __FUNCTION__, (void *)pthread_self());
159
160 return NULL;
161 }
162
163 static int
sched_terminate_load(void * load_token)164 sched_terminate_load(void *load_token)
165 {
166 int ret;
167 int i;
168 struct load_token_context *context = (struct load_token_context *)load_token;
169
170 context->threads_should_exit = 1;
171 OSMemoryBarrier();
172
173 for (i = 0; i < context->thread_count; i++) {
174 T_QUIET; T_LOG("%s: Joining thread %d (%p)\n", __FUNCTION__, i, (void *)context->threads[i]);
175 ret = pthread_join(context->threads[i], NULL);
176 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_join");
177 }
178
179 free(context->threads);
180 free(context);
181
182 return 0;
183 }
184
185
186 // Find the first num primes, simply as a means of doing work
187 static void
do_work(int num)188 do_work(int num)
189 {
190 volatile int i = 3, count, c;
191
192 for (count = 2; count <= num;) {
193 for (c = 2; c <= i; c++) {
194 if (i % c == 0) {
195 break;
196 }
197 }
198 if (c == i) {
199 count++;
200 }
201 i++;
202 }
203 }
204
205 #pragma mark pthread callbacks
206
207 static void
worker_cb(pthread_priority_t __unused priority)208 worker_cb(pthread_priority_t __unused priority)
209 {
210 T_FAIL("a worker thread was created");
211 }
212
213 static void
event_cb(void ** __unused events,int * __unused nevents)214 event_cb(void ** __unused events, int * __unused nevents)
215 {
216 T_FAIL("a kevent routine was called instead of workloop");
217 }
218
219 static uint32_t
get_user_promotion_basepri(void)220 get_user_promotion_basepri(void)
221 {
222 mach_msg_type_number_t count = THREAD_POLICY_STATE_COUNT;
223 struct thread_policy_state thread_policy;
224 boolean_t get_default = FALSE;
225 mach_port_t thread_port = pthread_mach_thread_np(pthread_self());
226
227 kern_return_t kr = thread_policy_get(thread_port, THREAD_POLICY_STATE,
228 (thread_policy_t)&thread_policy, &count, &get_default);
229 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_policy_get");
230 return thread_policy.thps_user_promotion_basepri;
231 }
232
233 static uint32_t
get_thread_base_priority(void)234 get_thread_base_priority(void)
235 {
236 kern_return_t kr;
237 mach_port_t thread_port = pthread_mach_thread_np(pthread_self());
238
239 policy_timeshare_info_data_t timeshare_info;
240 mach_msg_type_number_t count = POLICY_TIMESHARE_INFO_COUNT;
241
242 kr = thread_info(thread_port, THREAD_SCHED_TIMESHARE_INFO,
243 (thread_info_t)×hare_info, &count);
244 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
245
246 return (uint32_t)timeshare_info.base_priority;
247 }
248
249
250 #define LISTENER_WLID 0x100
251 #define CONN_WLID 0x200
252
253 static uint32_t
register_port_options(void)254 register_port_options(void)
255 {
256 return MACH_RCV_MSG | MACH_RCV_LARGE | MACH_RCV_LARGE_IDENTITY |
257 MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX) |
258 MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) |
259 MACH_RCV_VOUCHER;
260 }
261
262 static void
register_port(uint64_t wlid,mach_port_t port)263 register_port(uint64_t wlid, mach_port_t port)
264 {
265 int r;
266
267 struct kevent_qos_s kev = {
268 .ident = port,
269 .filter = EVFILT_MACHPORT,
270 .flags = EV_ADD | EV_UDATA_SPECIFIC | EV_DISPATCH | EV_VANISHED,
271 .fflags = register_port_options(),
272 .data = 1,
273 .qos = (int32_t)_pthread_qos_class_encode(QOS_CLASS_MAINTENANCE, 0, 0)
274 };
275
276 struct kevent_qos_s kev_err = { 0 };
277
278 /* Setup workloop for mach msg rcv */
279 r = kevent_id(wlid, &kev, 1, &kev_err, 1, NULL,
280 NULL, KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_ERROR_EVENTS);
281
282 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "kevent_id");
283 T_QUIET; T_ASSERT_EQ(r, 0, "no errors returned from kevent_id");
284 }
285
286 /*
287 * Basic WL handler callback, it checks the
288 * effective Qos of the servicer thread.
289 */
290 static void
workloop_cb_test_intransit(uint64_t * workloop_id,void ** eventslist,int * events)291 workloop_cb_test_intransit(uint64_t *workloop_id, void **eventslist, int *events)
292 {
293 static bool got_peer;
294
295 struct kevent_qos_s *kev = eventslist[0];
296 mach_msg_header_t *hdr;
297 struct test_msg *tmsg;
298
299 T_LOG("Workloop handler %s called. Received message on 0x%llx",
300 __func__, *workloop_id);
301
302 /* Skip the test if we can't check Qos */
303 if (geteuid() != 0) {
304 T_SKIP("kevent_qos test requires root privileges to run.");
305 }
306
307 T_QUIET; T_ASSERT_EQ(*events, 1, "should have one event");
308
309 T_EXPECT_REQUESTED_QOS_EQ(QOS_CLASS_MAINTENANCE, "message handler should have MT requested QoS");
310
311 hdr = (mach_msg_header_t *)kev->ext[0];
312 T_ASSERT_NOTNULL(hdr, "has a message");
313 T_ASSERT_EQ(hdr->msgh_size, (uint32_t)sizeof(struct test_msg), "of the right size");
314 tmsg = (struct test_msg *)hdr;
315
316 switch (*workloop_id) {
317 case LISTENER_WLID:
318 T_LOG("Registering peer connection");
319 T_QUIET; T_ASSERT_FALSE(got_peer, "Should not have seen peer yet");
320 got_peer = true;
321 break;
322
323 case CONN_WLID:
324 T_LOG("Received message on peer");
325 break;
326
327 default:
328 T_FAIL("???");
329 }
330
331 sleep(5);
332 T_LOG("Do some CPU work.");
333 do_work(5000);
334
335 /* Check if the override now is IN + 60 boost */
336 T_EXPECT_EFFECTIVE_QOS_EQ(QOS_CLASS_USER_INITIATED,
337 "dispatch_source event handler QoS should be QOS_CLASS_USER_INITIATED");
338 T_EXPECT_EQ(get_user_promotion_basepri(), 60u,
339 "dispatch_source event handler should be overridden at 60");
340
341 T_EXPECT_EQ(get_thread_base_priority(), 60u,
342 "dispatch_source event handler should have base pri at 60");
343
344 if (*workloop_id == LISTENER_WLID) {
345 register_port(CONN_WLID, tmsg->port_descriptor.name);
346
347 kev->flags = EV_ADD | EV_ENABLE | EV_UDATA_SPECIFIC | EV_DISPATCH | EV_VANISHED;
348 kev->fflags = register_port_options();
349 kev->ext[0] = kev->ext[1] = kev->ext[2] = kev->ext[3] = 0;
350 *events = 1;
351 } else {
352 /* this will unblock the waiter */
353 mach_msg_destroy(hdr);
354 *events = 0;
355
356 /*
357 * Destroying the message will send a send-once notification for reply port, once the
358 * send-once notification is consumed (by the waiting thread), only then the actual
359 * send-once right is destroyed and only then the push will go away.
360 */
361 T_LOG("Sleeping for 5 seconds so waiting thread is unblocked\n");
362 sleep(5);
363
364 /* now that the message is destroyed, the priority should be gone */
365 T_EXPECT_EFFECTIVE_QOS_EQ(QOS_CLASS_MAINTENANCE,
366 "dispatch_source event handler QoS should be QOS_CLASS_MAINTENANCE after destroying message");
367 T_EXPECT_LE(get_user_promotion_basepri(), 0u,
368 "dispatch_source event handler should not be overridden after destroying message");
369 T_EXPECT_LE(get_thread_base_priority(), 4u,
370 "dispatch_source event handler should have base pri at 4 or less after destroying message");
371 }
372 }
373
374 static void
run_client_server(const char * server_name,const char * client_name)375 run_client_server(const char *server_name, const char *client_name)
376 {
377 dt_helper_t helpers[] = {
378 dt_launchd_helper_domain("com.apple.xnu.test.turnstile_multihop.plist",
379 server_name, NULL, LAUNCH_SYSTEM_DOMAIN),
380 dt_fork_helper(client_name)
381 };
382 dt_run_helpers(helpers, 2, HELPER_TIMEOUT_SECS);
383 }
384
385 #pragma mark Mach receive
386
387 #define TURNSTILE_MULTIHOP_SERVICE_NAME "com.apple.xnu.test.turnstile_multihop"
388
389 static mach_port_t
get_server_port(void)390 get_server_port(void)
391 {
392 mach_port_t port;
393 kern_return_t kr = bootstrap_check_in(bootstrap_port,
394 TURNSTILE_MULTIHOP_SERVICE_NAME, &port);
395 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "server bootstrap_check_in");
396 return port;
397 }
398
399 static void
send(mach_port_t send_port,mach_port_t reply_port,mach_port_t msg_port,mach_msg_priority_t priority,mach_msg_option_t options)400 send(
401 mach_port_t send_port,
402 mach_port_t reply_port,
403 mach_port_t msg_port,
404 mach_msg_priority_t priority,
405 mach_msg_option_t options)
406 {
407 kern_return_t ret = 0;
408
409 struct test_msg send_msg = {
410 .header = {
411 .msgh_remote_port = send_port,
412 .msgh_local_port = reply_port,
413 .msgh_bits = MACH_MSGH_BITS_SET(MACH_MSG_TYPE_COPY_SEND,
414 reply_port ? MACH_MSG_TYPE_MAKE_SEND_ONCE : 0,
415 MACH_MSG_TYPE_MOVE_SEND,
416 MACH_MSGH_BITS_COMPLEX),
417 .msgh_id = EXPECTED_MESSAGE_ID,
418 .msgh_size = sizeof(send_msg),
419 },
420 .body = {
421 .msgh_descriptor_count = 1,
422 },
423 .port_descriptor = {
424 .name = msg_port,
425 .disposition = MACH_MSG_TYPE_MOVE_RECEIVE,
426 .type = MACH_MSG_PORT_DESCRIPTOR,
427 },
428 };
429
430 ret = mach_msg(&(send_msg.header),
431 MACH_SEND_MSG |
432 MACH_SEND_TIMEOUT |
433 MACH_SEND_OVERRIDE |
434 (test_noimportance ? MACH_SEND_NOIMPORTANCE : 0) |
435 options,
436 send_msg.header.msgh_size,
437 0,
438 MACH_PORT_NULL,
439 10000,
440 priority);
441
442 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "client mach_msg");
443 }
444
445 static mach_msg_id_t
receive(mach_port_t rcv_port,mach_port_t notify_port)446 receive(
447 mach_port_t rcv_port,
448 mach_port_t notify_port)
449 {
450 struct {
451 mach_msg_header_t header;
452 mach_msg_body_t body;
453 mach_msg_port_descriptor_t port_descriptor;
454 } rcv_msg = {
455 .header =
456 {
457 .msgh_remote_port = MACH_PORT_NULL,
458 .msgh_local_port = rcv_port,
459 .msgh_size = sizeof(rcv_msg),
460 },
461 };
462
463 T_LOG("Client: Starting sync receive\n");
464
465 kern_return_t kr;
466 kr = mach_msg(&(rcv_msg.header),
467 MACH_RCV_MSG |
468 MACH_RCV_SYNC_WAIT,
469 0,
470 rcv_msg.header.msgh_size,
471 rcv_port,
472 0,
473 notify_port);
474
475 T_ASSERT_MACH_SUCCESS(kr, "mach_msg rcv");
476
477 return rcv_msg.header.msgh_id;
478 }
479
480 static lock_t lock_DEF;
481 static lock_t lock_IN;
482 static lock_t lock_UI;
483
484 static mach_port_t main_thread_port;
485 static mach_port_t def_thread_port;
486 static mach_port_t in_thread_port;
487 static mach_port_t ui_thread_port;
488 static mach_port_t sixty_thread_port;
489
490 static uint64_t dispatch_sync_owner;
491
492 static int
get_pri(thread_t thread_port)493 get_pri(thread_t thread_port)
494 {
495 kern_return_t kr;
496
497 thread_extended_info_data_t extended_info;
498 mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
499 kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
500 (thread_info_t)&extended_info, &count);
501
502 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
503
504 return extended_info.pth_curpri;
505 }
506
507 static void
set_thread_name(const char * fn_name)508 set_thread_name(const char *fn_name)
509 {
510 char name[50] = "";
511
512 thread_t thread_port = pthread_mach_thread_np(pthread_self());
513
514 int pri = get_pri(thread_port);
515
516 snprintf(name, sizeof(name), "%s at pri %2d", fn_name, pri);
517 pthread_setname_np(name);
518 }
519
520 static void
thread_wait_to_block(mach_port_t thread_port)521 thread_wait_to_block(mach_port_t thread_port)
522 {
523 thread_extended_info_data_t extended_info;
524 kern_return_t kr;
525
526 while (1) {
527 mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
528 kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
529 (thread_info_t)&extended_info, &count);
530
531 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
532
533 if (extended_info.pth_run_state == TH_STATE_WAITING) {
534 T_LOG("Target thread blocked\n");
535 break;
536 }
537 thread_switch(thread_port, SWITCH_OPTION_DEPRESS, 0);
538 }
539 }
540
541 static void
thread_wait_to_boost(mach_port_t thread_port,mach_port_t yield_thread,int priority)542 thread_wait_to_boost(mach_port_t thread_port, mach_port_t yield_thread, int priority)
543 {
544 thread_extended_info_data_t extended_info;
545 kern_return_t kr;
546
547 while (1) {
548 mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
549 kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
550 (thread_info_t)&extended_info, &count);
551
552 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
553
554 if (extended_info.pth_priority >= priority) {
555 T_LOG("Target thread boosted\n");
556 break;
557 }
558 thread_switch(yield_thread, SWITCH_OPTION_DEPRESS, 0);
559 }
560 }
561
562 static void
dispatch_sync_wait(mach_port_t owner_thread,qos_class_t promote_qos)563 dispatch_sync_wait(mach_port_t owner_thread, qos_class_t promote_qos)
564 {
565 struct kevent_qos_s kev_err[] = {{ 0 }};
566 uint32_t fflags = 0;
567 uint64_t mask = 0;
568 uint16_t action = 0;
569 int r;
570
571 action = EV_ADD | EV_DISABLE;
572 fflags = NOTE_WL_SYNC_WAIT | NOTE_WL_DISCOVER_OWNER;
573
574 dispatch_sync_owner = owner_thread;
575
576 struct kevent_qos_s kev[] = {{
577 .ident = mach_thread_self(),
578 .filter = EVFILT_WORKLOOP,
579 .flags = action,
580 .fflags = fflags,
581 .udata = (uintptr_t) &def_thread_port,
582 .qos = (int32_t)_pthread_qos_class_encode(promote_qos, 0, 0),
583 .ext[EV_EXTIDX_WL_MASK] = mask,
584 .ext[EV_EXTIDX_WL_VALUE] = dispatch_sync_owner,
585 .ext[EV_EXTIDX_WL_ADDR] = (uint64_t)&dispatch_sync_owner,
586 }};
587
588 /* Setup workloop to fake dispatch sync wait on a workloop */
589 r = kevent_id(30, kev, 1, kev_err, 1, NULL,
590 NULL, KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_ERROR_EVENTS);
591 T_QUIET; T_LOG("dispatch_sync_wait returned\n");
592 }
593
594 static void
dispatch_sync_cancel(mach_port_t owner_thread,qos_class_t promote_qos)595 dispatch_sync_cancel(mach_port_t owner_thread, qos_class_t promote_qos)
596 {
597 struct kevent_qos_s kev_err[] = {{ 0 }};
598 uint32_t fflags = 0;
599 uint64_t mask = 0;
600 uint16_t action = 0;
601 int r;
602
603 action = EV_DELETE | EV_ENABLE;
604 fflags = NOTE_WL_SYNC_WAKE | NOTE_WL_END_OWNERSHIP;
605
606 dispatch_sync_owner = owner_thread;
607
608 struct kevent_qos_s kev[] = {{
609 .ident = def_thread_port,
610 .filter = EVFILT_WORKLOOP,
611 .flags = action,
612 .fflags = fflags,
613 .udata = (uintptr_t) &def_thread_port,
614 .qos = (int32_t)_pthread_qos_class_encode(promote_qos, 0, 0),
615 .ext[EV_EXTIDX_WL_MASK] = mask,
616 .ext[EV_EXTIDX_WL_VALUE] = dispatch_sync_owner,
617 .ext[EV_EXTIDX_WL_ADDR] = (uint64_t)&dispatch_sync_owner,
618 }};
619
620 /* Setup workloop to fake dispatch sync wake on a workloop */
621 r = kevent_id(30, kev, 1, kev_err, 1, NULL,
622 NULL, KEVENT_FLAG_WORKLOOP | KEVENT_FLAG_ERROR_EVENTS);
623 T_QUIET; T_LOG("dispatch_sync_cancel returned\n");
624 }
625
626 static void *
thread_at_sixty(void * arg __unused)627 thread_at_sixty(void *arg __unused)
628 {
629 int policy;
630 struct sched_param param;
631 int ret;
632 void *load_token;
633 uint64_t before_lock_time, after_lock_time;
634
635 sixty_thread_port = mach_thread_self();
636
637 set_thread_name(__FUNCTION__);
638
639 /* Change our priority to 60 */
640 ret = pthread_getschedparam(pthread_self(), &policy, ¶m);
641 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_getschedparam");
642
643 param.sched_priority = 60;
644
645 ret = pthread_setschedparam(pthread_self(), policy, ¶m);
646 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_setschedparam");
647
648 ret = pthread_getschedparam(pthread_self(), &policy, ¶m);
649 T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_getschedparam");
650
651 T_LOG("My priority is %d", param.sched_priority);
652
653 thread_wait_to_boost(in_thread_port, ui_thread_port, 46);
654
655 if (spin_for_ever) {
656 /* Schedule load at Default */
657 sched_create_load_at_qos(QOS_CLASS_DEFAULT, &load_token);
658 }
659
660 T_LOG("Thread at priority 60 trying to acquire UI lock");
661
662 before_lock_time = mach_absolute_time();
663 ull_lock(&lock_UI, 3, UL_UNFAIR_LOCK, 0);
664 after_lock_time = mach_absolute_time();
665
666 T_QUIET; T_LOG("The time for priority 60 thread to acquire lock was %llu \n",
667 (after_lock_time - before_lock_time));
668
669 T_LOG("Wait for 5 seconds for the server to terminate\n");
670 sleep(5);
671 T_END;
672 }
673
674 static void *
thread_at_ui(void * arg __unused)675 thread_at_ui(void *arg __unused)
676 {
677 ui_thread_port = mach_thread_self();
678
679 set_thread_name(__FUNCTION__);
680
681 /* Grab the first ulock */
682 ull_lock(&lock_UI, 2, UL_UNFAIR_LOCK, 0);
683
684 thread_wait_to_boost(def_thread_port, in_thread_port, 37);
685 thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, thread_at_sixty);
686
687 T_EXPECT_GE(get_thread_base_priority(), 46u,
688 "thread_at_ui should have base pri 46 or greater");
689
690 T_LOG("Thread at UI priority trying to acquire IN lock");
691 ull_lock(&lock_IN, 2, UL_UNFAIR_LOCK, 0);
692 ull_unlock(&lock_UI, 2, UL_UNFAIR_LOCK, 0);
693 return NULL;
694 }
695
696 static void *
thread_at_in(void * arg __unused)697 thread_at_in(void *arg __unused)
698 {
699 in_thread_port = mach_thread_self();
700
701 set_thread_name(__FUNCTION__);
702
703 /* Grab the first ulock */
704 ull_lock(&lock_IN, 2, UL_UNFAIR_LOCK, 0);
705
706 T_LOG("Thread at IN priority got first lock ");
707
708 thread_wait_to_boost(main_thread_port, def_thread_port, 31);
709
710 /* Create a new thread at QOS_CLASS_USER_INTERACTIVE qos */
711 thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, thread_at_ui);
712
713 T_LOG("Thread at IN priority trying to acquire default lock");
714 ull_lock(&lock_DEF, 1, UL_UNFAIR_LOCK, 0);
715 ull_unlock(&lock_IN, 2, UL_UNFAIR_LOCK, 0);
716 return NULL;
717 }
718
719 static void *
thread_at_default(void * arg __unused)720 thread_at_default(void *arg __unused)
721 {
722 def_thread_port = mach_thread_self();
723
724 set_thread_name(__FUNCTION__);
725
726 /* Grab the first ulock */
727 ull_lock(&lock_DEF, 1, UL_UNFAIR_LOCK, 0);
728
729 T_LOG("Thread at DEFAULT priority got first lock ");
730
731 thread_wait_to_block(main_thread_port);
732
733 /* Create a new thread at QOS_CLASS_USER_INITIATED qos */
734 thread_create_at_qos(QOS_CLASS_USER_INITIATED, thread_at_in);
735
736 T_LOG("Thread at Default priority trying to wait on dispatch sync for maintenance thread");
737 dispatch_sync_wait(main_thread_port, QOS_CLASS_DEFAULT);
738 ull_unlock(&lock_DEF, 1, UL_UNFAIR_LOCK, 0);
739 return NULL;
740 }
741
742 static void *
thread_at_maintenance(void * arg __unused)743 thread_at_maintenance(void *arg __unused)
744 {
745 mach_port_t service_port;
746 mach_port_t conn_port;
747 mach_port_t special_reply_port;
748 mach_port_options_t opts = {
749 .flags = MPO_INSERT_SEND_RIGHT,
750 };
751
752 main_thread_port = mach_thread_self();
753
754 set_thread_name(__FUNCTION__);
755
756 kern_return_t kr = bootstrap_look_up(bootstrap_port,
757 TURNSTILE_MULTIHOP_SERVICE_NAME, &service_port);
758 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "client bootstrap_look_up");
759
760 kr = mach_port_construct(mach_task_self(), &opts, 0ull, &conn_port);
761 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "mach_port_construct");
762
763 special_reply_port = thread_get_special_reply_port();
764 T_QUIET; T_ASSERT_TRUE(MACH_PORT_VALID(special_reply_port), "get_thread_special_reply_port");
765
766 /* Become the dispatch sync owner, dispatch_sync_owner will be set in dispatch_sync_wait function */
767
768 /* Send a sync message */
769 send(conn_port, special_reply_port, MACH_PORT_NULL,
770 mach_msg_priority_encode(0, THREAD_QOS_MAINTENANCE, 0), 0);
771
772 /* Send an async checkin message */
773 send(service_port, MACH_PORT_NULL, conn_port,
774 mach_msg_priority_encode(0, THREAD_QOS_MAINTENANCE, 0), 0);
775
776 /* Create a new thread at QOS_CLASS_DEFAULT qos */
777 thread_create_at_qos(QOS_CLASS_DEFAULT, thread_at_default);
778
779 /* Block on Sync IPC */
780 mach_msg_id_t message_id = receive(special_reply_port, conn_port);
781
782 T_ASSERT_EQ(message_id, MACH_NOTIFY_SEND_ONCE, "got the expected send-once notification");
783
784 T_LOG("received reply");
785
786 dispatch_sync_cancel(def_thread_port, QOS_CLASS_DEFAULT);
787 return NULL;
788 }
789
790 T_HELPER_DECL(three_ulock_sync_ipc_hop,
791 "Create chain of 4 threads with 3 ulocks and 1 sync IPC at different qos")
792 {
793 thread_create_at_qos(QOS_CLASS_MAINTENANCE, thread_at_maintenance);
794 sigsuspend(0);
795 }
796
797 T_HELPER_DECL(three_ulock_sync_ipc_hop_noimportance,
798 "Create chain of 4 threads with 3 ulocks and 1 no-importance sync IPC at different qos")
799 {
800 test_noimportance = true;
801 thread_create_at_qos(QOS_CLASS_MAINTENANCE, thread_at_maintenance);
802 sigsuspend(0);
803 }
804
805
806 static void
thread_create_at_qos(qos_class_t qos,void * (* function)(void *))807 thread_create_at_qos(qos_class_t qos, void * (*function)(void *))
808 {
809 qos_class_t qos_thread;
810 pthread_t thread;
811 pthread_attr_t attr;
812 int ret;
813
814 ret = setpriority(PRIO_DARWIN_ROLE, 0, PRIO_DARWIN_ROLE_UI_FOCAL);
815 if (ret != 0) {
816 T_LOG("set priority failed\n");
817 }
818
819 pthread_attr_init(&attr);
820 pthread_attr_set_qos_class_np(&attr, qos, 0);
821 pthread_create(&thread, &attr, function, NULL);
822
823 T_LOG("pthread created\n");
824 pthread_get_qos_class_np(thread, &qos_thread, NULL);
825 }
826
827 #pragma mark Mach receive - kevent_qos
828
829 T_HELPER_DECL(server_kevent_id,
830 "Reply with the QoS that a dispatch source event handler ran with")
831 {
832 T_QUIET; T_ASSERT_POSIX_ZERO(_pthread_workqueue_init_with_workloop(
833 worker_cb, event_cb,
834 (pthread_workqueue_function_workloop_t)workloop_cb_test_intransit, 0, 0), NULL);
835
836 register_port(LISTENER_WLID, get_server_port());
837 sigsuspend(0);
838 T_ASSERT_FAIL("should receive a message");
839 }
840
841 #define TEST_MULTIHOP(server_name, client_name, name) \
842 T_DECL(server_kevent_id_##name, \
843 "Event delivery using a kevent_id", \
844 T_META_ASROOT(YES)) \
845 { \
846 run_client_server(server_name, client_name); \
847 }
848
849 #define TEST_MULTIHOP_SPIN(server_name, client_name, name) \
850 T_DECL(server_kevent_id_##name, \
851 "Event delivery using a kevent_id", \
852 T_META_ASROOT(YES), T_META_ENABLED(FALSE)) \
853 { \
854 spin_for_ever = true; \
855 run_client_server(server_name, client_name); \
856 spin_for_ever = false; \
857 }
858
859 /*
860 * Test 1: Test multihop priority boosting with ulocks, dispatch sync and sync IPC.
861 *
862 * Create thread's at different Qos and acquire a ulock and block on next ulock/dispatch sync
863 * creating a sync chain. The last hop the chain is blocked on Sync IPC.
864 */
865 TEST_MULTIHOP("server_kevent_id", "three_ulock_sync_ipc_hop", three_ulock_sync_ipc_hop)
866
867 TEST_MULTIHOP("server_kevent_id", "three_ulock_sync_ipc_hop_noimportance", three_ulock_sync_ipc_hop_noimportance)
868
869 /*
870 * Test 2: Test multihop priority boosting with ulocks, dispatch sync and sync IPC.
871 *
872 * Create thread's at different Qos and acquire a ulock and block on next ulock/dispatch sync
873 * creating a sync chain. The last hop the chain is blocked on Sync IPC.
874 * Before the last priority 60 thread blocks on ulock, it also starts spinforeverd at priority 31.
875 */
876 TEST_MULTIHOP_SPIN("server_kevent_id", "three_ulock_sync_ipc_hop", three_ulock_sync_ipc_hop_spin)
877