1 #include <stdio.h>
2 #include <stdlib.h>
3 #include <unistd.h>
4 #include <string.h>
5 #include <stdatomic.h>
6 #include <mach/mach.h>
7 #include <mach/mach_time.h>
8 #include <spawn.h>
9 #include <pthread.h>
10 #include <TargetConditionals.h>
11 #include <sys/sysctl.h>
12 #include <os/tsd.h>
13 #include <machine/cpu_capabilities.h>
14 #include <sys/kdebug.h>
15
16 #include <darwintest.h>
17 #include <darwintest_utils.h>
18 #include "test_utils.h"
19 #include "sched_test_utils.h"
20
21 T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler"),
22 T_META_RADAR_COMPONENT_NAME("xnu"),
23 T_META_RADAR_COMPONENT_VERSION("scheduler"),
24 T_META_BOOTARGS_SET("enable_skstb=1"),
25 T_META_ASROOT(true),
26 T_META_TAG_VM_NOT_ELIGIBLE,
27 XNU_T_META_SOC_SPECIFIC);
28
29 static void *
spin_thread(__unused void * arg)30 spin_thread(__unused void *arg)
31 {
32 spin_for_duration(8);
33 return NULL;
34 }
35
36 static void *
spin_bound_thread(void * arg)37 spin_bound_thread(void *arg)
38 {
39 char type = (char)arg;
40 bind_to_cluster_of_type(type);
41 spin_for_duration(10);
42 return NULL;
43 }
44
45 #define SPINNER_THREAD_LOAD_FACTOR (4)
46
47 T_DECL(test_cluster_bound_thread_timeshare, "Make sure the low priority bound threads get CPU in the presence of non-bound CPU spinners",
48 T_META_ENABLED(TARGET_CPU_ARM64 && TARGET_OS_OSX))
49 {
50 pthread_setname_np("main thread");
51
52 kern_return_t kr;
53
54 int rv;
55 pthread_attr_t attr;
56
57 rv = pthread_attr_init(&attr);
58 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_init");
59
60 rv = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
61 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_setdetachstate");
62
63 rv = pthread_attr_set_qos_class_np(&attr, QOS_CLASS_USER_INITIATED, 0);
64 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_set_qos_class_np");
65
66 unsigned int ncpu = (unsigned int)dt_ncpu();
67 pthread_t unbound_thread;
68 pthread_t bound_thread;
69
70 wait_for_quiescence_default(argc, argv);
71 trace_handle_t trace = begin_collect_trace(argc, argv, "test_cluster_bound_thread_timeshare");
72
73 T_LOG("creating %u non-bound threads\n", ncpu * SPINNER_THREAD_LOAD_FACTOR);
74
75 for (unsigned int i = 0; i < ncpu * SPINNER_THREAD_LOAD_FACTOR; i++) {
76 rv = pthread_create(&unbound_thread, &attr, spin_thread, NULL);
77 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_create (non-bound)");
78 }
79
80 struct sched_param param = { .sched_priority = (int)20 };
81 T_ASSERT_POSIX_ZERO(pthread_attr_setschedparam(&attr, ¶m), "pthread_attr_setschedparam");
82
83 rv = pthread_create(&bound_thread, &attr, spin_bound_thread, (void *)(uintptr_t)'P');
84 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_create (P-bound)");
85
86 rv = pthread_attr_destroy(&attr);
87 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_attr_destroy");
88
89 sleep(8);
90
91 mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT;
92 mach_port_t thread_port = pthread_mach_thread_np(bound_thread);
93 thread_basic_info_data_t bound_thread_info;
94
95 kr = thread_info(thread_port, THREAD_BASIC_INFO, (thread_info_t)&bound_thread_info, &count);
96 if (kr != KERN_SUCCESS) {
97 T_FAIL("%#x == thread_info(bound_thread, THREAD_BASIC_INFO)", kr);
98 }
99
100 end_collect_trace(trace);
101
102 uint64_t bound_usr_usec = (uint64_t)bound_thread_info.user_time.seconds * USEC_PER_SEC + (uint64_t)bound_thread_info.user_time.microseconds;
103
104 T_ASSERT_GT(bound_usr_usec, 75000ULL, "Check that bound thread got atleast 75ms CPU time");
105 T_PASS("Low priority bound threads got some CPU time in the presence of high priority unbound spinners");
106 }
107
108 static uint64_t
observe_thread_user_time(pthread_t thread,unsigned int seconds)109 observe_thread_user_time(pthread_t thread, unsigned int seconds)
110 {
111 kern_return_t kr;
112 mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT;
113 mach_port_t port = pthread_mach_thread_np(thread);
114 thread_basic_info_data_t basic_thread_info;
115 uint64_t before_user_us = 0;
116 uint64_t after_user_us = 0;
117
118 kr = thread_info(port, THREAD_BASIC_INFO, (thread_info_t)&basic_thread_info, &count);
119 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info(THREAD_BASIC_INFO)");
120 before_user_us = (uint64_t)basic_thread_info.user_time.seconds * USEC_PER_SEC +
121 (uint64_t)basic_thread_info.user_time.microseconds;
122
123 sleep(seconds);
124
125 kr = thread_info(port, THREAD_BASIC_INFO, (thread_info_t)&basic_thread_info, &count);
126 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info(THREAD_BASIC_INFO)");
127 after_user_us = (uint64_t)basic_thread_info.user_time.seconds * USEC_PER_SEC +
128 (uint64_t)basic_thread_info.user_time.microseconds;
129
130 T_QUIET; T_ASSERT_GE(after_user_us, before_user_us, "increasing user_time values");
131 return after_user_us - before_user_us;
132 }
133
134 T_DECL(cluster_soft_binding,
135 "Make sure that cluster-binding is \"soft\" and a bound thread can run elsewhere when"
136 "its bound cluster is derecommended",
137 T_META_ENABLED(TARGET_CPU_ARM64))
138 {
139 T_SETUPBEGIN;
140 if (!platform_is_amp()) {
141 T_SKIP("Platform is symmetric, skipping cluster-binding test");
142 }
143
144 wait_for_quiescence_default(argc, argv);
145
146 trace_handle_t trace = begin_collect_trace(argc, argv, "cluster_soft_binding");
147 T_SETUPEND;
148
149 for (int p = 0; p < 2; p++) {
150 /* Ensure all cores recommended */
151 char *restore_dynamic_control_args[] = {"-d", NULL};
152 execute_clpcctrl(restore_dynamic_control_args, false);
153 bool all_cores_recommended = check_recommended_core_mask(NULL);
154 T_QUIET; T_EXPECT_TRUE(all_cores_recommended, "Not all cores are recommended for scheduling");
155
156 void *arg;
157 if (p == 0) {
158 arg = (void *)'P';
159 } else {
160 arg = (void *)'E';
161 }
162 pthread_t bound_thread;
163 create_thread(&bound_thread, NULL, spin_bound_thread, arg);
164 sleep(1);
165
166 double runtime_threshold = 0.2; // Ran at least 20% of expected time
167 unsigned int observe_seconds = 3;
168 uint64_t recommended_user_us = observe_thread_user_time(bound_thread, observe_seconds);
169 T_LOG("%c-bound thread ran %lluus with all cores recommended", (char)arg, recommended_user_us);
170 T_QUIET; T_EXPECT_GE(recommended_user_us * 1.0, runtime_threshold * observe_seconds * USEC_PER_SEC,
171 "%c-bound thread didn't run at least %f of %d seconds", (char)arg, runtime_threshold, observe_seconds);
172
173 /* Derecommend the bound cluster type */
174 char *derecommend_args[] = {"-C", "X", NULL};
175 if (p == 0) {
176 derecommend_args[1] = "e";
177 } else {
178 derecommend_args[1] = "p";
179 }
180 execute_clpcctrl(derecommend_args, false);
181 check_recommended_core_mask(NULL);
182 sleep(1);
183
184 uint64_t derecommended_user_us = observe_thread_user_time(bound_thread, observe_seconds);
185 T_LOG("%c-bound thread ran %lluus with %c-cores derecommended", (char)arg, derecommended_user_us, (char)arg);
186 T_EXPECT_GE(recommended_user_us * 1.0, runtime_threshold * observe_seconds * USEC_PER_SEC,
187 "%c-bound thread ran at least %f of %d seconds when %c-cores were derecommended",
188 (char)arg, runtime_threshold, observe_seconds, (char)arg);
189 }
190
191 stop_spinning_threads();
192 end_collect_trace(trace);
193 }
194
195 static int num_cluster_bind_trials = 100000;
196
197 static void *
spin_cluster_binding(void *)198 spin_cluster_binding(void *)
199 {
200 uint8_t num_clusters = COMM_PAGE_READ(uint8_t, CPU_CLUSTERS);
201 for (int t = 0; t < num_cluster_bind_trials; t++) {
202 int bind_cluster = rand() % (num_clusters + 1);
203 bool unbind = bind_cluster == num_clusters;
204 if (unbind) {
205 bind_cluster = -1;
206 }
207 bind_to_cluster_id(bind_cluster);
208 if (!unbind) {
209 int running_on_cluster = (int)_os_cpu_cluster_number();
210 T_QUIET; T_EXPECT_EQ(running_on_cluster, bind_cluster, "Failed to reach the bound cluster");
211 if (running_on_cluster != bind_cluster) {
212 T_LOG("Failed on iteration %d", t);
213 /* Mark this failure in the recorded trace */
214 kdebug_trace(ARIADNEDBG_CODE(0, 0), (uint64_t)t, (uint64_t)bind_cluster, (uint64_t)running_on_cluster, 0);
215 }
216 }
217 }
218 return NULL;
219 }
220
221 T_DECL(cluster_bind_migrate,
222 "Ensure cluster-binding triggers a context-switch if needed to get to the bound cluster",
223 T_META_ENABLED(TARGET_CPU_ARM64),
224 T_META_MAYFAIL("rdar://132360557, need a reasonable expectation that cores will not quickly disable"))
225 {
226 T_SETUPBEGIN;
227 if (!platform_is_amp()) {
228 T_SKIP("Platform is symmetric, skipping cluster-binding test");
229 }
230
231 char *policy_name = platform_sched_policy();
232 if (strcmp(policy_name, "edge") != 0) {
233 T_SKIP("Platform is running the \"%s\" scheduler, which lacks strong enough cluster-binding", policy_name);
234 }
235
236 wait_for_quiescence_default(argc, argv);
237 bool all_cores_recommended = check_recommended_core_mask(NULL);
238 T_QUIET; T_EXPECT_TRUE(all_cores_recommended, "Not all cores are recommended for scheduling");
239
240 srand(777767777);
241
242 trace_handle_t trace = begin_collect_trace(argc, argv, "cluster_bind_migrate");
243 T_SETUPEND;
244
245 pthread_t *threads = create_threads(dt_ncpu(), 31, eJoinable, QOS_CLASS_UNSPECIFIED,
246 eSchedDefault, DEFAULT_STACK_SIZE, spin_cluster_binding, NULL);
247 for (int i = 0; i < dt_ncpu(); i++) {
248 pthread_join(threads[i], NULL);
249 }
250
251 if (T_FAILCOUNT == 0) {
252 T_PASS("Correctly migrated to the bound cluster for %d trials", num_cluster_bind_trials);
253 } else {
254 T_FAIL("%d fails for %d cluster-bind attempts", T_FAILCOUNT, num_cluster_bind_trials);
255 }
256 end_collect_trace(trace);
257 }
258