1 // Copyright (c) 2023 Apple Inc. All rights reserved.
2
3 #include <unistd.h>
4 #include <stdlib.h>
5 #include <pthread.h>
6 #include <string.h>
7 #include <mach/mach.h>
8 #include <mach/mach_time.h>
9 #include <sys/stat.h>
10 #include <sys/sysctl.h>
11 #include <sys/time.h>
12 #include <stdatomic.h>
13 #include <time.h>
14
15 #include <machine/cpu_capabilities.h>
16 #include <os/tsd.h>
17
18 #include <darwintest.h>
19 #include <darwintest_utils.h>
20 #include "test_utils.h"
21 #include "sched_test_utils.h"
22
23 T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler"),
24 T_META_RADAR_COMPONENT_NAME("xnu"),
25 T_META_RADAR_COMPONENT_VERSION("scheduler"),
26 T_META_TAG_VM_NOT_ELIGIBLE);
27
28 /*
29 * As a successor of clpc_disabling_cores_test_21636137, this test ensures that threads
30 * are naturally being scheduled on all of the logical cores (without binding). The test
31 * fails if CLPC has derecommended any cores.
32 */
33
34 static _Atomic uint64_t visited_cores_bitmask = 0;
35 static uint64_t spin_deadline_timestamp = 0;
36
37 static void *
spin_thread_fn(__unused void * arg)38 spin_thread_fn(__unused void *arg)
39 {
40 while (mach_absolute_time() < spin_deadline_timestamp) {
41 unsigned int curr_cpu = _os_cpu_number();
42 atomic_fetch_or_explicit(&visited_cores_bitmask, (1ULL << curr_cpu), memory_order_relaxed);
43 }
44 return NULL;
45 }
46
47 static void
start_threads(pthread_t * threads,void * (* start_routine)(void *),int priority,unsigned int num_threads)48 start_threads(pthread_t *threads, void *(*start_routine)(void *), int priority, unsigned int num_threads)
49 {
50 int rv;
51 pthread_attr_t attr;
52
53 rv = pthread_attr_init(&attr);
54 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_init");
55
56 for (unsigned int i = 0; i < num_threads; i++) {
57 struct sched_param param = { .sched_priority = (int)priority };
58
59 rv = pthread_attr_setschedparam(&attr, ¶m);
60 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_setschedparam");
61
62 rv = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
63 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_setdetachstate");
64
65 rv = pthread_create(&threads[i], &attr, start_routine, NULL);
66 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_create");
67 }
68
69 rv = pthread_attr_destroy(&attr);
70 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_destroy");
71 }
72
73 static host_t host;
74 static processor_port_array_t cpu_ports;
75 static mach_msg_type_number_t cpu_count;
76
77 static void
init_host_and_cpu_count(void)78 init_host_and_cpu_count(void)
79 {
80 kern_return_t kr;
81 host_t priv_host;
82
83 host = mach_host_self();
84
85 kr = host_get_host_priv_port(host, &priv_host);
86 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "host_get_host_priv_port");
87
88 kr = host_processors(priv_host, &cpu_ports, &cpu_count);
89 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "host_processors");
90
91 T_QUIET; T_ASSERT_EQ(cpu_count, (unsigned int)dt_ncpu(), "cpu counts between host_processors() and hw.ncpu don't match");
92 }
93
94 static void
record_cpu_loads(struct processor_cpu_load_info * cpu_loads)95 record_cpu_loads(struct processor_cpu_load_info *cpu_loads)
96 {
97 kern_return_t kr;
98 mach_msg_type_number_t info_count = PROCESSOR_CPU_LOAD_INFO_COUNT;
99 for (unsigned int i = 0; i < cpu_count; i++) {
100 kr = processor_info(cpu_ports[i], PROCESSOR_CPU_LOAD_INFO, &host, (processor_info_t)&cpu_loads[i], &info_count);
101 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "processor_info");
102 }
103 }
104
105 static void
cpu_loads_delta(struct processor_cpu_load_info * start_loads,struct processor_cpu_load_info * finish_loads,unsigned int * non_idle_ticks)106 cpu_loads_delta(struct processor_cpu_load_info *start_loads,
107 struct processor_cpu_load_info *finish_loads,
108 unsigned int *non_idle_ticks)
109 {
110 struct processor_cpu_load_info delta_loads[cpu_count];
111 T_LOG("Non-idle time per CPU:");
112 for (unsigned int i = 0; i < cpu_count; i++) {
113 uint64_t delta_sum = 0;
114 for (int state = CPU_STATE_USER; state < CPU_STATE_MAX; state++) {
115 T_QUIET; T_ASSERT_GE(finish_loads[i].cpu_ticks[state], start_loads[i].cpu_ticks[state], "non-monotonic ticks for state %d", state);
116 delta_loads[i].cpu_ticks[state] = finish_loads[i].cpu_ticks[state] - start_loads[i].cpu_ticks[state];
117 delta_sum += delta_loads[i].cpu_ticks[state];
118 }
119 T_QUIET; T_ASSERT_GT(delta_sum, 0ULL, "Failed to read meaningful load data for the core. Was the amfi_get_out_of_my_way=1 boot-arg missing?");
120 non_idle_ticks[i] = delta_loads[i].cpu_ticks[CPU_STATE_USER] + delta_loads[i].cpu_ticks[CPU_STATE_SYSTEM];
121 T_LOG("\tCore %d non-idle ticks: %d", i, non_idle_ticks[i]);
122 }
123 }
124
125 #define KERNEL_BOOTARGS_MAX_SIZE 1024
126 static char kernel_bootargs[KERNEL_BOOTARGS_MAX_SIZE];
127
128 static const int DEFAULT_THREAD_PRI = 31;
129
130 T_DECL(all_cores_running,
131 "Verify that we are using all available cores on the system",
132 /* Required to get around the rate limit for processor_info() */
133 T_META_BOOTARGS_SET("amfi_get_out_of_my_way=1"),
134 T_META_ASROOT(true),
135 XNU_T_META_SOC_SPECIFIC)
136 {
137 T_SETUPBEGIN;
138 int rv;
139
140 /* Warn if amfi_get_out_of_my_way is not set and fail later on if we actually run into the rate limit */
141 size_t kernel_bootargs_size = sizeof(kernel_bootargs);
142 rv = sysctlbyname("kern.bootargs", kernel_bootargs, &kernel_bootargs_size, NULL, 0);
143 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.bootargs");
144 if (strstr(kernel_bootargs, "amfi_get_out_of_my_way=1") == NULL) {
145 T_LOG("WARNING: amfi_get_out_of_my_way=1 boot-arg is missing, required to reliably capture CPU load data");
146 }
147
148 wait_for_quiescence_default();
149
150 init_host_and_cpu_count();
151 T_LOG("System has %d logical cores", cpu_count);
152
153 check_recommended_core_mask(NULL);
154
155 trace_handle_t trace_handle = begin_collect_trace("sched_all_cores_running");
156
157 T_SETUPEND;
158
159 struct processor_cpu_load_info start_cpu_loads[cpu_count];
160 record_cpu_loads(start_cpu_loads);
161
162 /* Wait 100ms for the system to settle down */
163 usleep(100000);
164
165 const uint64_t spin_seconds = 3;
166 spin_deadline_timestamp = mach_absolute_time() + nanos_to_abs(spin_seconds * NSEC_PER_SEC);
167 unsigned int num_threads = (unsigned int)dt_ncpu() * 2;
168 T_LOG("Launching %u threads to spin for %lld seconds...", num_threads, spin_seconds);
169
170 pthread_t threads[num_threads];
171 start_threads(threads, &spin_thread_fn, DEFAULT_THREAD_PRI, num_threads);
172
173 /* Wait for threads to perform spinning work */
174 sleep(spin_seconds);
175 T_LOG("...%lld seconds have elapsed", spin_seconds);
176
177 struct processor_cpu_load_info finish_cpu_loads[cpu_count];
178 record_cpu_loads(finish_cpu_loads);
179
180 uint64_t final_visited_cores_bitmask = atomic_load(&visited_cores_bitmask);
181 T_LOG("Visited cores bitmask: %llx", final_visited_cores_bitmask);
182
183 unsigned int non_idle_ticks[cpu_count];
184 cpu_loads_delta(start_cpu_loads, finish_cpu_loads, non_idle_ticks);
185
186 end_collect_trace(trace_handle);
187
188 /*
189 * Now after we have logged all of the relevant information, enforce that each
190 * of the cores was recommended and had test threads scheduled on it.
191 */
192 T_ASSERT_EQ((unsigned int)__builtin_popcountll(final_visited_cores_bitmask), cpu_count, "Each core ran at least one of the test threads");
193 for (unsigned int i = 0; i < cpu_count; i++) {
194 T_QUIET; T_ASSERT_GT(non_idle_ticks[i], 0, "One or more cores were idle during the work period");
195 }
196 T_PASS("Each core performed work during the work period");
197
198 T_END;
199 }
200
201 T_DECL(recommended_cores_mask,
202 "Tests that the mask of recommended cores includes all logical cores according to hw.ncpu",
203 XNU_T_META_SOC_SPECIFIC)
204 {
205 int ret;
206
207 uint32_t ncpu = (uint32_t)dt_ncpu();
208 T_LOG("hw.ncpu: %d\n", ncpu);
209
210 T_ASSERT_LE(ncpu, 64, "Core count isn't too high to reflect in the system's 64-bit wide core masks");
211
212 int passed_test = 0;
213 int tries = 0;
214 int MAX_RETRIES = 3;
215 while (!passed_test && tries < MAX_RETRIES) {
216 uint64_t recommended_cores_mask = 0;
217 size_t recommended_cores_mask_size = sizeof(recommended_cores_mask);
218 ret = sysctlbyname("kern.sched_recommended_cores", &recommended_cores_mask, &recommended_cores_mask_size, NULL, 0);
219 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "kern.sched_recommended_cores");
220 T_LOG("kern.sched_recommended_cores: 0x%016llx", recommended_cores_mask);
221
222 uint64_t expected_set_mask = ~0ULL >> (64 - ncpu);
223 T_LOG("Expected bits set for all cores: 0x%016llx", expected_set_mask);
224
225 if ((recommended_cores_mask & expected_set_mask) == expected_set_mask) {
226 passed_test = 1;
227 } else {
228 /*
229 * Maybe some of the cores are derecommended due to thermals.
230 * Sleep to give the system a chance to quiesce and try again.
231 */
232 unsigned int sleep_seconds = 10;
233 T_LOG("Missing expected bits. Sleeping for %u seconds before retrying", sleep_seconds);
234 sleep(sleep_seconds);
235 tries++;
236 }
237 }
238
239 T_ASSERT_EQ(passed_test, 1, "kern.sched_recommended_cores reflects that all expected cores are recommended");
240 }
241