1 // Copyright (c) 2023 Apple Inc. All rights reserved.
2
3 #include <unistd.h>
4 #include <stdlib.h>
5 #include <pthread.h>
6 #include <string.h>
7 #include <mach/mach.h>
8 #include <mach/mach_time.h>
9 #include <sys/stat.h>
10 #include <sys/sysctl.h>
11 #include <sys/time.h>
12 #include <stdatomic.h>
13 #include <time.h>
14
15 #include <machine/cpu_capabilities.h>
16 #include <os/tsd.h>
17
18 #include <darwintest.h>
19 #include <darwintest_utils.h>
20 #include "test_utils.h"
21 #include "sched_test_utils.h"
22
23 T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler"),
24 T_META_RADAR_COMPONENT_NAME("xnu"),
25 T_META_RADAR_COMPONENT_VERSION("scheduler"),
26 T_META_TAG_VM_NOT_ELIGIBLE);
27
28 /*
29 * As a successor of clpc_disabling_cores_test_21636137, this test ensures that threads
30 * are naturally being scheduled on all of the logical cores (without binding). The test
31 * fails if CLPC has derecommended any cores.
32 */
33
34 static _Atomic uint64_t visited_cores_bitmask = 0;
35 static uint64_t spin_deadline_timestamp = 0;
36
37 static void *
spin_thread_fn(__unused void * arg)38 spin_thread_fn(__unused void *arg)
39 {
40 while (mach_absolute_time() < spin_deadline_timestamp) {
41 unsigned int curr_cpu = _os_cpu_number();
42 atomic_fetch_or_explicit(&visited_cores_bitmask, (1ULL << curr_cpu), memory_order_relaxed);
43 }
44 return NULL;
45 }
46
47 static void
start_threads(pthread_t * threads,void * (* start_routine)(void *),int priority,unsigned int num_threads)48 start_threads(pthread_t *threads, void *(*start_routine)(void *), int priority, unsigned int num_threads)
49 {
50 int rv;
51 pthread_attr_t attr;
52
53 rv = pthread_attr_init(&attr);
54 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_init");
55
56 for (unsigned int i = 0; i < num_threads; i++) {
57 struct sched_param param = { .sched_priority = (int)priority };
58
59 rv = pthread_attr_setschedparam(&attr, ¶m);
60 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_setschedparam");
61
62 rv = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
63 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_setdetachstate");
64
65 rv = pthread_create(&threads[i], &attr, start_routine, NULL);
66 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_create");
67 }
68
69 rv = pthread_attr_destroy(&attr);
70 T_QUIET; T_ASSERT_POSIX_ZERO(rv, "pthread_attr_destroy");
71 }
72
73 static host_t host;
74 static processor_port_array_t cpu_ports;
75 static mach_msg_type_number_t cpu_count;
76
77 static void
init_host_and_cpu_count(void)78 init_host_and_cpu_count(void)
79 {
80 kern_return_t kr;
81 host_t priv_host;
82
83 host = mach_host_self();
84
85 kr = host_get_host_priv_port(host, &priv_host);
86 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "host_get_host_priv_port");
87
88 kr = host_processors(priv_host, &cpu_ports, &cpu_count);
89 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "host_processors");
90
91 T_QUIET; T_ASSERT_EQ(cpu_count, (unsigned int)dt_ncpu(), "cpu counts between host_processors() and hw.ncpu don't match");
92 }
93
94 static void
record_cpu_loads(struct processor_cpu_load_info * cpu_loads)95 record_cpu_loads(struct processor_cpu_load_info *cpu_loads)
96 {
97 kern_return_t kr;
98 mach_msg_type_number_t info_count = PROCESSOR_CPU_LOAD_INFO_COUNT;
99 for (unsigned int i = 0; i < cpu_count; i++) {
100 kr = processor_info(cpu_ports[i], PROCESSOR_CPU_LOAD_INFO, &host, (processor_info_t)&cpu_loads[i], &info_count);
101 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "processor_info");
102 }
103 }
104
105 static void
cpu_loads_delta(struct processor_cpu_load_info * start_loads,struct processor_cpu_load_info * finish_loads,unsigned int * non_idle_ticks)106 cpu_loads_delta(struct processor_cpu_load_info *start_loads,
107 struct processor_cpu_load_info *finish_loads,
108 unsigned int *non_idle_ticks)
109 {
110 struct processor_cpu_load_info delta_loads[cpu_count];
111 T_LOG("Non-idle time per CPU:");
112 for (unsigned int i = 0; i < cpu_count; i++) {
113 uint64_t delta_sum = 0;
114 for (int state = CPU_STATE_USER; state < CPU_STATE_MAX; state++) {
115 T_QUIET; T_ASSERT_GE(finish_loads[i].cpu_ticks[state], start_loads[i].cpu_ticks[state], "non-monotonic ticks for state %d", state);
116 delta_loads[i].cpu_ticks[state] = finish_loads[i].cpu_ticks[state] - start_loads[i].cpu_ticks[state];
117 delta_sum += delta_loads[i].cpu_ticks[state];
118 }
119 T_QUIET; T_ASSERT_GT(delta_sum, 0ULL, "Failed to read meaningful load data for the core. Was the amfi_get_out_of_my_way=1 boot-arg missing?");
120 non_idle_ticks[i] = delta_loads[i].cpu_ticks[CPU_STATE_USER] + delta_loads[i].cpu_ticks[CPU_STATE_SYSTEM];
121 T_LOG("\tCore %d non-idle ticks: %d", i, non_idle_ticks[i]);
122 }
123 }
124
125 #define KERNEL_BOOTARGS_MAX_SIZE 1024
126 static char kernel_bootargs[KERNEL_BOOTARGS_MAX_SIZE];
127
128 static const int DEFAULT_THREAD_PRI = 31;
129
130 T_DECL(all_cores_running,
131 "Verify that we are using all available cores on the system",
132 /* Required to get around the rate limit for processor_info() */
133 T_META_BOOTARGS_SET("amfi_get_out_of_my_way=1"),
134 T_META_ASROOT(true),
135 XNU_T_META_SOC_SPECIFIC,
136 T_META_ENABLED(TARGET_CPU_ARM64 /* rdar://133956403 */))
137 {
138 T_SETUPBEGIN;
139 int rv;
140
141 /* Warn if amfi_get_out_of_my_way is not set and fail later on if we actually run into the rate limit */
142 size_t kernel_bootargs_size = sizeof(kernel_bootargs);
143 rv = sysctlbyname("kern.bootargs", kernel_bootargs, &kernel_bootargs_size, NULL, 0);
144 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.bootargs");
145 if (strstr(kernel_bootargs, "amfi_get_out_of_my_way=1") == NULL) {
146 T_LOG("WARNING: amfi_get_out_of_my_way=1 boot-arg is missing, required to reliably capture CPU load data");
147 }
148
149 wait_for_quiescence_default(argc, argv);
150
151 init_host_and_cpu_count();
152 T_LOG("System has %d logical cores", cpu_count);
153
154 check_recommended_core_mask(NULL);
155
156 trace_handle_t trace_handle = begin_collect_trace(argc, argv, "sched_all_cores_running");
157
158 T_SETUPEND;
159
160 struct processor_cpu_load_info start_cpu_loads[cpu_count];
161 record_cpu_loads(start_cpu_loads);
162
163 /* Wait 100ms for the system to settle down */
164 usleep(100000);
165
166 const uint64_t spin_seconds = 3;
167 spin_deadline_timestamp = mach_absolute_time() + nanos_to_abs(spin_seconds * NSEC_PER_SEC);
168 unsigned int num_threads = (unsigned int)dt_ncpu() * 2;
169 T_LOG("Launching %u threads to spin for %lld seconds...", num_threads, spin_seconds);
170
171 pthread_t threads[num_threads];
172 start_threads(threads, &spin_thread_fn, DEFAULT_THREAD_PRI, num_threads);
173
174 /* Wait for threads to perform spinning work */
175 sleep(spin_seconds);
176 T_LOG("...%lld seconds have elapsed", spin_seconds);
177
178 struct processor_cpu_load_info finish_cpu_loads[cpu_count];
179 record_cpu_loads(finish_cpu_loads);
180
181 uint64_t final_visited_cores_bitmask = atomic_load(&visited_cores_bitmask);
182 T_LOG("Visited cores bitmask: %llx", final_visited_cores_bitmask);
183
184 unsigned int non_idle_ticks[cpu_count];
185 cpu_loads_delta(start_cpu_loads, finish_cpu_loads, non_idle_ticks);
186
187 end_collect_trace(trace_handle);
188
189 /*
190 * Now after we have logged all of the relevant information, enforce that each
191 * of the cores was recommended and had test threads scheduled on it.
192 */
193 T_ASSERT_EQ((unsigned int)__builtin_popcountll(final_visited_cores_bitmask), cpu_count, "Each core ran at least one of the test threads");
194 for (unsigned int i = 0; i < cpu_count; i++) {
195 T_QUIET; T_ASSERT_GT(non_idle_ticks[i], 0, "One or more cores were idle during the work period");
196 }
197 T_PASS("Each core performed work during the work period");
198
199 T_END;
200 }
201
202 T_DECL(recommended_cores_mask,
203 "Tests that the mask of recommended cores includes all logical cores according to hw.ncpu",
204 XNU_T_META_SOC_SPECIFIC)
205 {
206 int ret;
207
208 uint32_t ncpu = (uint32_t)dt_ncpu();
209 T_LOG("hw.ncpu: %d\n", ncpu);
210
211 T_ASSERT_LE(ncpu, 64, "Core count isn't too high to reflect in the system's 64-bit wide core masks");
212
213 int passed_test = 0;
214 int tries = 0;
215 int MAX_RETRIES = 3;
216 while (!passed_test && tries < MAX_RETRIES) {
217 uint64_t recommended_cores_mask = 0;
218 size_t recommended_cores_mask_size = sizeof(recommended_cores_mask);
219 ret = sysctlbyname("kern.sched_recommended_cores", &recommended_cores_mask, &recommended_cores_mask_size, NULL, 0);
220 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "kern.sched_recommended_cores");
221 T_LOG("kern.sched_recommended_cores: 0x%016llx", recommended_cores_mask);
222
223 uint64_t expected_set_mask = ~0ULL >> (64 - ncpu);
224 T_LOG("Expected bits set for all cores: 0x%016llx", expected_set_mask);
225
226 if ((recommended_cores_mask & expected_set_mask) == expected_set_mask) {
227 passed_test = 1;
228 } else {
229 /*
230 * Maybe some of the cores are derecommended due to thermals.
231 * Sleep to give the system a chance to quiesce and try again.
232 */
233 unsigned int sleep_seconds = 10;
234 T_LOG("Missing expected bits. Sleeping for %u seconds before retrying", sleep_seconds);
235 sleep(sleep_seconds);
236 tries++;
237 }
238 }
239
240 T_ASSERT_EQ(passed_test, 1, "kern.sched_recommended_cores reflects that all expected cores are recommended");
241 }
242