1 /*
2 * Tests to validate that:
3 * - we can schedule threads on all hw.ncpus cores according to _os_cpu_number
4 * - we can schedule threads on all hw.cpuclusters clusters according to _os_cpu_cluster_number
5 * - the cluster id returned by _os_cpu_cluster_number aligns with mappings from IORegistry
6 *
7 * <rdar://problem/29545645>
8 * <rdar://problem/30445216>
9 *
10 * xcrun -sdk macosx.internal clang -o cpucount cpucount.c -ldarwintest -framework IOKit -framework CoreFoundation -g -Weverything
11 * xcrun -sdk iphoneos.internal clang -arch arm64 -o cpucount-ios cpucount.c -ldarwintest -framework IOKit -framework CoreFoundation -g -Weverything
12 * xcrun -sdk macosx.internal clang -o cpucount cpucount.c -ldarwintest -framework IOKit -framework CoreFoundation -arch arm64e -Weverything
13 */
14
15 #include <darwintest.h>
16 #include "test_utils.h"
17
18 #include <stdio.h>
19 #include <stdlib.h>
20 #include <unistd.h>
21 #include <pthread.h>
22 #include <sys/commpage.h>
23 #include <sys/sysctl.h>
24 #include <sys/proc_info.h>
25 #include <libproc.h>
26
27 #include <CoreFoundation/CoreFoundation.h>
28 #include <IOKit/IOKitLib.h>
29
30 #include <mach/mach.h>
31 #include <mach/mach_time.h>
32 #include <machine/cpu_capabilities.h>
33
34 #include <os/tsd.h> /* private header for _os_cpu_number, _os_cpu_cluster_number */
35
36 T_GLOBAL_META(
37 T_META_RUN_CONCURRENTLY(false),
38 T_META_BOOTARGS_SET("enable_skstb=1"),
39 T_META_CHECK_LEAKS(false),
40 T_META_ASROOT(true),
41 T_META_ALL_VALID_ARCHS(true),
42 T_META_RADAR_COMPONENT_NAME("xnu"),
43 T_META_RADAR_COMPONENT_VERSION("scheduler"),
44 T_META_OWNER("jarrad")
45 );
46
47 #define KERNEL_BOOTARGS_MAX_SIZE 1024
48 static char kernel_bootargs[KERNEL_BOOTARGS_MAX_SIZE];
49
50 #define KERNEL_VERSION_MAX_SIZE 1024
51 static char kernel_version[KERNEL_VERSION_MAX_SIZE];
52
53 static mach_timebase_info_data_t timebase_info;
54
55 // Source: libktrace:corefoundation_helpers.c
56
57 static void
dict_number_internal(CFDictionaryRef dict,CFStringRef key,void * dst_out,CFNumberType nbr_type)58 dict_number_internal(CFDictionaryRef dict, CFStringRef key, void *dst_out, CFNumberType nbr_type)
59 {
60 bool success;
61 T_QUIET; T_ASSERT_NOTNULL(dict, "dict must not be null");
62 T_QUIET; T_ASSERT_NOTNULL(key, " key must not be null");
63 T_QUIET; T_ASSERT_NOTNULL(dst_out, "dst out must not be null");
64
65 CFTypeRef val = CFDictionaryGetValue(dict, key);
66 T_QUIET; T_ASSERT_NOTNULL(val, "unable to get value for key %s", CFStringGetCStringPtr(key, kCFStringEncodingASCII));
67
68 CFTypeID type = CFGetTypeID(val);
69 if (type == CFNumberGetTypeID()) {
70 CFNumberRef val_nbr = (CFNumberRef)val;
71 success = CFNumberGetValue(val_nbr, nbr_type, dst_out);
72 T_QUIET; T_ASSERT_TRUE(success, "dictionary number at key '%s' is not the right type", CFStringGetCStringPtr(key, kCFStringEncodingASCII));
73 } else if (type == CFDataGetTypeID()) {
74 CFDataRef val_data = (CFDataRef)val;
75 size_t raw_size = (size_t)CFDataGetLength(val_data);
76 T_QUIET; T_ASSERT_EQ(raw_size, (size_t)4, "cannot convert CFData of size %zu to number", raw_size);
77 CFDataGetBytes(val_data, CFRangeMake(0, (CFIndex)raw_size), dst_out);
78 } else {
79 T_ASSERT_FAIL("dictionary value at key '%s' should be a number or data", CFStringGetCStringPtr(key, kCFStringEncodingASCII));
80 }
81 }
82
83 static void
dict_uint32(CFDictionaryRef dict,CFStringRef key,uint32_t * dst_out)84 dict_uint32(CFDictionaryRef dict, CFStringRef key, uint32_t *dst_out)
85 {
86 dict_number_internal(dict, key, dst_out, kCFNumberSInt32Type);
87 }
88
89 static uint64_t
abs_to_nanos(uint64_t abs)90 abs_to_nanos(uint64_t abs)
91 {
92 return abs * timebase_info.numer / timebase_info.denom;
93 }
94
95 static int32_t
get_csw_count(void)96 get_csw_count(void)
97 {
98 struct proc_taskinfo taskinfo;
99 int rv;
100
101 rv = proc_pidinfo(getpid(), PROC_PIDTASKINFO, 0, &taskinfo, sizeof(taskinfo));
102 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "PROC_PIDTASKINFO");
103
104 return taskinfo.pti_csw;
105 }
106
107 // noinline hopefully keeps the optimizer from hoisting it out of the loop
108 // until rdar://68253516 is fixed.
109 __attribute__((noinline))
110 static uint32_t
fixed_os_cpu_number(void)111 fixed_os_cpu_number(void)
112 {
113 uint32_t cpu_number = _os_cpu_number();
114 return cpu_number;
115 }
116
117 static unsigned int
commpage_cpu_cluster_number(void)118 commpage_cpu_cluster_number(void)
119 {
120 uint8_t cpu_number = (uint8_t)fixed_os_cpu_number();
121 volatile uint8_t *cpu_to_cluster = COMM_PAGE_SLOT(uint8_t, CPU_TO_CLUSTER);
122 return (unsigned int)*(cpu_to_cluster + cpu_number);
123 }
124
125 static void
cpucount_setup(void)126 cpucount_setup(void)
127 {
128 int rv;
129 kern_return_t kr;
130
131 T_SETUPBEGIN;
132
133 setvbuf(stdout, NULL, _IONBF, 0);
134 setvbuf(stderr, NULL, _IONBF, 0);
135
136 /* Validate what kind of kernel we're on */
137 size_t kernel_version_size = sizeof(kernel_version);
138 rv = sysctlbyname("kern.version", kernel_version, &kernel_version_size, NULL, 0);
139 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.version");
140
141 T_LOG("kern.version: %s\n", kernel_version);
142
143 /* Double check that darwintest set the boot arg we requested */
144 size_t kernel_bootargs_size = sizeof(kernel_bootargs);
145 rv = sysctlbyname("kern.bootargs", kernel_bootargs, &kernel_bootargs_size, NULL, 0);
146 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.bootargs");
147
148 T_LOG("kern.bootargs: %s\n", kernel_bootargs);
149
150 if (NULL == strstr(kernel_bootargs, "enable_skstb=1")) {
151 T_ASSERT_FAIL("enable_skstb=1 boot-arg is missing");
152 }
153
154 kr = mach_timebase_info(&timebase_info);
155 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "mach_timebase_info");
156
157 struct sched_param param = {.sched_priority = 63};
158
159 rv = pthread_setschedparam(pthread_self(), SCHED_FIFO, ¶m);
160 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "pthread_setschedparam");
161
162 T_SETUPEND;
163 }
164
165
166 T_DECL(count_cpus,
167 "Tests we can schedule bound threads on all hw.ncpus cores and that _os_cpu_number matches",
168 XNU_T_META_SOC_SPECIFIC)
169 {
170 int rv;
171
172 cpucount_setup();
173
174 int bound_cpu_out = 0;
175 size_t bound_cpu_out_size = sizeof(bound_cpu_out);
176 rv = sysctlbyname("kern.sched_thread_bind_cpu", &bound_cpu_out, &bound_cpu_out_size, NULL, 0);
177
178 if (rv == -1) {
179 if (errno == ENOENT) {
180 T_ASSERT_FAIL("kern.sched_thread_bind_cpu doesn't exist, must set enable_skstb=1 boot-arg on development kernel");
181 }
182 if (errno == EPERM) {
183 T_ASSERT_FAIL("must run as root");
184 }
185 }
186
187 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cpu");
188 T_QUIET; T_ASSERT_EQ(bound_cpu_out, -1, "kern.sched_thread_bind_cpu should exist, start unbound");
189
190 uint32_t sysctl_ncpu = 0;
191 size_t ncpu_size = sizeof(sysctl_ncpu);
192 rv = sysctlbyname("hw.ncpu", &sysctl_ncpu, &ncpu_size, NULL, 0);
193 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "sysctlbyname(hw.ncpu)");
194
195 T_LOG("hw.ncpu: %2d\n", sysctl_ncpu);
196
197 T_ASSERT_GT(sysctl_ncpu, 0, "at least one CPU exists");
198
199 for (uint32_t cpu_to_bind = 0; cpu_to_bind < sysctl_ncpu; cpu_to_bind++) {
200 int32_t before_csw_count = get_csw_count();
201 T_LOG("(csw %4d) attempting to bind to cpu %2d\n", before_csw_count, cpu_to_bind);
202
203 uint64_t start = mach_absolute_time();
204
205 rv = sysctlbyname("kern.sched_thread_bind_cpu", NULL, 0, &cpu_to_bind, sizeof(cpu_to_bind));
206
207 uint64_t end = mach_absolute_time();
208
209 if (rv == -1 && errno == ENOTSUP) {
210 T_SKIP("Binding is available, but this process doesn't support binding (e.g. Rosetta on Aruba)");
211 }
212
213 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.sched_thread_bind_cpu(%u)", cpu_to_bind);
214
215 uint32_t os_cpu_number_reported = fixed_os_cpu_number();
216
217 bound_cpu_out = 0;
218 rv = sysctlbyname("kern.sched_thread_bind_cpu", &bound_cpu_out, &bound_cpu_out_size, NULL, 0);
219 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cpu");
220
221 T_QUIET; T_EXPECT_EQ((int)cpu_to_bind, bound_cpu_out,
222 "should report bound cpu id matching requested bind target");
223
224 uint64_t delta_abs = end - start;
225 uint64_t delta_ns = abs_to_nanos(delta_abs);
226
227 int32_t after_csw_count = get_csw_count();
228
229 T_LOG("(csw %4d) bound to cpu %2d in %f milliseconds\n",
230 after_csw_count, cpu_to_bind,
231 ((double)delta_ns / 1000000.0));
232
233 if (cpu_to_bind > 0) {
234 T_QUIET; T_EXPECT_LT(before_csw_count, after_csw_count,
235 "should have had to context switch to execute the bind");
236 }
237
238 T_LOG("cpu %2d reported id %2d\n",
239 cpu_to_bind, os_cpu_number_reported);
240
241 T_QUIET;
242 T_EXPECT_EQ(cpu_to_bind, os_cpu_number_reported,
243 "should report same CPU number as was bound to");
244 }
245
246 int unbind = -1; /* pass -1 in order to unbind the thread */
247
248 rv = sysctlbyname("kern.sched_thread_bind_cpu", NULL, 0, &unbind, sizeof(unbind));
249
250 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.sched_thread_bind_cpu(%u)", unbind);
251
252 rv = sysctlbyname("kern.sched_thread_bind_cpu", &bound_cpu_out, &bound_cpu_out_size, NULL, 0);
253
254 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cpu");
255 T_QUIET; T_ASSERT_EQ(bound_cpu_out, -1, "thread should be unbound at the end");
256
257 T_PASS("test has run threads on all CPUS");
258 }
259
260 T_DECL(count_clusters,
261 "Tests we can schedule bound threads on all cpu clusters and that _os_cpu_cluster_number matches",
262 XNU_T_META_SOC_SPECIFIC)
263 {
264 int rv;
265
266 cpucount_setup();
267
268 uint8_t cpuclusters = COMM_PAGE_READ(uint8_t, CPU_CLUSTERS);
269 T_LOG("cpuclusters: %2d\n", cpuclusters);
270 T_QUIET; T_ASSERT_GT(cpuclusters, 0, "at least one CPU cluster exists");
271 if (cpuclusters == 1) {
272 T_SKIP("Test is unsupported on non-AMP platforms");
273 }
274
275 uint32_t sysctl_ncpu = 0;
276 size_t ncpu_size = sizeof(sysctl_ncpu);
277 rv = sysctlbyname("hw.ncpu", &sysctl_ncpu, &ncpu_size, NULL, 0);
278 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "sysctlbyname(hw.ncpu)");
279 T_LOG("hw.ncpu: %2d\n", sysctl_ncpu);
280
281 uint64_t recommended_cores = 0;
282 size_t recommended_cores_size = sizeof(recommended_cores);
283 rv = sysctlbyname("kern.sched_recommended_cores", &recommended_cores, &recommended_cores_size, NULL, 0);
284 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "sysctlbyname(kern.sched_recommended_cores)");
285 T_LOG("kern.sched_recommended_cores: %llu", recommended_cores);
286 if ((uint32_t)__builtin_popcountll(recommended_cores) != sysctl_ncpu) {
287 T_SKIP("Missing recommended cores");
288 }
289
290 int bound_cluster_out = 0;
291 size_t bound_cluster_out_size = sizeof(bound_cluster_out);
292 rv = sysctlbyname("kern.sched_thread_bind_cluster_id", &bound_cluster_out, &bound_cluster_out_size, NULL, 0);
293
294 if (rv == -1) {
295 if (errno == ENOENT) {
296 T_ASSERT_FAIL("kern.sched_thread_bind_cluster_id doesn't exist, must set enable_skstb=1 boot-arg on development kernel");
297 }
298 if (errno == EPERM) {
299 T_ASSERT_FAIL("must run as root");
300 }
301 }
302
303 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cluster_id");
304 T_QUIET; T_ASSERT_EQ(bound_cluster_out, -1, "kern.sched_thread_bind_cluster_id should exist, start unbound");
305
306 for (uint32_t cluster_to_bind = 0; cluster_to_bind < cpuclusters; cluster_to_bind++) {
307 int32_t before_csw_count = get_csw_count();
308 T_LOG("(csw %4d) attempting to bind to cluster %2d\n", before_csw_count, cluster_to_bind);
309
310 uint64_t start = mach_absolute_time();
311
312 rv = sysctlbyname("kern.sched_thread_bind_cluster_id", NULL, 0, &cluster_to_bind, sizeof(cluster_to_bind));
313
314 uint64_t end = mach_absolute_time();
315
316 if (rv == -1 && errno == ENOTSUP) {
317 T_SKIP("Binding is available, but this process doesn't support binding (e.g. Rosetta on Aruba)");
318 }
319
320 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.sched_thread_bind_cluster_id(%u)", cluster_to_bind);
321
322 T_LOG("CPU ID: %d", fixed_os_cpu_number());
323
324 #if TARGET_CPU_X86_64
325 T_LOG("_os_cpu_cluster_number unsupported under x86.");
326 #else
327 unsigned int os_cluster_number_reported = _os_cpu_cluster_number();
328 T_LOG("OS reported cluster number: %2d\n",
329 os_cluster_number_reported);
330 T_QUIET; T_EXPECT_EQ(cluster_to_bind, os_cluster_number_reported,
331 "_os_cpu_cluster_number should report same cluster number as was bound to");
332 #endif
333
334 unsigned int commpage_cluster_number_reported = commpage_cpu_cluster_number();
335 T_LOG("Comm Page reported cluster number: %u", commpage_cluster_number_reported);
336 T_EXPECT_EQ(commpage_cluster_number_reported, cluster_to_bind, "comm page cluster number matches commpage for this CPU");
337
338 bound_cluster_out = 0;
339 rv = sysctlbyname("kern.sched_thread_bind_cluster_id", &bound_cluster_out, &bound_cluster_out_size, NULL, 0);
340 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cluster_id");
341
342 T_QUIET; T_EXPECT_EQ((int)cluster_to_bind, bound_cluster_out,
343 "bound cluster id matches requested bind target");
344
345 uint64_t delta_abs = end - start;
346 uint64_t delta_ns = abs_to_nanos(delta_abs);
347
348 int32_t after_csw_count = get_csw_count();
349
350 T_LOG("(csw %4d) bound to cluster %2d in %f milliseconds\n",
351 after_csw_count, cluster_to_bind,
352 ((double)delta_ns / 1000000.0));
353
354 if (cluster_to_bind > 0) {
355 T_QUIET; T_EXPECT_LT(before_csw_count, after_csw_count,
356 "should have had to context switch to execute the bind");
357 }
358 }
359
360 int unbind = -1; /* pass -1 in order to unbind the thread */
361
362 rv = sysctlbyname("kern.sched_thread_bind_cluster_id", NULL, 0, &unbind, sizeof(unbind));
363
364 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.sched_thread_bind_cluster_id(%u)", unbind);
365
366 rv = sysctlbyname("kern.sched_thread_bind_cluster_id", &bound_cluster_out, &bound_cluster_out_size, NULL, 0);
367
368 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cluster_id");
369 T_QUIET; T_ASSERT_EQ(bound_cluster_out, -1, "thread should be unbound at the end");
370
371 T_PASS("test has run threads on all clusters");
372 }
373
374 T_DECL(check_cpu_topology,
375 "Verify _os_cpu_cluster_number(), _os_cpu_number() against IORegistry",
376 XNU_T_META_SOC_SPECIFIC,
377 T_META_ENABLED(TARGET_CPU_ARM || TARGET_CPU_ARM64))
378 {
379 int rv;
380 uint32_t cpu_id, cluster_id;
381 kern_return_t kr;
382 io_iterator_t cpus_iter = 0;
383 io_service_t cpus_service = 0;
384 io_service_t cpu_service = 0;
385 CFDictionaryRef match = NULL;
386
387 cpucount_setup();
388
389 int bound_cpu_out = 0;
390 size_t bound_cpu_out_size = sizeof(bound_cpu_out);
391 rv = sysctlbyname("kern.sched_thread_bind_cpu", &bound_cpu_out, &bound_cpu_out_size, NULL, 0);
392
393 if (rv == -1) {
394 if (errno == ENOENT) {
395 T_FAIL("kern.sched_thread_bind_cpu doesn't exist, must set enable_skstb=1 boot-arg on development kernel");
396 }
397 if (errno == EPERM) {
398 T_FAIL("must run as root");
399 }
400 }
401
402 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "read kern.sched_thread_bind_cpu");
403 T_QUIET; T_ASSERT_EQ(bound_cpu_out, -1, "kern.sched_thread_bind_cpu should exist, start unbound");
404
405 match = IOServiceNameMatching("cpus");
406 cpus_service = IOServiceGetMatchingService(kIOMainPortDefault, match);
407 match = NULL; // consumes reference to match
408 T_QUIET; T_ASSERT_NE(cpus_service, (io_service_t)0, "Failed get cpus IOService");
409
410 kr = IORegistryEntryGetChildIterator(cpus_service, "IODeviceTree", &cpus_iter);
411 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "IORegistryEntryGetChildIterator");
412
413 while ((cpu_service = IOIteratorNext(cpus_iter)) != 0) {
414 CFMutableDictionaryRef props = NULL;
415 kr = IORegistryEntryCreateCFProperties(cpu_service, &props, kCFAllocatorDefault, 0);
416 T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "IORegistryEntryCreateCFProperties");
417
418 dict_uint32(props, CFSTR("logical-cpu-id"), &cpu_id);
419 T_LOG("IORegistry logical cpu id: %u", cpu_id);
420 dict_uint32(props, CFSTR("logical-cluster-id"), &cluster_id);
421 T_LOG("IORegistry logical cpu cluster id: %u", cluster_id);
422
423 T_LOG("Binding thread to cpu %u", cpu_id);
424 rv = sysctlbyname("kern.sched_thread_bind_cpu", NULL, 0, &cpu_id, sizeof(cpu_id));
425 if (rv == -1 && errno == ENOTSUP) {
426 T_SKIP("Binding is available, but this process doesn't support binding (e.g. Rosetta on Aruba)");
427 }
428 T_QUIET; T_ASSERT_POSIX_SUCCESS(rv, "kern.sched_thread_bind_cpu(%u)", cpu_id);
429
430 unsigned int os_cpu_number_reported = fixed_os_cpu_number();
431 T_EXPECT_EQ(os_cpu_number_reported, cpu_id, "_os_cpu_number matches IORegistry entry for this CPU");
432 unsigned int os_cluster_number_reported = _os_cpu_cluster_number();
433 T_EXPECT_EQ(os_cluster_number_reported, cluster_id, "_os_cpu_cluster_number matches IORegistry entry for this CPU");
434 unsigned int commpage_cluster_number_reported = commpage_cpu_cluster_number();
435 T_EXPECT_EQ(commpage_cluster_number_reported, cluster_id, "comm page cluster number matches IORegistry entry for this CPU");
436
437 CFRelease(props);
438 IOObjectRelease(cpu_service);
439 }
440 T_PASS("All cluster IDs match with IORegistry");
441 }
442