1 // Copyright (c) 2021-2022 Apple Inc. All rights reserved.
2
3 #include <darwintest.h>
4 #include "test_utils.h"
5 #include <fcntl.h>
6 #include <inttypes.h>
7 #ifndef PRIVATE
8 /*
9 * Need new CPU families.
10 */
11 #define PRIVATE
12 #include <mach/machine.h>
13 #undef PRIVATE
14 #else /* !defined(PRIVATE) */
15 #include <mach/machine.h>
16 #endif /* defined(PRIVATE) */
17 #include <stdint.h>
18 #include <System/sys/guarded.h>
19 #include <System/sys/monotonic.h>
20 #include <sys/ioctl.h>
21 #include <sys/sysctl.h>
22 #include <unistd.h>
23
24 T_GLOBAL_META(
25 T_META_NAMESPACE("xnu.monotonic"),
26 T_META_CHECK_LEAKS(false),
27 XNU_T_META_REQUIRES_DEVELOPMENT_KERNEL
28 );
29
30 static bool
device_supports_uncore(void)31 device_supports_uncore(void)
32 {
33 int r;
34 int type, subtype;
35 unsigned int family;
36 size_t size = sizeof(type);
37
38 /*
39 * Only arm64 Monsoon devices support uncore counters.
40 */
41
42 r = sysctlbyname("hw.cputype", &type, &size, NULL, 0);
43 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cputype\")");
44 r = sysctlbyname("hw.cpusubtype", &subtype, &size, NULL, 0);
45 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cpusubtype\")");
46 r = sysctlbyname("hw.cpufamily", &family, &size, NULL, 0);
47 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cpufamily\")");
48
49 if (type == CPU_TYPE_ARM64 &&
50 subtype == CPU_SUBTYPE_ARM64_V8 &&
51 (family == CPUFAMILY_ARM_MONSOON_MISTRAL ||
52 family == CPUFAMILY_ARM_VORTEX_TEMPEST)) {
53 return true;
54 }
55
56 return false;
57 }
58
59 #define UNCORE_DEV_PATH "/dev/monotonic/uncore"
60
61 static int
open_uncore_error(int * error)62 open_uncore_error(int *error)
63 {
64 guardid_t guard;
65 int fd;
66
67 guard = 0xa5adcafe;
68
69 T_SETUPBEGIN;
70
71 fd = guarded_open_np(UNCORE_DEV_PATH, &guard,
72 GUARD_CLOSE | GUARD_DUP | GUARD_WRITE, O_CLOEXEC | O_EXCL);
73 if (fd < 0 && errno == ENOENT) {
74 T_ASSERT_FALSE(device_supports_uncore(),
75 "lack of dev node implies no uncore support");
76 T_SKIP("uncore counters are unsupported");
77 __builtin_unreachable();
78 }
79
80 if (error == NULL) {
81 T_ASSERT_POSIX_SUCCESS(fd, "open '%s'", UNCORE_DEV_PATH);
82 } else {
83 *error = errno;
84 }
85
86 T_SETUPEND;
87
88 return fd;
89 }
90
91 static void
uncore_counts(int fd,uint64_t ctr_mask,uint64_t * counts)92 uncore_counts(int fd, uint64_t ctr_mask, uint64_t *counts)
93 {
94 int r;
95 union monotonic_ctl_counts *cts_ctl;
96
97 cts_ctl = (union monotonic_ctl_counts *)counts;
98 cts_ctl->in.ctr_mask = ctr_mask;
99
100 r = ioctl(fd, MT_IOC_COUNTS, cts_ctl);
101 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "MT_IOC_COUNTS got counter values");
102 }
103
104 #define REF_TIMEBASE_EVENT 0x3
105 #define CTRS_MAX 128
106
107 T_DECL(uncore_max_counters,
108 "ensure that the maximum number of uncore countes is sane",
109 XNU_T_META_SOC_SPECIFIC,
110 T_META_ASROOT(true))
111 {
112 int nctrs = 0;
113 int fd;
114
115 fd = open_uncore_error(NULL);
116
117 do {
118 union monotonic_ctl_add add_ctl;
119 int r;
120
121 add_ctl.in.config.event = REF_TIMEBASE_EVENT;
122 add_ctl.in.config.allowed_ctr_mask = UINT64_MAX;
123
124 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
125 if (r < 0 && errno == E2BIG) {
126 break;
127 }
128
129 T_QUIET;
130 T_ASSERT_POSIX_SUCCESS(r, "added reference timebase event to counters");
131 nctrs++;
132 } while (nctrs < CTRS_MAX);
133
134 T_EXPECT_LT(nctrs, CTRS_MAX, "able to allocate %d uncore PMCs", nctrs);
135 }
136
137 static uint32_t
uncore_add(int fd,uint64_t event,uint64_t allowed_ctrs,int error)138 uncore_add(int fd, uint64_t event, uint64_t allowed_ctrs, int error)
139 {
140 int save_errno;
141 int r;
142 uint32_t ctr;
143 union monotonic_ctl_add add_ctl;
144
145 add_ctl.in.config.event = event;
146 add_ctl.in.config.allowed_ctr_mask = allowed_ctrs;
147 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
148 if (error) {
149 save_errno = errno;
150 T_EXPECT_LT(r, 0, "adding event to counter should fail");
151 T_EXPECT_EQ(save_errno, error,
152 "adding event to counter should fail with %d: %s",
153 error, strerror(error));
154 return UINT32_MAX;
155 } else {
156 T_QUIET;
157 T_ASSERT_POSIX_SUCCESS(r,
158 "added event %#" PRIx64 " to counters", event);
159 }
160
161 ctr = add_ctl.out.ctr;
162 T_QUIET; T_ASSERT_LT(ctr, (uint32_t)CTRS_MAX, "counter returned should be sane");
163 return ctr;
164 }
165
166 T_DECL(uncore_collision,
167 "ensure that trying to add an event on the same counter fails",
168 T_META_ASROOT(true))
169 {
170 int fd;
171 uint32_t ctr;
172
173 fd = open_uncore_error(NULL);
174
175 ctr = uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, 0);
176 T_LOG("added event to uncore counter %d\n", ctr);
177
178 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1) << ctr, ENOSPC);
179 }
180
181 static void
uncore_enable(int fd)182 uncore_enable(int fd)
183 {
184 union monotonic_ctl_enable en_ctl = {
185 .in = { .enable = true }
186 };
187
188 T_ASSERT_POSIX_SUCCESS(ioctl(fd, MT_IOC_ENABLE, &en_ctl),
189 "enabling counters");
190 }
191
192 T_DECL(uncore_enabled_busy,
193 "ensure that trying to add an event while enabled fails",
194 T_META_ASROOT(true))
195 {
196 int fd;
197
198 fd = open_uncore_error(NULL);
199
200 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, 0);
201
202 uncore_enable(fd);
203 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, EBUSY);
204 }
205
206 T_DECL(uncore_reset,
207 "ensure that resetting the counters works")
208 {
209 int fd;
210 int r;
211
212 fd = open_uncore_error(NULL);
213
214 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), 0);
215 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), ENOSPC);
216
217 r = ioctl(fd, MT_IOC_RESET);
218 T_ASSERT_POSIX_SUCCESS(r, "resetting succeeds");
219
220 T_LOG("adding event to same counter after reset");
221 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), 0);
222 }
223
224 #define SLEEP_USECS (500 * 1000)
225
226 static int
uncore_add_all(int fd,uint64_t event,int * nmonitors)227 uncore_add_all(int fd, uint64_t event, int *nmonitors)
228 {
229 int nctrs = 0;
230 int r;
231
232 do {
233 union monotonic_ctl_add add_ctl;
234
235 add_ctl.in.config.event = event;
236 add_ctl.in.config.allowed_ctr_mask = UINT64_MAX;
237
238 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
239 if (r < 0 && errno == E2BIG) {
240 break;
241 }
242
243 T_QUIET;
244 T_ASSERT_POSIX_SUCCESS(r, "added event %#" PRIx64 " to counters",
245 event);
246 nctrs++;
247 } while (nctrs < CTRS_MAX);
248
249 if (nmonitors) {
250 union monotonic_ctl_info info_ctl;
251 r = ioctl(fd, MT_IOC_GET_INFO, &info_ctl);
252 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "got info about uncore counters");
253
254 *nmonitors = (int)info_ctl.out.nmonitors;
255 }
256
257 return nctrs;
258 }
259
260 T_DECL(uncore_accuracy,
261 "ensure that the uncore counters count accurately",
262 T_META_ASROOT(true),
263 XNU_T_META_SOC_SPECIFIC,
264 T_META_MAYFAIL("rdar://88973518, threads need to be forced onto clusters"))
265 {
266 int fd;
267 int nctrs = 0;
268 int nmonitors = 0;
269 uint64_t ctr_mask;
270 uint64_t counts[2][CTRS_MAX];
271 uint64_t times[2];
272
273 fd = open_uncore_error(NULL);
274
275 /*
276 * The reference timebase event counts the same as mach_continuous_time
277 * (on hardware supporting uncore counters). Make sure that the counter
278 * is close to the values returned from the trap.
279 *
280 * Fill all the counters with this event.
281 */
282 nctrs = uncore_add_all(fd, REF_TIMEBASE_EVENT, &nmonitors);
283 ctr_mask = (UINT64_C(1) << nctrs) - 1;
284
285 T_LOG("added %d counters to check", nctrs);
286
287 uncore_enable(fd);
288
289 /*
290 * First, make sure there's an upper bound on the counter -- take the
291 * time around getting the counter values.
292 */
293
294 times[0] = mach_absolute_time();
295 uncore_counts(fd, ctr_mask, counts[0]);
296
297 usleep(SLEEP_USECS);
298
299 uncore_counts(fd, ctr_mask, counts[1]);
300 times[1] = mach_absolute_time();
301
302 T_QUIET; T_EXPECT_GT(times[1], times[0],
303 "mach_continuous_time is monotonically increasing");
304 for (int i = 0; i < nctrs; i++) {
305 T_EXPECT_GT(counts[1][i], counts[0][i],
306 "uncore counter %d value is monotonically increasing", i);
307 T_EXPECT_LT(counts[1][i] - counts[0][i], times[1] - times[0],
308 "reference timebase on uncore counter %d satisfies upper bound "
309 "from mach_absolute_time", i);
310 }
311
312 /*
313 * Next, the lower bound -- put mach_absolute_time inside getting the
314 * counter values.
315 */
316
317 uncore_counts(fd, ctr_mask, counts[0]);
318 times[0] = mach_absolute_time();
319
320 volatile int iterations = 100000;
321 while (iterations--) {
322 ;
323 }
324
325 times[1] = mach_absolute_time();
326 uncore_counts(fd, ctr_mask, counts[1]);
327
328 for (int mon = 0; mon < nmonitors; mon++) {
329 for (int i = 0; i < nctrs; i++) {
330 uint64_t after = counts[1][i * mon];
331 uint64_t before = counts[0][i * mon];
332 uint64_t delta = after - before;
333 T_LOG("%d,%2d: %12" PRIu64 "%12" PRIu64 " = %10" PRIu64, mon, i,
334 before, after, delta);
335 T_QUIET;
336 T_EXPECT_GT(after, before,
337 "uncore %d counter %d value is monotonically increasing",
338 mon, i);
339 T_QUIET;
340 T_EXPECT_GT(counts[1][i * mon] - counts[0][i * mon],
341 times[1] - times[0],
342 "reference timebase on uncore %d counter %d satisfies "
343 "lower bound from mach_absolute_time", mon, i);
344 }
345 }
346 }
347
348 T_DECL(uncore_ownership,
349 "ensure the dev node cannot be open in two places",
350 T_META_ASROOT(true))
351 {
352 int fd;
353 int other_fd;
354 int error;
355
356 fd = open_uncore_error(NULL);
357
358 other_fd = open_uncore_error(&error);
359 T_ASSERT_LT(other_fd, 0, "opening a second uncore fd should fail");
360 }
361
362 T_DECL(uncore_root_required,
363 "ensure the dev node cannot be opened by non-root users",
364 T_META_ASROOT(false))
365 {
366 int fd;
367 int error = 0;
368
369 T_SKIP("libdarwintest doesn't drop privileges properly");
370
371 fd = open_uncore_error(&error);
372 T_ASSERT_LT(fd, 0, "opening dev node should not return an fd");
373 T_ASSERT_EQ(error, EPERM,
374 "opening dev node as non-root user should fail with EPERM");
375 }
376
377 T_DECL(perf_uncore,
378 "measure the latency of accessing the counters",
379 XNU_T_META_SOC_SPECIFIC,
380 T_META_TAG_PERF)
381 {
382 int fd;
383 int nctrs;
384 int nmonitors;
385 int r;
386 uint64_t ctr_mask;
387 dt_stat_thread_instructions_t counts_instrs;
388 dt_stat_t counter_deltas;
389
390 counts_instrs = dt_stat_thread_instructions_create("ioctl_counts");
391 counter_deltas = dt_stat_create("abs_time", "between_each_counter");
392
393 fd = open_uncore_error(NULL);
394
395 nctrs = uncore_add_all(fd, REF_TIMEBASE_EVENT, &nmonitors);
396 ctr_mask = (UINT64_C(1) << nctrs) - 1;
397
398 uncore_enable(fd);
399
400 do {
401 dt_stat_token token;
402 uint64_t counts[nctrs * nmonitors];
403 union monotonic_ctl_counts *cts_ctl;
404
405 cts_ctl = (union monotonic_ctl_counts *)counts;
406 cts_ctl->in.ctr_mask = ctr_mask;
407
408 token = dt_stat_thread_instructions_begin(counts_instrs);
409 r = ioctl(fd, MT_IOC_COUNTS, cts_ctl);
410 dt_stat_thread_instructions_end(counts_instrs, token);
411 T_QUIET;
412 T_ASSERT_POSIX_SUCCESS(r,
413 "getting uncore counter values %#" PRIx64, ctr_mask);
414
415 for (int i = 0; i < (nctrs - 1); i++) {
416 dt_stat_add(counter_deltas, (double)(counts[i + 1] - counts[i]));
417 }
418 } while (!dt_stat_stable(counts_instrs) || !dt_stat_stable(counter_deltas));
419
420 dt_stat_finalize(counts_instrs);
421 dt_stat_finalize(counter_deltas);
422 }
423