1 // Copyright (c) 2021-2022 Apple Inc. All rights reserved.
2
3 #include <darwintest.h>
4 #include "test_utils.h"
5 #include <fcntl.h>
6 #include <inttypes.h>
7 #ifndef PRIVATE
8 /*
9 * Need new CPU families.
10 */
11 #define PRIVATE
12 #include <mach/machine.h>
13 #undef PRIVATE
14 #else /* !defined(PRIVATE) */
15 #include <mach/machine.h>
16 #endif /* defined(PRIVATE) */
17 #include <stdint.h>
18 #include <System/sys/guarded.h>
19 #include <System/sys/monotonic.h>
20 #include <sys/ioctl.h>
21 #include <sys/sysctl.h>
22 #include <unistd.h>
23
24 T_GLOBAL_META(
25 T_META_NAMESPACE("xnu.monotonic"),
26 T_META_CHECK_LEAKS(false)
27 );
28
29 static bool
device_supports_uncore(void)30 device_supports_uncore(void)
31 {
32 int r;
33 int type, subtype;
34 unsigned int family;
35 size_t size = sizeof(type);
36
37 /*
38 * Only arm64 Monsoon devices support uncore counters.
39 */
40
41 r = sysctlbyname("hw.cputype", &type, &size, NULL, 0);
42 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cputype\")");
43 r = sysctlbyname("hw.cpusubtype", &subtype, &size, NULL, 0);
44 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cpusubtype\")");
45 r = sysctlbyname("hw.cpufamily", &family, &size, NULL, 0);
46 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cpufamily\")");
47
48 if (type == CPU_TYPE_ARM64 &&
49 subtype == CPU_SUBTYPE_ARM64_V8 &&
50 (family == CPUFAMILY_ARM_MONSOON_MISTRAL ||
51 family == CPUFAMILY_ARM_VORTEX_TEMPEST)) {
52 return true;
53 }
54
55 return false;
56 }
57
58 #define UNCORE_DEV_PATH "/dev/monotonic/uncore"
59
60 static int
open_uncore_error(int * error)61 open_uncore_error(int *error)
62 {
63 guardid_t guard;
64 int fd;
65
66 guard = 0xa5adcafe;
67
68 T_SETUPBEGIN;
69
70 fd = guarded_open_np(UNCORE_DEV_PATH, &guard,
71 GUARD_CLOSE | GUARD_DUP | GUARD_WRITE, O_CLOEXEC | O_EXCL);
72 if (fd < 0 && errno == ENOENT) {
73 T_ASSERT_FALSE(device_supports_uncore(),
74 "lack of dev node implies no uncore support");
75 T_SKIP("uncore counters are unsupported");
76 __builtin_unreachable();
77 }
78
79 if (error == NULL) {
80 T_ASSERT_POSIX_SUCCESS(fd, "open '%s'", UNCORE_DEV_PATH);
81 } else {
82 *error = errno;
83 }
84
85 T_SETUPEND;
86
87 return fd;
88 }
89
90 static void
uncore_counts(int fd,uint64_t ctr_mask,uint64_t * counts)91 uncore_counts(int fd, uint64_t ctr_mask, uint64_t *counts)
92 {
93 int r;
94 union monotonic_ctl_counts *cts_ctl;
95
96 cts_ctl = (union monotonic_ctl_counts *)counts;
97 cts_ctl->in.ctr_mask = ctr_mask;
98
99 r = ioctl(fd, MT_IOC_COUNTS, cts_ctl);
100 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "MT_IOC_COUNTS got counter values");
101 }
102
103 #define REF_TIMEBASE_EVENT 0x3
104 #define CTRS_MAX 128
105
106 T_DECL(uncore_max_counters,
107 "ensure that the maximum number of uncore countes is sane",
108 XNU_T_META_SOC_SPECIFIC,
109 T_META_ASROOT(true))
110 {
111 int nctrs = 0;
112 int fd;
113
114 fd = open_uncore_error(NULL);
115
116 do {
117 union monotonic_ctl_add add_ctl;
118 int r;
119
120 add_ctl.in.config.event = REF_TIMEBASE_EVENT;
121 add_ctl.in.config.allowed_ctr_mask = UINT64_MAX;
122
123 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
124 if (r < 0 && errno == E2BIG) {
125 break;
126 }
127
128 T_QUIET;
129 T_ASSERT_POSIX_SUCCESS(r, "added reference timebase event to counters");
130 nctrs++;
131 } while (nctrs < CTRS_MAX);
132
133 T_EXPECT_LT(nctrs, CTRS_MAX, "able to allocate %d uncore PMCs", nctrs);
134 }
135
136 static uint32_t
uncore_add(int fd,uint64_t event,uint64_t allowed_ctrs,int error)137 uncore_add(int fd, uint64_t event, uint64_t allowed_ctrs, int error)
138 {
139 int save_errno;
140 int r;
141 uint32_t ctr;
142 union monotonic_ctl_add add_ctl;
143
144 add_ctl.in.config.event = event;
145 add_ctl.in.config.allowed_ctr_mask = allowed_ctrs;
146 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
147 if (error) {
148 save_errno = errno;
149 T_EXPECT_LT(r, 0, "adding event to counter should fail");
150 T_EXPECT_EQ(save_errno, error,
151 "adding event to counter should fail with %d: %s",
152 error, strerror(error));
153 return UINT32_MAX;
154 } else {
155 T_QUIET;
156 T_ASSERT_POSIX_SUCCESS(r,
157 "added event %#" PRIx64 " to counters", event);
158 }
159
160 ctr = add_ctl.out.ctr;
161 T_QUIET; T_ASSERT_LT(ctr, (uint32_t)CTRS_MAX, "counter returned should be sane");
162 return ctr;
163 }
164
165 T_DECL(uncore_collision,
166 "ensure that trying to add an event on the same counter fails",
167 T_META_ASROOT(true))
168 {
169 int fd;
170 uint32_t ctr;
171
172 fd = open_uncore_error(NULL);
173
174 ctr = uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, 0);
175 T_LOG("added event to uncore counter %d\n", ctr);
176
177 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1) << ctr, ENOSPC);
178 }
179
180 static void
uncore_enable(int fd)181 uncore_enable(int fd)
182 {
183 union monotonic_ctl_enable en_ctl = {
184 .in = { .enable = true }
185 };
186
187 T_ASSERT_POSIX_SUCCESS(ioctl(fd, MT_IOC_ENABLE, &en_ctl),
188 "enabling counters");
189 }
190
191 T_DECL(uncore_enabled_busy,
192 "ensure that trying to add an event while enabled fails",
193 T_META_ASROOT(true))
194 {
195 int fd;
196
197 fd = open_uncore_error(NULL);
198
199 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, 0);
200
201 uncore_enable(fd);
202 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, EBUSY);
203 }
204
205 T_DECL(uncore_reset,
206 "ensure that resetting the counters works")
207 {
208 int fd;
209 int r;
210
211 fd = open_uncore_error(NULL);
212
213 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), 0);
214 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), ENOSPC);
215
216 r = ioctl(fd, MT_IOC_RESET);
217 T_ASSERT_POSIX_SUCCESS(r, "resetting succeeds");
218
219 T_LOG("adding event to same counter after reset");
220 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), 0);
221 }
222
223 #define SLEEP_USECS (500 * 1000)
224
225 static int
uncore_add_all(int fd,uint64_t event,int * nmonitors)226 uncore_add_all(int fd, uint64_t event, int *nmonitors)
227 {
228 int nctrs = 0;
229 int r;
230
231 do {
232 union monotonic_ctl_add add_ctl;
233
234 add_ctl.in.config.event = event;
235 add_ctl.in.config.allowed_ctr_mask = UINT64_MAX;
236
237 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
238 if (r < 0 && errno == E2BIG) {
239 break;
240 }
241
242 T_QUIET;
243 T_ASSERT_POSIX_SUCCESS(r, "added event %#" PRIx64 " to counters",
244 event);
245 nctrs++;
246 } while (nctrs < CTRS_MAX);
247
248 if (nmonitors) {
249 union monotonic_ctl_info info_ctl;
250 r = ioctl(fd, MT_IOC_GET_INFO, &info_ctl);
251 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "got info about uncore counters");
252
253 *nmonitors = (int)info_ctl.out.nmonitors;
254 }
255
256 return nctrs;
257 }
258
259 T_DECL(uncore_accuracy,
260 "ensure that the uncore counters count accurately",
261 T_META_ASROOT(true),
262 XNU_T_META_SOC_SPECIFIC,
263 T_META_MAYFAIL("rdar://88973518, threads need to be forced onto clusters"))
264 {
265 int fd;
266 int nctrs = 0;
267 int nmonitors = 0;
268 uint64_t ctr_mask;
269 uint64_t counts[2][CTRS_MAX];
270 uint64_t times[2];
271
272 fd = open_uncore_error(NULL);
273
274 /*
275 * The reference timebase event counts the same as mach_continuous_time
276 * (on hardware supporting uncore counters). Make sure that the counter
277 * is close to the values returned from the trap.
278 *
279 * Fill all the counters with this event.
280 */
281 nctrs = uncore_add_all(fd, REF_TIMEBASE_EVENT, &nmonitors);
282 ctr_mask = (UINT64_C(1) << nctrs) - 1;
283
284 T_LOG("added %d counters to check", nctrs);
285
286 uncore_enable(fd);
287
288 /*
289 * First, make sure there's an upper bound on the counter -- take the
290 * time around getting the counter values.
291 */
292
293 times[0] = mach_absolute_time();
294 uncore_counts(fd, ctr_mask, counts[0]);
295
296 usleep(SLEEP_USECS);
297
298 uncore_counts(fd, ctr_mask, counts[1]);
299 times[1] = mach_absolute_time();
300
301 T_QUIET; T_EXPECT_GT(times[1], times[0],
302 "mach_continuous_time is monotonically increasing");
303 for (int i = 0; i < nctrs; i++) {
304 T_EXPECT_GT(counts[1][i], counts[0][i],
305 "uncore counter %d value is monotonically increasing", i);
306 T_EXPECT_LT(counts[1][i] - counts[0][i], times[1] - times[0],
307 "reference timebase on uncore counter %d satisfies upper bound "
308 "from mach_absolute_time", i);
309 }
310
311 /*
312 * Next, the lower bound -- put mach_absolute_time inside getting the
313 * counter values.
314 */
315
316 uncore_counts(fd, ctr_mask, counts[0]);
317 times[0] = mach_absolute_time();
318
319 volatile int iterations = 100000;
320 while (iterations--) {
321 ;
322 }
323
324 times[1] = mach_absolute_time();
325 uncore_counts(fd, ctr_mask, counts[1]);
326
327 for (int mon = 0; mon < nmonitors; mon++) {
328 for (int i = 0; i < nctrs; i++) {
329 uint64_t after = counts[1][i * mon];
330 uint64_t before = counts[0][i * mon];
331 uint64_t delta = after - before;
332 T_LOG("%d,%2d: %12" PRIu64 "%12" PRIu64 " = %10" PRIu64, mon, i,
333 before, after, delta);
334 T_QUIET;
335 T_EXPECT_GT(after, before,
336 "uncore %d counter %d value is monotonically increasing",
337 mon, i);
338 T_QUIET;
339 T_EXPECT_GT(counts[1][i * mon] - counts[0][i * mon],
340 times[1] - times[0],
341 "reference timebase on uncore %d counter %d satisfies "
342 "lower bound from mach_absolute_time", mon, i);
343 }
344 }
345 }
346
347 T_DECL(uncore_ownership,
348 "ensure the dev node cannot be open in two places",
349 T_META_ASROOT(true))
350 {
351 int fd;
352 int other_fd;
353 int error;
354
355 fd = open_uncore_error(NULL);
356
357 other_fd = open_uncore_error(&error);
358 T_ASSERT_LT(other_fd, 0, "opening a second uncore fd should fail");
359 }
360
361 T_DECL(uncore_root_required,
362 "ensure the dev node cannot be opened by non-root users",
363 T_META_ASROOT(false))
364 {
365 int fd;
366 int error = 0;
367
368 T_SKIP("libdarwintest doesn't drop privileges properly");
369
370 fd = open_uncore_error(&error);
371 T_ASSERT_LT(fd, 0, "opening dev node should not return an fd");
372 T_ASSERT_EQ(error, EPERM,
373 "opening dev node as non-root user should fail with EPERM");
374 }
375
376 T_DECL(perf_uncore,
377 "measure the latency of accessing the counters",
378 XNU_T_META_SOC_SPECIFIC,
379 T_META_TAG_PERF)
380 {
381 int fd;
382 int nctrs;
383 int nmonitors;
384 int r;
385 uint64_t ctr_mask;
386 dt_stat_thread_instructions_t counts_instrs;
387 dt_stat_t counter_deltas;
388
389 counts_instrs = dt_stat_thread_instructions_create("ioctl_counts");
390 counter_deltas = dt_stat_create("abs_time", "between_each_counter");
391
392 fd = open_uncore_error(NULL);
393
394 nctrs = uncore_add_all(fd, REF_TIMEBASE_EVENT, &nmonitors);
395 ctr_mask = (UINT64_C(1) << nctrs) - 1;
396
397 uncore_enable(fd);
398
399 do {
400 dt_stat_token token;
401 uint64_t counts[nctrs * nmonitors];
402 union monotonic_ctl_counts *cts_ctl;
403
404 cts_ctl = (union monotonic_ctl_counts *)counts;
405 cts_ctl->in.ctr_mask = ctr_mask;
406
407 token = dt_stat_thread_instructions_begin(counts_instrs);
408 r = ioctl(fd, MT_IOC_COUNTS, cts_ctl);
409 dt_stat_thread_instructions_end(counts_instrs, token);
410 T_QUIET;
411 T_ASSERT_POSIX_SUCCESS(r,
412 "getting uncore counter values %#" PRIx64, ctr_mask);
413
414 for (int i = 0; i < (nctrs - 1); i++) {
415 dt_stat_add(counter_deltas, (double)(counts[i + 1] - counts[i]));
416 }
417 } while (!dt_stat_stable(counts_instrs) || !dt_stat_stable(counter_deltas));
418
419 dt_stat_finalize(counts_instrs);
420 dt_stat_finalize(counter_deltas);
421 }
422