1 /*
2 * Must come before including darwintest.h
3 */
4 #ifdef T_NAMESPACE
5 #undef T_NAMESPACE
6 #endif /* defined(T_NAMESPACE) */
7
8 #include <darwintest.h>
9 #include <fcntl.h>
10 #include <inttypes.h>
11 #ifndef PRIVATE
12 /*
13 * Need new CPU families.
14 */
15 #define PRIVATE
16 #include <mach/machine.h>
17 #undef PRIVATE
18 #else /* !defined(PRIVATE) */
19 #include <mach/machine.h>
20 #endif /* defined(PRIVATE) */
21 #include <stdint.h>
22 #include <System/sys/guarded.h>
23 #include <System/sys/monotonic.h>
24 #include <sys/ioctl.h>
25 #include <sys/sysctl.h>
26 #include <unistd.h>
27
28 T_GLOBAL_META(
29 T_META_NAMESPACE("xnu.monotonic"),
30 T_META_CHECK_LEAKS(false)
31 );
32
33 static bool
device_supports_uncore(void)34 device_supports_uncore(void)
35 {
36 int r;
37 int type, subtype;
38 unsigned int family;
39 size_t size = sizeof(type);
40
41 /*
42 * Only arm64 Monsoon devices support uncore counters.
43 */
44
45 r = sysctlbyname("hw.cputype", &type, &size, NULL, 0);
46 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cputype\")");
47 r = sysctlbyname("hw.cpusubtype", &subtype, &size, NULL, 0);
48 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cpusubtype\")");
49 r = sysctlbyname("hw.cpufamily", &family, &size, NULL, 0);
50 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "sysctlbyname(\"hw.cpufamily\")");
51
52 if (type == CPU_TYPE_ARM64 &&
53 subtype == CPU_SUBTYPE_ARM64_V8 &&
54 (family == CPUFAMILY_ARM_MONSOON_MISTRAL ||
55 family == CPUFAMILY_ARM_VORTEX_TEMPEST)) {
56 return true;
57 }
58
59 return false;
60 }
61
62 #define UNCORE_DEV_PATH "/dev/monotonic/uncore"
63
64 static int
open_uncore_error(int * error)65 open_uncore_error(int *error)
66 {
67 guardid_t guard;
68 int fd;
69
70 guard = 0xa5adcafe;
71
72 T_SETUPBEGIN;
73
74 fd = guarded_open_np(UNCORE_DEV_PATH, &guard,
75 GUARD_CLOSE | GUARD_DUP | GUARD_WRITE, O_CLOEXEC | O_EXCL);
76 if (fd < 0 && errno == ENOENT) {
77 T_ASSERT_FALSE(device_supports_uncore(),
78 "lack of dev node implies no uncore support");
79 T_SKIP("uncore counters are unsupported");
80 __builtin_unreachable();
81 }
82
83 if (error == NULL) {
84 T_ASSERT_POSIX_SUCCESS(fd, "open '%s'", UNCORE_DEV_PATH);
85 } else {
86 *error = errno;
87 }
88
89 T_SETUPEND;
90
91 return fd;
92 }
93
94 static void
uncore_counts(int fd,uint64_t ctr_mask,uint64_t * counts)95 uncore_counts(int fd, uint64_t ctr_mask, uint64_t *counts)
96 {
97 int r;
98 union monotonic_ctl_counts *cts_ctl;
99
100 cts_ctl = (union monotonic_ctl_counts *)counts;
101 cts_ctl->in.ctr_mask = ctr_mask;
102
103 r = ioctl(fd, MT_IOC_COUNTS, cts_ctl);
104 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "MT_IOC_COUNTS got counter values");
105 }
106
107 #define REF_TIMEBASE_EVENT 0x3
108 #define CTRS_MAX 128
109
110 T_DECL(uncore_max_counters,
111 "ensure that the maximum number of uncore countes is sane",
112 T_META_ASROOT(true))
113 {
114 int nctrs = 0;
115 int fd;
116
117 fd = open_uncore_error(NULL);
118
119 do {
120 union monotonic_ctl_add add_ctl;
121 int r;
122
123 add_ctl.in.config.event = REF_TIMEBASE_EVENT;
124 add_ctl.in.config.allowed_ctr_mask = UINT64_MAX;
125
126 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
127 if (r < 0 && errno == E2BIG) {
128 break;
129 }
130
131 T_QUIET;
132 T_ASSERT_POSIX_SUCCESS(r, "added reference timebase event to counters");
133 nctrs++;
134 } while (nctrs < CTRS_MAX);
135
136 T_EXPECT_LT(nctrs, CTRS_MAX, "able to allocate %d uncore PMCs", nctrs);
137 }
138
139 static uint32_t
uncore_add(int fd,uint64_t event,uint64_t allowed_ctrs,int error)140 uncore_add(int fd, uint64_t event, uint64_t allowed_ctrs, int error)
141 {
142 int save_errno;
143 int r;
144 uint32_t ctr;
145 union monotonic_ctl_add add_ctl;
146
147 add_ctl.in.config.event = event;
148 add_ctl.in.config.allowed_ctr_mask = allowed_ctrs;
149 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
150 if (error) {
151 save_errno = errno;
152 T_EXPECT_LT(r, 0, "adding event to counter should fail");
153 T_EXPECT_EQ(save_errno, error,
154 "adding event to counter should fail with %d: %s",
155 error, strerror(error));
156 return UINT32_MAX;
157 } else {
158 T_QUIET;
159 T_ASSERT_POSIX_SUCCESS(r,
160 "added event %#" PRIx64 " to counters", event);
161 }
162
163 ctr = add_ctl.out.ctr;
164 T_QUIET; T_ASSERT_LT(ctr, (uint32_t)CTRS_MAX, "counter returned should be sane");
165 return ctr;
166 }
167
168 T_DECL(uncore_collision,
169 "ensure that trying to add an event on the same counter fails",
170 T_META_ASROOT(true))
171 {
172 int fd;
173 uint32_t ctr;
174
175 fd = open_uncore_error(NULL);
176
177 ctr = uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, 0);
178 T_LOG("added event to uncore counter %d\n", ctr);
179
180 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1) << ctr, ENOSPC);
181 }
182
183 static void
uncore_enable(int fd)184 uncore_enable(int fd)
185 {
186 union monotonic_ctl_enable en_ctl = {
187 .in = { .enable = true }
188 };
189
190 T_ASSERT_POSIX_SUCCESS(ioctl(fd, MT_IOC_ENABLE, &en_ctl),
191 "enabling counters");
192 }
193
194 T_DECL(uncore_enabled_busy,
195 "ensure that trying to add an event while enabled fails",
196 T_META_ASROOT(true))
197 {
198 int fd;
199
200 fd = open_uncore_error(NULL);
201
202 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, 0);
203
204 uncore_enable(fd);
205 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_MAX, EBUSY);
206 }
207
208 T_DECL(uncore_reset,
209 "ensure that resetting the counters works")
210 {
211 int fd;
212 int r;
213
214 fd = open_uncore_error(NULL);
215
216 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), 0);
217 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), ENOSPC);
218
219 r = ioctl(fd, MT_IOC_RESET);
220 T_ASSERT_POSIX_SUCCESS(r, "resetting succeeds");
221
222 T_LOG("adding event to same counter after reset");
223 (void)uncore_add(fd, REF_TIMEBASE_EVENT, UINT64_C(1), 0);
224 }
225
226 #define SLEEP_USECS (500 * 1000)
227
228 static int
uncore_add_all(int fd,uint64_t event,int * nmonitors)229 uncore_add_all(int fd, uint64_t event, int *nmonitors)
230 {
231 int nctrs = 0;
232 int r;
233
234 do {
235 union monotonic_ctl_add add_ctl;
236
237 add_ctl.in.config.event = event;
238 add_ctl.in.config.allowed_ctr_mask = UINT64_MAX;
239
240 r = ioctl(fd, MT_IOC_ADD, &add_ctl);
241 if (r < 0 && errno == E2BIG) {
242 break;
243 }
244
245 T_QUIET;
246 T_ASSERT_POSIX_SUCCESS(r, "added event %#" PRIx64 " to counters",
247 event);
248 nctrs++;
249 } while (nctrs < CTRS_MAX);
250
251 if (nmonitors) {
252 union monotonic_ctl_info info_ctl;
253 r = ioctl(fd, MT_IOC_GET_INFO, &info_ctl);
254 T_QUIET; T_ASSERT_POSIX_SUCCESS(r, "got info about uncore counters");
255
256 *nmonitors = (int)info_ctl.out.nmonitors;
257 }
258
259 return nctrs;
260 }
261
262 T_DECL(uncore_accuracy,
263 "ensure that the uncore counters count accurately",
264 T_META_ASROOT(true),
265 T_META_MAYFAIL("rdar://88973518, threads need to be forced onto clusters"))
266 {
267 int fd;
268 int nctrs = 0;
269 int nmonitors = 0;
270 uint64_t ctr_mask;
271 uint64_t counts[2][CTRS_MAX];
272 uint64_t times[2];
273
274 fd = open_uncore_error(NULL);
275
276 /*
277 * The reference timebase event counts the same as mach_continuous_time
278 * (on hardware supporting uncore counters). Make sure that the counter
279 * is close to the values returned from the trap.
280 *
281 * Fill all the counters with this event.
282 */
283 nctrs = uncore_add_all(fd, REF_TIMEBASE_EVENT, &nmonitors);
284 ctr_mask = (UINT64_C(1) << nctrs) - 1;
285
286 T_LOG("added %d counters to check", nctrs);
287
288 uncore_enable(fd);
289
290 /*
291 * First, make sure there's an upper bound on the counter -- take the
292 * time around getting the counter values.
293 */
294
295 times[0] = mach_absolute_time();
296 uncore_counts(fd, ctr_mask, counts[0]);
297
298 usleep(SLEEP_USECS);
299
300 uncore_counts(fd, ctr_mask, counts[1]);
301 times[1] = mach_absolute_time();
302
303 T_QUIET; T_EXPECT_GT(times[1], times[0],
304 "mach_continuous_time is monotonically increasing");
305 for (int i = 0; i < nctrs; i++) {
306 T_EXPECT_GT(counts[1][i], counts[0][i],
307 "uncore counter %d value is monotonically increasing", i);
308 T_EXPECT_LT(counts[1][i] - counts[0][i], times[1] - times[0],
309 "reference timebase on uncore counter %d satisfies upper bound "
310 "from mach_absolute_time", i);
311 }
312
313 /*
314 * Next, the lower bound -- put mach_absolute_time inside getting the
315 * counter values.
316 */
317
318 uncore_counts(fd, ctr_mask, counts[0]);
319 times[0] = mach_absolute_time();
320
321 volatile int iterations = 100000;
322 while (iterations--) {
323 ;
324 }
325
326 times[1] = mach_absolute_time();
327 uncore_counts(fd, ctr_mask, counts[1]);
328
329 for (int mon = 0; mon < nmonitors; mon++) {
330 for (int i = 0; i < nctrs; i++) {
331 uint64_t after = counts[1][i * mon];
332 uint64_t before = counts[0][i * mon];
333 uint64_t delta = after - before;
334 T_LOG("%d,%2d: %12" PRIu64 "%12" PRIu64 " = %10" PRIu64, mon, i,
335 before, after, delta);
336 T_QUIET;
337 T_EXPECT_GT(after, before,
338 "uncore %d counter %d value is monotonically increasing",
339 mon, i);
340 T_QUIET;
341 T_EXPECT_GT(counts[1][i * mon] - counts[0][i * mon],
342 times[1] - times[0],
343 "reference timebase on uncore %d counter %d satisfies "
344 "lower bound from mach_absolute_time", mon, i);
345 }
346 }
347 }
348
349 T_DECL(uncore_ownership,
350 "ensure the dev node cannot be open in two places",
351 T_META_ASROOT(true))
352 {
353 int fd;
354 int other_fd;
355 int error;
356
357 fd = open_uncore_error(NULL);
358
359 other_fd = open_uncore_error(&error);
360 T_ASSERT_LT(other_fd, 0, "opening a second uncore fd should fail");
361 }
362
363 T_DECL(uncore_root_required,
364 "ensure the dev node cannot be opened by non-root users",
365 T_META_ASROOT(false))
366 {
367 int fd;
368 int error = 0;
369
370 T_SKIP("libdarwintest doesn't drop privileges properly");
371
372 fd = open_uncore_error(&error);
373 T_ASSERT_LT(fd, 0, "opening dev node should not return an fd");
374 T_ASSERT_EQ(error, EPERM,
375 "opening dev node as non-root user should fail with EPERM");
376 }
377
378 T_DECL(perf_uncore,
379 "measure the latency of accessing the counters",
380 T_META_TAG_PERF)
381 {
382 int fd;
383 int nctrs;
384 int nmonitors;
385 int r;
386 uint64_t ctr_mask;
387 dt_stat_thread_instructions_t counts_instrs;
388 dt_stat_t counter_deltas;
389
390 counts_instrs = dt_stat_thread_instructions_create("ioctl_counts");
391 counter_deltas = dt_stat_create("abs_time", "between_each_counter");
392
393 fd = open_uncore_error(NULL);
394
395 nctrs = uncore_add_all(fd, REF_TIMEBASE_EVENT, &nmonitors);
396 ctr_mask = (UINT64_C(1) << nctrs) - 1;
397
398 uncore_enable(fd);
399
400 do {
401 dt_stat_token token;
402 uint64_t counts[nctrs * nmonitors];
403 union monotonic_ctl_counts *cts_ctl;
404
405 cts_ctl = (union monotonic_ctl_counts *)counts;
406 cts_ctl->in.ctr_mask = ctr_mask;
407
408 token = dt_stat_thread_instructions_begin(counts_instrs);
409 r = ioctl(fd, MT_IOC_COUNTS, cts_ctl);
410 dt_stat_thread_instructions_end(counts_instrs, token);
411 T_QUIET;
412 T_ASSERT_POSIX_SUCCESS(r,
413 "getting uncore counter values %#" PRIx64, ctr_mask);
414
415 for (int i = 0; i < (nctrs - 1); i++) {
416 dt_stat_add(counter_deltas, (double)(counts[i + 1] - counts[i]));
417 }
418 } while (!dt_stat_stable(counts_instrs) || !dt_stat_stable(counter_deltas));
419
420 dt_stat_finalize(counts_instrs);
421 dt_stat_finalize(counter_deltas);
422 }
423