1 // Copyright (c) 2021-2023 Apple Inc. All rights reserved.
2
3 #include <darwintest.h>
4 #include <stdlib.h>
5 #include <sys/resource_private.h>
6 #include <sys/sysctl.h>
7
8 #include "test_utils.h"
9 #include "recount_test_utils.h"
10
11 T_GLOBAL_META(
12 T_META_RADAR_COMPONENT_NAME("xnu"),
13 T_META_RADAR_COMPONENT_VERSION("cpu counters"),
14 T_META_OWNER("mwidmann"),
15 T_META_CHECK_LEAKS(false));
16
17 static char *amp_fail_reason = "P-binding on AMP scheduler";
18
19 static void
_check_cpi(struct thsc_cpi * before,struct thsc_cpi * after,const char * name)20 _check_cpi(struct thsc_cpi *before, struct thsc_cpi *after, const char *name)
21 {
22 T_QUIET; T_MAYFAIL_IF_ENABLED(amp_fail_reason);
23 T_EXPECT_GT(before->tcpi_instructions, UINT64_C(0),
24 "%s: instructions non-zero", name);
25 T_QUIET; T_MAYFAIL_IF_ENABLED(amp_fail_reason);
26 T_EXPECT_GT(before->tcpi_cycles, UINT64_C(0), "%s: cycles non-zero",
27 name);
28
29 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
30 T_EXPECT_GT(after->tcpi_instructions, before->tcpi_instructions,
31 "%s: instructions monotonically-increasing", name);
32 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
33 T_EXPECT_GT(after->tcpi_cycles, before->tcpi_cycles,
34 "%s: cycles monotonically-increasing", name);
35 }
36
37 static void
_check_no_cpi(struct thsc_cpi * before,struct thsc_cpi * after,const char * name)38 _check_no_cpi(struct thsc_cpi *before, struct thsc_cpi *after, const char *name)
39 {
40 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
41 T_EXPECT_EQ(after->tcpi_instructions, before->tcpi_instructions,
42 "%s: instructions should not increase", name);
43 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
44 T_EXPECT_EQ(after->tcpi_cycles, before->tcpi_cycles,
45 "%s: cycles should not increase", name);
46 }
47
48 static struct thsc_cpi
_remove_time_from_cpi(struct thsc_time_cpi * time_cpi)49 _remove_time_from_cpi(struct thsc_time_cpi *time_cpi)
50 {
51 return (struct thsc_cpi){
52 .tcpi_instructions = time_cpi->ttci_instructions,
53 .tcpi_cycles = time_cpi->ttci_cycles,
54 };
55 }
56
57 static void
_check_time_cpi(struct thsc_time_cpi * before,struct thsc_time_cpi * after,const char * name)58 _check_time_cpi(struct thsc_time_cpi *before, struct thsc_time_cpi *after,
59 const char *name)
60 {
61 struct thsc_cpi before_cpi = _remove_time_from_cpi(before);
62 struct thsc_cpi after_cpi = _remove_time_from_cpi(after);
63 _check_cpi(&before_cpi, &after_cpi, name);
64
65 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
66 T_EXPECT_GT(after->ttci_user_time_mach, before->ttci_user_time_mach,
67 "%s: user time monotonically-increasing", name);
68
69 if (has_user_system_times()) {
70 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
71 T_EXPECT_GT(after->ttci_system_time_mach, before->ttci_system_time_mach,
72 "%s: system time monotonically-increasing", name);
73 }
74 }
75
76 static void
_check_no_time_cpi(struct thsc_time_cpi * before,struct thsc_time_cpi * after,const char * name)77 _check_no_time_cpi(struct thsc_time_cpi *before, struct thsc_time_cpi *after,
78 const char *name)
79 {
80 struct thsc_cpi before_cpi = _remove_time_from_cpi(before);
81 struct thsc_cpi after_cpi = _remove_time_from_cpi(after);
82 _check_no_cpi(&before_cpi, &after_cpi, name);
83
84 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
85 T_EXPECT_EQ(after->ttci_user_time_mach, before->ttci_user_time_mach,
86 "%s: user time should not change", name);
87
88 if (has_user_system_times()) {
89 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
90 T_EXPECT_EQ(after->ttci_system_time_mach, before->ttci_system_time_mach,
91 "%s: system time should not change", name);
92 }
93 }
94
95 static struct thsc_time_cpi
_remove_energy_from_cpi(struct thsc_time_energy_cpi * energy_cpi)96 _remove_energy_from_cpi(struct thsc_time_energy_cpi *energy_cpi)
97 {
98 return (struct thsc_time_cpi){
99 .ttci_instructions = energy_cpi->ttec_instructions,
100 .ttci_cycles = energy_cpi->ttec_cycles,
101 .ttci_system_time_mach = energy_cpi->ttec_system_time_mach,
102 .ttci_user_time_mach = energy_cpi->ttec_user_time_mach,
103 };
104 }
105
106 static void
_check_usage(struct thsc_time_energy_cpi * before,struct thsc_time_energy_cpi * after,const char * name)107 _check_usage(struct thsc_time_energy_cpi *before,
108 struct thsc_time_energy_cpi *after, const char *name)
109 {
110 struct thsc_time_cpi before_time = _remove_energy_from_cpi(before);
111 struct thsc_time_cpi after_time = _remove_energy_from_cpi(after);
112 _check_time_cpi(&before_time, &after_time, name);
113
114 if (has_energy()) {
115 T_MAYFAIL_IF_ENABLED(amp_fail_reason);
116 T_EXPECT_GT(after->ttec_energy_nj, UINT64_C(0),
117 "%s: energy monotonically-increasing", name);
118 }
119 }
120
121 static void
_check_no_usage(struct thsc_time_energy_cpi * before,struct thsc_time_energy_cpi * after,const char * name)122 _check_no_usage(struct thsc_time_energy_cpi *before,
123 struct thsc_time_energy_cpi *after, const char *name)
124 {
125 struct thsc_time_cpi before_time = _remove_energy_from_cpi(before);
126 struct thsc_time_cpi after_time = _remove_energy_from_cpi(after);
127 _check_no_time_cpi(&before_time, &after_time, name);
128 }
129
130 T_DECL(thread_selfcounts_cpi_sanity, "check the current thread's CPI",
131 REQUIRE_RECOUNT_PMCS, T_META_TAG_VM_NOT_ELIGIBLE)
132 {
133 int err;
134 struct thsc_cpi counts[2] = { 0 };
135
136 err = thread_selfcounts(THSC_CPI, &counts[0], sizeof(counts[0]));
137 T_ASSERT_POSIX_ZERO(err, "thread_selfcounts(THSC_CPI, ...)");
138 err = thread_selfcounts(THSC_CPI, &counts[1], sizeof(counts[1]));
139 T_ASSERT_POSIX_ZERO(err, "thread_selfcounts(THSC_CPI, ...)");
140
141 _check_cpi(&counts[0], &counts[1], "anywhere");
142 }
143
144 T_DECL(thread_selfcounts_perf_level_sanity,
145 "check per-perf level time, energy, and CPI",
146 REQUIRE_RECOUNT_PMCS,
147 REQUIRE_MULTIPLE_PERF_LEVELS,
148 SET_THREAD_BIND_BOOTARG,
149 T_META_ASROOT(true), T_META_TAG_VM_NOT_ELIGIBLE)
150 {
151 unsigned int level_count = perf_level_count();
152 T_QUIET; T_ASSERT_GT(level_count, 1, "Platform should be AMP");
153
154 struct thsc_time_energy_cpi *before = calloc(level_count, sizeof(*before));
155 struct thsc_time_energy_cpi *after = calloc(level_count, sizeof(*after));
156
157 run_on_all_perf_levels();
158
159 int err = thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, before,
160 level_count * sizeof(*before));
161 T_ASSERT_POSIX_ZERO(err,
162 "thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, ...)");
163
164 run_on_all_perf_levels();
165
166 err = thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, after,
167 level_count * sizeof(*after));
168 T_ASSERT_POSIX_ZERO(err,
169 "thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, ...)");
170
171 for (unsigned int i = 0; i < level_count; i++) {
172 _check_usage(&before[i], &after[i], perf_level_name(i));
173 }
174
175 free(before);
176 free(after);
177 }
178
179 static void
_expect_counts_on_perf_level(unsigned int perf_level_index,struct thsc_time_energy_cpi * before,struct thsc_time_energy_cpi * after)180 _expect_counts_on_perf_level(unsigned int perf_level_index,
181 struct thsc_time_energy_cpi *before,
182 struct thsc_time_energy_cpi *after)
183 {
184 unsigned int level_count = perf_level_count();
185 int err = thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, before,
186 level_count * sizeof(*before));
187 T_ASSERT_POSIX_ZERO(err,
188 "thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, ...)");
189 (void)getppid();
190 // Allow time for CLPC to read energy counters
191 usleep(10000);
192 err = thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, after,
193 level_count * sizeof(*after));
194 T_ASSERT_POSIX_ZERO(err,
195 "thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, ...)");
196
197 const char *name = perf_level_name(perf_level_index);
198 _check_usage(&before[perf_level_index], &after[perf_level_index], name);
199 }
200
201 static void
_expect_no_counts_on_perf_level(unsigned int perf_level_index,struct thsc_time_energy_cpi * before,struct thsc_time_energy_cpi * after)202 _expect_no_counts_on_perf_level(unsigned int perf_level_index,
203 struct thsc_time_energy_cpi *before,
204 struct thsc_time_energy_cpi *after)
205 {
206 unsigned int level_count = perf_level_count();
207 int err = thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, before,
208 level_count * sizeof(*before));
209 T_ASSERT_POSIX_ZERO(err,
210 "thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, ...)");
211 (void)getppid();
212 // Allow time for CLPC to read energy counters
213 usleep(10000);
214 err = thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, after,
215 level_count * sizeof(*after));
216 T_ASSERT_POSIX_ZERO(err,
217 "thread_selfcounts(THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL, ...)");
218
219 const char *name = perf_level_name(perf_level_index);
220 _check_no_usage(&before[perf_level_index], &after[perf_level_index], name);
221 }
222
223 T_DECL(thread_selfcounts_perf_level_correct,
224 "check that runtimes on each perf level match binding request",
225 REQUIRE_RECOUNT_PMCS,
226 REQUIRE_MULTIPLE_PERF_LEVELS,
227 SET_THREAD_BIND_BOOTARG,
228 T_META_ASROOT(true), T_META_TAG_VM_NOT_ELIGIBLE)
229 {
230 unsigned int level_count = perf_level_count();
231 T_QUIET; T_ASSERT_GT(level_count, 1, "Platform should be AMP");
232
233 T_LOG("Currently running the \"%s\" scheduler policy", sched_policy_name());
234 bool is_edge_scheduler = strcmp(sched_policy_name(), "edge") == 0;
235
236 struct thsc_time_energy_cpi *before = calloc(level_count, sizeof(*before));
237 struct thsc_time_energy_cpi *after = calloc(level_count, sizeof(*after));
238
239 for (unsigned int i = 0; i < level_count; i++) {
240 T_LOG("Binding to \"%s\" cluster, should only see counts from %c-cores",
241 perf_level_name(i), perf_level_name(i)[0]);
242
243 T_SETUPBEGIN;
244 bind_to_cluster(perf_level_name(i)[0]);
245
246 if (!is_edge_scheduler && (perf_level_name(i)[0] == 'P')) {
247 T_QUIET; T_EXPECT_EQ_STR(sched_policy_name(), "amp", "Unexpected multicluster scheduling policy");
248 T_LOG("The AMP scheduler doesn't guarantee that a P-bound thread will "
249 "only run on P-cores, so the following expects may fail.");
250 set_expects_may_fail(true);
251 }
252 T_SETUPEND;
253
254 _expect_counts_on_perf_level(i, before, after);
255 for (unsigned int j = 0; j < level_count; j++) {
256 if (j != i) {
257 _expect_no_counts_on_perf_level(j, before, after);
258 }
259 }
260 }
261
262 free(before);
263 free(after);
264 }
265
266 T_DECL(thread_selfcounts_cpi_perf,
267 "test the overhead of thread_selfcounts(2) THSC_CPI", T_META_TAG_PERF,
268 REQUIRE_RECOUNT_PMCS, T_META_TAG_VM_NOT_ELIGIBLE)
269 {
270 struct thsc_cpi counts[2];
271
272 T_SETUPBEGIN;
273 dt_stat_t instrs = dt_stat_create("thread_selfcounts_cpi_instrs",
274 "instructions");
275 dt_stat_t cycles = dt_stat_create("thread_selfcounts_cpi_cycles", "cycles");
276 T_SETUPEND;
277
278 while (!dt_stat_stable(instrs) || !dt_stat_stable(cycles)) {
279 int r1 = thread_selfcounts(THSC_CPI, &counts[0], sizeof(counts[0]));
280 int r2 = thread_selfcounts(THSC_CPI, &counts[1], sizeof(counts[1]));
281 T_QUIET; T_ASSERT_POSIX_ZERO(r1, "thread_selfcounts(THSC_CPI, ...)");
282 T_QUIET; T_ASSERT_POSIX_ZERO(r2, "thread_selfcounts(THSC_CPI, ...)");
283
284 dt_stat_add(instrs, counts[1].tcpi_instructions -
285 counts[0].tcpi_instructions);
286 dt_stat_add(cycles, counts[1].tcpi_cycles - counts[0].tcpi_cycles);
287 }
288
289 dt_stat_finalize(instrs);
290 dt_stat_finalize(cycles);
291 }
292