1 // Copyright (c) 2023 Apple Inc. All rights reserved.
2
3 #include <stdint.h>
4 #include <stdio.h>
5 #include <sys/kdebug.h>
6
7 /* Harness interface */
8 #include "sched_clutch_harness.h"
9
10 /* Include kernel header depdencies */
11 #include "shadow_headers/misc_needed_defines.h"
12
13 /* Header for Clutch policy code under-test */
14 #include <kern/sched_clutch.h>
15
16 /* Include non-header dependencies */
17 #define KERNEL_DEBUG_CONSTANT_IST(a0, a1, a2, a3, a4, a5, a6) clutch_impl_log_tracepoint(a1, a2, a3, a4, a5)
18 #include "shadow_headers/misc_needed_deps.c"
19 #include "shadow_headers/sched_prim.c"
20
21 static test_hw_topology_t curr_hw_topo = {
22 .num_psets = 0,
23 .psets = NULL,
24 };
25 static int _curr_cpu = 0;
26
27 unsigned int
ml_get_cluster_count(void)28 ml_get_cluster_count(void)
29 {
30 return (unsigned int)curr_hw_topo.num_psets;
31 }
32
33 /*
34 * Mocked HW details
35 * For simplicity, we mock a platform with 1 pset comprised of 1 CPU
36 */
37 uint32_t processor_avail_count = 0;
38
39 static struct processor_set *psets[MAX_PSETS];
40 static struct processor *cpus[MAX_CPUS];
41
42 /* Boot pset and CPU */
43 struct processor_set pset0;
44 struct processor cpu0;
45
46 /* Mocked-out Clutch functions */
47 static boolean_t
sched_thread_sched_pri_promoted(thread_t thread)48 sched_thread_sched_pri_promoted(thread_t thread)
49 {
50 (void)thread;
51 return FALSE;
52 }
53
54 /* Clutch policy code under-test, safe to include now after satisfying its dependencies */
55 #include <kern/sched_clutch.c>
56
57 /* Implementation of sched_clutch_harness.h interface */
58
59 int root_bucket_to_highest_pri[TH_BUCKET_SCHED_MAX] = {
60 MAXPRI_USER,
61 BASEPRI_FOREGROUND,
62 BASEPRI_USER_INITIATED,
63 BASEPRI_DEFAULT,
64 BASEPRI_UTILITY,
65 MAXPRI_THROTTLE
66 };
67
68 int clutch_interactivity_score_max = -1;
69 uint64_t clutch_root_bucket_wcel_us[TH_BUCKET_SCHED_MAX];
70 uint64_t clutch_root_bucket_warp_us[TH_BUCKET_SCHED_MAX];
71 unsigned int CLUTCH_THREAD_SELECT = -1;
72
73 /* Implementation of sched_runqueue_harness.h interface */
74
75 static test_pset_t single_pset = {
76 .cpu_type = TEST_CPU_TYPE_PERFORMANCE,
77 .num_cpus = 1,
78 .die_id = 0,
79 };
80 test_hw_topology_t single_core = {
81 .psets = &single_pset,
82 .num_psets = 1,
83 };
84
85 static char
test_cpu_type_to_char(test_cpu_type_t cpu_type)86 test_cpu_type_to_char(test_cpu_type_t cpu_type)
87 {
88 switch (cpu_type) {
89 case TEST_CPU_TYPE_PERFORMANCE:
90 return 'P';
91 case TEST_CPU_TYPE_EFFICIENCY:
92 return 'E';
93 default:
94 return '?';
95 }
96 }
97
98 void
clutch_impl_init_topology(test_hw_topology_t hw_topology)99 clutch_impl_init_topology(test_hw_topology_t hw_topology)
100 {
101 printf("��️ Mock HW Topology: %d psets {", hw_topology.num_psets);
102 assert(hw_topology.num_psets <= MAX_PSETS);
103 int total_cpus = 0;
104 for (int i = 0; i < hw_topology.num_psets; i++) {
105 assert((total_cpus + hw_topology.psets[i].num_cpus) <= MAX_CPUS);
106 if (i == 0) {
107 psets[0] = &pset0;
108 } else {
109 psets[i] = (struct processor_set *)malloc(sizeof(struct processor_set));
110 }
111 psets[i]->pset_cluster_id = i;
112 psets[i]->pset_id = i;
113 psets[i]->cpu_set_low = total_cpus;
114 psets[i]->cpu_bitmask = 0;
115 printf(" (%d: %d %c CPUs)", i, hw_topology.psets[i].num_cpus, test_cpu_type_to_char(hw_topology.psets[i].cpu_type));
116 for (int c = total_cpus; c < total_cpus + hw_topology.psets[i].num_cpus; c++) {
117 if (c == 0) {
118 cpus[0] = &cpu0;
119 } else {
120 cpus[c] = (struct processor *)malloc(sizeof(struct processor));
121 }
122 cpus[c]->cpu_id = c;
123 cpus[c]->processor_set = psets[i];
124 bit_set(psets[i]->cpu_bitmask, c);
125 cpus[c]->active_thread = NULL;
126 }
127 psets[i]->recommended_bitmask = psets[i]->cpu_bitmask;
128 psets[i]->cpu_available_map = psets[i]->cpu_bitmask;
129 total_cpus += hw_topology.psets[i].num_cpus;
130 }
131 processor_avail_count = total_cpus;
132 printf(" }\n");
133 }
134
135 static uint64_t unique_tg_id = 0;
136 static uint64_t unique_thread_id = 0;
137 #define NUM_LOGGED_TRACE_CODES 1
138 #define NUM_TRACEPOINT_FIELDS 5
139 static uint64_t logged_trace_codes[NUM_LOGGED_TRACE_CODES];
140 #define MAX_LOGGED_TRACEPOINTS 10000
141 static uint64_t *logged_tracepoints = NULL;
142 static uint32_t curr_tracepoint_ind = 0;
143 static uint32_t expect_tracepoint_ind = 0;
144
145 void
clutch_impl_init_params(void)146 clutch_impl_init_params(void)
147 {
148 /* Read out Clutch-internal fields for use by the test harness */
149 clutch_interactivity_score_max = 2 * sched_clutch_bucket_group_interactive_pri;
150 for (int b = TH_BUCKET_FIXPRI; b < TH_BUCKET_SCHED_MAX; b++) {
151 clutch_root_bucket_wcel_us[b] = sched_clutch_root_bucket_wcel_us[b] == SCHED_CLUTCH_INVALID_TIME_32 ? 0 : sched_clutch_root_bucket_wcel_us[b];
152 clutch_root_bucket_warp_us[b] = sched_clutch_root_bucket_warp_us[b] == SCHED_CLUTCH_INVALID_TIME_32 ? 0 : sched_clutch_root_bucket_warp_us[b];
153 }
154 CLUTCH_THREAD_SELECT = MACH_SCHED_CLUTCH_THREAD_SELECT;
155 }
156
157 void
clutch_impl_init_tracepoints(void)158 clutch_impl_init_tracepoints(void)
159 {
160 /* All filter-included tracepoints */
161 logged_trace_codes[0] = MACH_SCHED_CLUTCH_THREAD_SELECT;
162 /* Init harness-internal allocators */
163 logged_tracepoints = malloc(MAX_LOGGED_TRACEPOINTS * 5 * sizeof(uint64_t));
164 }
165
166 struct thread_group *
clutch_impl_create_tg(int interactivity_score)167 clutch_impl_create_tg(int interactivity_score)
168 {
169 struct thread_group *tg = malloc(sizeof(struct thread_group));
170 sched_clutch_init_with_thread_group(&tg->tg_sched_clutch, tg);
171 if (interactivity_score != INITIAL_INTERACTIVITY_SCORE) {
172 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) {
173 tg->tg_sched_clutch.sc_clutch_groups[bucket].scbg_interactivity_data.scct_count = interactivity_score;
174 tg->tg_sched_clutch.sc_clutch_groups[bucket].scbg_interactivity_data.scct_timestamp = mach_absolute_time();
175 }
176 }
177 tg->tg_id = unique_tg_id++;
178 return tg;
179 }
180
181 test_thread_t
clutch_impl_create_thread(int root_bucket,struct thread_group * tg,int pri)182 clutch_impl_create_thread(int root_bucket, struct thread_group *tg, int pri)
183 {
184 assert((sched_bucket_t)root_bucket == sched_convert_pri_to_bucket(pri) || (sched_bucket_t)root_bucket == TH_BUCKET_FIXPRI);
185 assert(tg != NULL);
186 thread_t thread = malloc(sizeof(struct thread));
187 thread->base_pri = pri;
188 thread->sched_pri = pri;
189 thread->thread_group = tg;
190 thread->th_sched_bucket = root_bucket;
191 thread->bound_processor = NULL;
192 thread->__runq.runq = PROCESSOR_NULL;
193 thread->thread_id = unique_thread_id++;
194 #if CONFIG_SCHED_EDGE
195 thread->th_bound_cluster_enqueued = false;
196 for (cluster_shared_rsrc_type_t shared_rsrc_type = CLUSTER_SHARED_RSRC_TYPE_MIN; shared_rsrc_type < CLUSTER_SHARED_RSRC_TYPE_COUNT; shared_rsrc_type++) {
197 thread->th_shared_rsrc_enqueued[shared_rsrc_type] = false;
198 thread->th_shared_rsrc_heavy_user[shared_rsrc_type] = false;
199 thread->th_shared_rsrc_heavy_perf_control[shared_rsrc_type] = false;
200 }
201 #endif /* CONFIG_SCHED_EDGE */
202 thread->th_bound_cluster_id = THREAD_BOUND_CLUSTER_NONE;
203 thread->reason = AST_NONE;
204 thread->sched_mode = TH_MODE_TIMESHARE;
205 thread->sched_flags = 0;
206 return thread;
207 }
208 void
clutch_impl_set_thread_sched_mode(test_thread_t thread,int mode)209 clutch_impl_set_thread_sched_mode(test_thread_t thread, int mode)
210 {
211 ((thread_t)thread)->sched_mode = (sched_mode_t)mode;
212 }
213 void
clutch_impl_set_thread_processor_bound(test_thread_t thread,int cpu_id)214 clutch_impl_set_thread_processor_bound(test_thread_t thread, int cpu_id)
215 {
216 ((thread_t)thread)->bound_processor = cpus[cpu_id];
217 }
218
219 void
clutch_impl_cpu_set_thread_current(int cpu_id,test_thread_t thread)220 clutch_impl_cpu_set_thread_current(int cpu_id, test_thread_t thread)
221 {
222 cpus[cpu_id]->active_thread = thread;
223 cpus[cpu_id]->first_timeslice = true;
224 /* Equivalent logic of processor_state_update_from_thread() */
225 cpus[cpu_id]->current_pri = ((thread_t)thread)->sched_pri;
226 cpus[cpu_id]->current_thread_group = ((thread_t)thread)->thread_group;
227 cpus[cpu_id]->current_is_bound = ((thread_t)thread)->bound_processor != PROCESSOR_NULL;
228 }
229
230 void
clutch_impl_cpu_clear_thread_current(int cpu_id)231 clutch_impl_cpu_clear_thread_current(int cpu_id)
232 {
233 cpus[cpu_id]->active_thread = NULL;
234 }
235
236 static bool
is_logged_clutch_trace_code(uint64_t clutch_trace_code)237 is_logged_clutch_trace_code(uint64_t clutch_trace_code)
238 {
239 for (int i = 0; i < NUM_LOGGED_TRACE_CODES; i++) {
240 if (logged_trace_codes[i] == clutch_trace_code) {
241 return true;
242 }
243 }
244 return false;
245 }
246
247 static bool
is_logged_trace_code(uint64_t trace_code)248 is_logged_trace_code(uint64_t trace_code)
249 {
250 if (KDBG_EXTRACT_CLASS(trace_code) == DBG_MACH && KDBG_EXTRACT_SUBCLASS(trace_code) == DBG_MACH_SCHED_CLUTCH) {
251 if (is_logged_clutch_trace_code(KDBG_EXTRACT_CODE(trace_code))) {
252 return true;
253 }
254 }
255 return false;
256 }
257
258 void
clutch_impl_log_tracepoint(uint64_t trace_code,uint64_t a1,uint64_t a2,uint64_t a3,uint64_t a4)259 clutch_impl_log_tracepoint(uint64_t trace_code, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4)
260 {
261 if (is_logged_trace_code(trace_code)) {
262 if (curr_tracepoint_ind < MAX_LOGGED_TRACEPOINTS) {
263 logged_tracepoints[curr_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 0] = KDBG_EXTRACT_CODE(trace_code);
264 logged_tracepoints[curr_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 1] = a1;
265 logged_tracepoints[curr_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 2] = a2;
266 logged_tracepoints[curr_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 3] = a3;
267 logged_tracepoints[curr_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 4] = a4;
268 } else if (curr_tracepoint_ind == MAX_LOGGED_TRACEPOINTS) {
269 printf("Ran out of pre-allocated memory to log tracepoints (%d points)...will no longer log tracepoints\n",
270 MAX_LOGGED_TRACEPOINTS);
271 }
272 curr_tracepoint_ind++;
273 }
274 }
275
276 void
clutch_impl_pop_tracepoint(uint64_t * clutch_trace_code,uint64_t * arg1,uint64_t * arg2,uint64_t * arg3,uint64_t * arg4)277 clutch_impl_pop_tracepoint(uint64_t *clutch_trace_code, uint64_t *arg1, uint64_t *arg2, uint64_t *arg3, uint64_t *arg4)
278 {
279 assert(expect_tracepoint_ind < curr_tracepoint_ind);
280 *clutch_trace_code = logged_tracepoints[expect_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 0];
281 *arg1 = logged_tracepoints[expect_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 1];
282 *arg2 = logged_tracepoints[expect_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 2];
283 *arg3 = logged_tracepoints[expect_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 3];
284 *arg4 = logged_tracepoints[expect_tracepoint_ind * NUM_TRACEPOINT_FIELDS + 4];
285 expect_tracepoint_ind++;
286 }
287