1 /*
2 * Copyright (c) 2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/mach_types.h>
30 #include <mach/machine.h>
31
32 #include <machine/machine_routines.h>
33 #include <machine/sched_param.h>
34 #include <machine/machine_cpu.h>
35
36 #include <kern/kern_types.h>
37 #include <kern/debug.h>
38 #include <kern/machine.h>
39 #include <kern/misc_protos.h>
40 #include <kern/processor.h>
41 #include <kern/queue.h>
42 #include <kern/sched.h>
43 #include <kern/sched_prim.h>
44 #include <kern/task.h>
45 #include <kern/thread.h>
46 #include <kern/thread_group.h>
47 #include <kern/sched_amp_common.h>
48
49 #include <sys/kdebug.h>
50
51 #if __AMP__ && !CONFIG_SCHED_EDGE
52
53 static thread_t
54 sched_amp_steal_thread(processor_set_t pset);
55
56 static void
57 sched_amp_thread_update_scan(sched_update_scan_context_t scan_context);
58
59 static boolean_t
60 sched_amp_processor_enqueue(processor_t processor, thread_t thread,
61 sched_options_t options);
62
63 static boolean_t
64 sched_amp_processor_queue_remove(processor_t processor, thread_t thread);
65
66 static ast_t
67 sched_amp_processor_csw_check(processor_t processor);
68
69 static boolean_t
70 sched_amp_processor_queue_has_priority(processor_t processor, int priority, boolean_t gte);
71
72 static int
73 sched_amp_runq_count(processor_t processor);
74
75 static boolean_t
76 sched_amp_processor_queue_empty(processor_t processor);
77
78 static uint64_t
79 sched_amp_runq_stats_count_sum(processor_t processor);
80
81 static int
82 sched_amp_processor_bound_count(processor_t processor);
83
84 static void
85 sched_amp_pset_init(processor_set_t pset);
86
87 static void
88 sched_amp_processor_init(processor_t processor);
89
90 static thread_t
91 sched_amp_choose_thread(processor_t processor, int priority, __unused thread_t prev, ast_t reason);
92
93 static void
94 sched_amp_processor_queue_shutdown(processor_t processor);
95
96 static sched_mode_t
97 sched_amp_initial_thread_sched_mode(task_t parent_task);
98
99 static processor_t
100 sched_amp_choose_processor(processor_set_t pset, processor_t processor, thread_t thread);
101
102 static bool
103 sched_amp_thread_avoid_processor(processor_t processor, thread_t thread, __unused ast_t reason);
104
105 static bool
106 sched_amp_thread_should_yield(processor_t processor, thread_t thread);
107
108 static void
109 sched_amp_thread_group_recommendation_change(struct thread_group *tg, cluster_type_t new_recommendation);
110
111 static bool
112 sched_amp_thread_eligible_for_pset(thread_t thread, processor_set_t pset);
113
114 static void
115 sched_amp_cpu_init_completed(void);
116
117 const struct sched_dispatch_table sched_amp_dispatch = {
118 .sched_name = "amp",
119 .init = sched_amp_init,
120 .timebase_init = sched_timeshare_timebase_init,
121 .processor_init = sched_amp_processor_init,
122 .pset_init = sched_amp_pset_init,
123 .maintenance_continuation = sched_timeshare_maintenance_continue,
124 .choose_thread = sched_amp_choose_thread,
125 .steal_thread_enabled = sched_amp_steal_thread_enabled,
126 .steal_thread = sched_amp_steal_thread,
127 .compute_timeshare_priority = sched_compute_timeshare_priority,
128 .choose_node = sched_amp_choose_node,
129 .choose_processor = sched_amp_choose_processor,
130 .processor_enqueue = sched_amp_processor_enqueue,
131 .processor_queue_shutdown = sched_amp_processor_queue_shutdown,
132 .processor_queue_remove = sched_amp_processor_queue_remove,
133 .processor_queue_empty = sched_amp_processor_queue_empty,
134 .priority_is_urgent = priority_is_urgent,
135 .processor_csw_check = sched_amp_processor_csw_check,
136 .processor_queue_has_priority = sched_amp_processor_queue_has_priority,
137 .initial_quantum_size = sched_timeshare_initial_quantum_size,
138 .initial_thread_sched_mode = sched_amp_initial_thread_sched_mode,
139 .can_update_priority = can_update_priority,
140 .update_priority = update_priority,
141 .lightweight_update_priority = lightweight_update_priority,
142 .quantum_expire = sched_default_quantum_expire,
143 .processor_runq_count = sched_amp_runq_count,
144 .processor_runq_stats_count_sum = sched_amp_runq_stats_count_sum,
145 .processor_bound_count = sched_amp_processor_bound_count,
146 .thread_update_scan = sched_amp_thread_update_scan,
147 .multiple_psets_enabled = TRUE,
148 .avoid_processor_enabled = TRUE,
149 .thread_avoid_processor = sched_amp_thread_avoid_processor,
150 .processor_balance = sched_amp_balance,
151
152 .rt_runq = sched_rtlocal_runq,
153 .rt_init = sched_rtlocal_init,
154 .rt_queue_shutdown = sched_rtlocal_queue_shutdown,
155 .rt_runq_scan = sched_rtlocal_runq_scan,
156 .rt_runq_count_sum = sched_rtlocal_runq_count_sum,
157 .rt_steal_thread = sched_rtlocal_steal_thread,
158
159 .qos_max_parallelism = sched_amp_qos_max_parallelism,
160 .check_spill = sched_amp_check_spill,
161 .ipi_policy = sched_amp_ipi_policy,
162 .thread_should_yield = sched_amp_thread_should_yield,
163 .run_count_incr = sched_run_incr,
164 .run_count_decr = sched_run_decr,
165 .update_thread_bucket = sched_update_thread_bucket,
166 .pset_made_schedulable = sched_pset_made_schedulable,
167 .thread_group_recommendation_change = sched_amp_thread_group_recommendation_change,
168 .cpu_init_completed = sched_amp_cpu_init_completed,
169 .thread_eligible_for_pset = sched_amp_thread_eligible_for_pset,
170 };
171
172 extern processor_set_t ecore_set;
173 extern processor_set_t pcore_set;
174
175 __attribute__((always_inline))
176 static inline run_queue_t
amp_main_runq(processor_t processor)177 amp_main_runq(processor_t processor)
178 {
179 return &processor->processor_set->pset_runq;
180 }
181
182 __attribute__((always_inline))
183 static inline run_queue_t
amp_bound_runq(processor_t processor)184 amp_bound_runq(processor_t processor)
185 {
186 return &processor->runq;
187 }
188
189 __attribute__((always_inline))
190 static inline run_queue_t
amp_runq_for_thread(processor_t processor,thread_t thread)191 amp_runq_for_thread(processor_t processor, thread_t thread)
192 {
193 if (thread->bound_processor == PROCESSOR_NULL) {
194 return amp_main_runq(processor);
195 } else {
196 assert(thread->bound_processor == processor);
197 return amp_bound_runq(processor);
198 }
199 }
200
201 static sched_mode_t
sched_amp_initial_thread_sched_mode(task_t parent_task)202 sched_amp_initial_thread_sched_mode(task_t parent_task)
203 {
204 if (parent_task == kernel_task) {
205 return TH_MODE_FIXED;
206 } else {
207 return TH_MODE_TIMESHARE;
208 }
209 }
210
211 static void
sched_amp_processor_init(processor_t processor)212 sched_amp_processor_init(processor_t processor)
213 {
214 run_queue_init(&processor->runq);
215 }
216
217 static void
sched_amp_pset_init(processor_set_t pset)218 sched_amp_pset_init(processor_set_t pset)
219 {
220 if (pset->pset_cluster_type == PSET_AMP_P) {
221 pset->pset_type = CLUSTER_TYPE_P;
222 pcore_set = pset;
223 } else {
224 assert(pset->pset_cluster_type == PSET_AMP_E);
225 pset->pset_type = CLUSTER_TYPE_E;
226 ecore_set = pset;
227 }
228 run_queue_init(&pset->pset_runq);
229 }
230
231 static thread_t
sched_amp_choose_thread(processor_t processor,int priority,__unused thread_t prev_thread,__unused ast_t reason)232 sched_amp_choose_thread(
233 processor_t processor,
234 int priority,
235 __unused thread_t prev_thread,
236 __unused ast_t reason)
237 {
238 processor_set_t pset = processor->processor_set;
239 bool spill_pending = false;
240 int spill_pri = -1;
241
242 if (pset == ecore_set && bit_test(pset->pending_spill_cpu_mask, processor->cpu_id)) {
243 spill_pending = true;
244 spill_pri = pcore_set->pset_runq.highq;
245 }
246
247 run_queue_t main_runq = amp_main_runq(processor);
248 run_queue_t bound_runq = amp_bound_runq(processor);
249 run_queue_t chosen_runq;
250
251 if ((bound_runq->highq < priority) &&
252 (main_runq->highq < priority) &&
253 (spill_pri < priority)) {
254 return THREAD_NULL;
255 }
256
257 if ((spill_pri > bound_runq->highq) &&
258 (spill_pri > main_runq->highq)) {
259 /*
260 * There is a higher priority thread on the P-core runq,
261 * so returning THREAD_NULL here will cause thread_select()
262 * to call sched_amp_steal_thread() to try to get it.
263 */
264 return THREAD_NULL;
265 }
266
267 if (bound_runq->highq >= main_runq->highq) {
268 chosen_runq = bound_runq;
269 } else {
270 chosen_runq = main_runq;
271 }
272
273 return run_queue_dequeue(chosen_runq, SCHED_HEADQ);
274 }
275
276 static boolean_t
sched_amp_processor_enqueue(processor_t processor,thread_t thread,sched_options_t options)277 sched_amp_processor_enqueue(
278 processor_t processor,
279 thread_t thread,
280 sched_options_t options)
281 {
282 run_queue_t rq = amp_runq_for_thread(processor, thread);
283 boolean_t result;
284
285 result = run_queue_enqueue(rq, thread, options);
286 thread_set_runq_locked(thread, processor);
287
288 return result;
289 }
290
291 static boolean_t
sched_amp_processor_queue_empty(processor_t processor)292 sched_amp_processor_queue_empty(processor_t processor)
293 {
294 processor_set_t pset = processor->processor_set;
295 bool spill_pending = bit_test(pset->pending_spill_cpu_mask, processor->cpu_id);
296
297 return (amp_main_runq(processor)->count == 0) &&
298 (amp_bound_runq(processor)->count == 0) &&
299 !spill_pending;
300 }
301
302 static bool
sched_amp_thread_should_yield(processor_t processor,thread_t thread)303 sched_amp_thread_should_yield(processor_t processor, thread_t thread)
304 {
305 if (!sched_amp_processor_queue_empty(processor) || (rt_runq_count(processor->processor_set) > 0)) {
306 return true;
307 }
308
309 if ((processor->processor_set->pset_cluster_type == PSET_AMP_E) && (recommended_pset_type(thread) == PSET_AMP_P)) {
310 return pcore_set && pcore_set->pset_runq.count > 0;
311 }
312
313 return false;
314 }
315
316 static ast_t
sched_amp_processor_csw_check(processor_t processor)317 sched_amp_processor_csw_check(processor_t processor)
318 {
319 boolean_t has_higher;
320 int pri;
321
322 run_queue_t main_runq = amp_main_runq(processor);
323 run_queue_t bound_runq = amp_bound_runq(processor);
324
325 assert(processor->active_thread != NULL);
326
327 processor_set_t pset = processor->processor_set;
328 bool spill_pending = false;
329 int spill_pri = -1;
330 int spill_urgency = 0;
331
332 if (pset == ecore_set && bit_test(pset->pending_spill_cpu_mask, processor->cpu_id)) {
333 spill_pending = true;
334 spill_pri = pcore_set->pset_runq.highq;
335 spill_urgency = pcore_set->pset_runq.urgency;
336 }
337
338 pri = MAX(main_runq->highq, bound_runq->highq);
339 if (spill_pending) {
340 pri = MAX(pri, spill_pri);
341 }
342
343 if (processor->first_timeslice) {
344 has_higher = (pri > processor->current_pri);
345 } else {
346 has_higher = (pri >= processor->current_pri);
347 }
348
349 if (has_higher) {
350 if (main_runq->urgency > 0) {
351 return AST_PREEMPT | AST_URGENT;
352 }
353
354 if (bound_runq->urgency > 0) {
355 return AST_PREEMPT | AST_URGENT;
356 }
357
358 if (spill_urgency > 0) {
359 return AST_PREEMPT | AST_URGENT;
360 }
361
362 return AST_PREEMPT;
363 }
364
365 return AST_NONE;
366 }
367
368 static boolean_t
sched_amp_processor_queue_has_priority(processor_t processor,int priority,boolean_t gte)369 sched_amp_processor_queue_has_priority(processor_t processor,
370 int priority,
371 boolean_t gte)
372 {
373 bool spill_pending = false;
374 int spill_pri = -1;
375 processor_set_t pset = processor->processor_set;
376
377 if (pset == ecore_set && bit_test(pset->pending_spill_cpu_mask, processor->cpu_id)) {
378 spill_pending = true;
379 spill_pri = pcore_set->pset_runq.highq;
380 }
381 run_queue_t main_runq = amp_main_runq(processor);
382 run_queue_t bound_runq = amp_bound_runq(processor);
383
384 int qpri = MAX(main_runq->highq, bound_runq->highq);
385 if (spill_pending) {
386 qpri = MAX(qpri, spill_pri);
387 }
388
389 if (gte) {
390 return qpri >= priority;
391 } else {
392 return qpri > priority;
393 }
394 }
395
396 static int
sched_amp_runq_count(processor_t processor)397 sched_amp_runq_count(processor_t processor)
398 {
399 return amp_main_runq(processor)->count + amp_bound_runq(processor)->count;
400 }
401
402 static uint64_t
sched_amp_runq_stats_count_sum(processor_t processor)403 sched_amp_runq_stats_count_sum(processor_t processor)
404 {
405 uint64_t bound_sum = amp_bound_runq(processor)->runq_stats.count_sum;
406
407 if (processor->cpu_id == processor->processor_set->cpu_set_low) {
408 return bound_sum + amp_main_runq(processor)->runq_stats.count_sum;
409 } else {
410 return bound_sum;
411 }
412 }
413 static int
sched_amp_processor_bound_count(processor_t processor)414 sched_amp_processor_bound_count(processor_t processor)
415 {
416 return amp_bound_runq(processor)->count;
417 }
418
419 static void
sched_amp_processor_queue_shutdown(processor_t processor)420 sched_amp_processor_queue_shutdown(processor_t processor)
421 {
422 processor_set_t pset = processor->processor_set;
423 run_queue_t rq = amp_main_runq(processor);
424 thread_t thread;
425 queue_head_t tqueue;
426
427 /* We only need to migrate threads if this is the last active or last recommended processor in the pset */
428 if ((pset->online_processor_count > 0) && pset_is_recommended(pset)) {
429 pset_unlock(pset);
430 return;
431 }
432
433 queue_init(&tqueue);
434
435 while (rq->count > 0) {
436 thread = run_queue_dequeue(rq, SCHED_HEADQ);
437 enqueue_tail(&tqueue, &thread->runq_links);
438 }
439
440 pset_unlock(pset);
441
442 qe_foreach_element_safe(thread, &tqueue, runq_links) {
443 remqueue(&thread->runq_links);
444
445 thread_lock(thread);
446
447 thread_setrun(thread, SCHED_TAILQ);
448
449 thread_unlock(thread);
450 }
451 }
452
453 static boolean_t
sched_amp_processor_queue_remove(processor_t processor,thread_t thread)454 sched_amp_processor_queue_remove(
455 processor_t processor,
456 thread_t thread)
457 {
458 processor_set_t pset = processor->processor_set;
459
460 pset_lock(pset);
461
462 if (processor == thread_get_runq_locked(thread)) {
463 /*
464 * Thread is on a run queue and we have a lock on
465 * that run queue.
466 */
467 run_queue_t rq = amp_runq_for_thread(processor, thread);
468 run_queue_remove(rq, thread);
469 } else {
470 /*
471 * The thread left the run queue before we could
472 * lock the run queue.
473 */
474 thread_assert_runq_null(thread);
475 processor = PROCESSOR_NULL;
476 }
477
478 pset_unlock(pset);
479
480 return processor != PROCESSOR_NULL;
481 }
482
483 /*
484 * sched_amp_steal_thread()
485 *
486 */
487 thread_t
sched_amp_steal_thread(processor_set_t pset)488 sched_amp_steal_thread(processor_set_t pset)
489 {
490 thread_t thread = THREAD_NULL;
491 processor_set_t nset = pset;
492
493 assert(pset->pset_cluster_type != PSET_AMP_P);
494
495 processor_t processor = current_processor();
496 assert(pset == processor->processor_set);
497
498 bool spill_pending = bit_test(pset->pending_spill_cpu_mask, processor->cpu_id);
499 bit_clear(pset->pending_spill_cpu_mask, processor->cpu_id);
500
501 if (!pcore_set) {
502 return THREAD_NULL;
503 }
504
505 nset = pcore_set;
506
507 assert(nset != pset);
508
509 if (sched_get_pset_load_average(nset, 0) >= sched_amp_steal_threshold(nset, spill_pending)) {
510 pset_unlock(pset);
511
512 pset = nset;
513
514 pset_lock(pset);
515
516 /* Allow steal if load average still OK, no idle cores, and more threads on runq than active cores DISPATCHING */
517 if ((sched_get_pset_load_average(pset, 0) >= sched_amp_steal_threshold(pset, spill_pending)) &&
518 (pset->pset_runq.count > bit_count(pset->cpu_state_map[PROCESSOR_DISPATCHING])) &&
519 (bit_count(pset->recommended_bitmask & pset->cpu_state_map[PROCESSOR_IDLE]) == 0)) {
520 thread = run_queue_dequeue(&pset->pset_runq, SCHED_HEADQ);
521 KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_AMP_STEAL) | DBG_FUNC_NONE, spill_pending, 0, 0, 0);
522 sched_update_pset_load_average(pset, 0);
523 }
524 }
525
526 pset_unlock(pset);
527 return thread;
528 }
529
530
531
532 static void
sched_amp_thread_update_scan(sched_update_scan_context_t scan_context)533 sched_amp_thread_update_scan(sched_update_scan_context_t scan_context)
534 {
535 boolean_t restart_needed = FALSE;
536 processor_t processor;
537 processor_set_t pset;
538 thread_t thread;
539 spl_t s;
540
541 /*
542 * We update the threads associated with each processor (bound and idle threads)
543 * and then update the threads in each pset runqueue.
544 */
545
546 do {
547 for (int i = 0; i < machine_info.logical_cpu_max; i++) {
548 processor = processor_array[i];
549 if (processor == NULL) {
550 continue;
551 }
552
553 pset = processor->processor_set;
554
555 s = splsched();
556 pset_lock(pset);
557
558 restart_needed = runq_scan(amp_bound_runq(processor), scan_context);
559
560 pset_unlock(pset);
561 splx(s);
562
563 if (restart_needed) {
564 break;
565 }
566
567 thread = processor->idle_thread;
568 if (thread != THREAD_NULL && thread->sched_stamp != sched_tick) {
569 if (thread_update_add_thread(thread) == FALSE) {
570 restart_needed = TRUE;
571 break;
572 }
573 }
574 }
575
576 /* Ok, we now have a collection of candidates -- fix them. */
577 thread_update_process_threads();
578 } while (restart_needed);
579
580 pset_node_t node = &pset_node0;
581 pset = node->psets;
582
583 do {
584 do {
585 restart_needed = FALSE;
586 while (pset != NULL) {
587 s = splsched();
588 pset_lock(pset);
589
590 restart_needed = runq_scan(&pset->pset_runq, scan_context);
591
592 pset_unlock(pset);
593 splx(s);
594
595 if (restart_needed) {
596 break;
597 }
598
599 pset = pset->pset_list;
600 }
601
602 if (restart_needed) {
603 break;
604 }
605 } while (((node = node->node_list) != NULL) && ((pset = node->psets) != NULL));
606
607 /* Ok, we now have a collection of candidates -- fix them. */
608 thread_update_process_threads();
609 } while (restart_needed);
610 }
611
612 static bool
pcores_recommended(thread_t thread)613 pcores_recommended(thread_t thread)
614 {
615 if (!pcore_set) {
616 return false;
617 }
618
619 if (pcore_set->online_processor_count == 0) {
620 /* No pcores available */
621 return false;
622 }
623
624 if (!pset_is_recommended(ecore_set)) {
625 /* No E cores recommended, must use P cores */
626 return true;
627 }
628
629 if (recommended_pset_type(thread) == PSET_AMP_E) {
630 return false;
631 }
632
633 return pset_is_recommended(pcore_set);
634 }
635
636 /* Return true if this thread should not continue running on this processor */
637 static bool
sched_amp_thread_avoid_processor(processor_t processor,thread_t thread,__unused ast_t reason)638 sched_amp_thread_avoid_processor(processor_t processor, thread_t thread, __unused ast_t reason)
639 {
640 if (processor->processor_set->pset_cluster_type == PSET_AMP_E) {
641 if (pcores_recommended(thread)) {
642 return true;
643 }
644 } else if (processor->processor_set->pset_cluster_type == PSET_AMP_P) {
645 if (!pcores_recommended(thread)) {
646 return true;
647 }
648 }
649
650 return false;
651 }
652
653 static processor_t
sched_amp_choose_processor(processor_set_t pset,processor_t processor,thread_t thread)654 sched_amp_choose_processor(processor_set_t pset, processor_t processor, thread_t thread)
655 {
656 /* Bound threads don't call this function */
657 assert(thread->bound_processor == PROCESSOR_NULL);
658
659 processor_set_t nset = pset;
660 bool choose_pcores;
661
662
663 choose_pcores = pcores_recommended(thread);
664
665 if (choose_pcores && (pset->pset_cluster_type != PSET_AMP_P)) {
666 nset = pcore_set;
667 assert(nset != NULL);
668 } else if (!choose_pcores && (pset->pset_cluster_type != PSET_AMP_E)) {
669 nset = ecore_set;
670 assert(nset != NULL);
671 }
672
673 if (nset != pset) {
674 pset_unlock(pset);
675 pset_lock(nset);
676 }
677
678 /* Now that the chosen pset is definitely locked, make sure nothing important has changed */
679 if (!pset_is_recommended(nset)) {
680 pset_unlock(nset);
681 return PROCESSOR_NULL;
682 }
683
684 return choose_processor(nset, processor, thread);
685 }
686
687 void
sched_amp_thread_group_recommendation_change(struct thread_group * tg,cluster_type_t new_recommendation)688 sched_amp_thread_group_recommendation_change(struct thread_group *tg, cluster_type_t new_recommendation)
689 {
690 thread_group_update_recommendation(tg, new_recommendation);
691
692 if (new_recommendation != CLUSTER_TYPE_P) {
693 return;
694 }
695
696 sched_amp_bounce_thread_group_from_ecores(ecore_set, tg);
697 }
698
699 static bool
sched_amp_thread_eligible_for_pset(thread_t thread,processor_set_t pset)700 sched_amp_thread_eligible_for_pset(thread_t thread, processor_set_t pset)
701 {
702 if (recommended_pset_type(thread) == PSET_AMP_P) {
703 /* P-recommended threads are eligible to execute on either E or P clusters */
704 return true;
705 } else {
706 /* E-recommended threads are eligible to execute on E clusters only */
707 return pset->pset_cluster_type == PSET_AMP_E;
708 }
709 }
710
711 static char *pct_name[] = {
712 "PSET_SMP",
713 "PSET_AMP_E",
714 "PSET_AMP_P"
715 };
716
717 static void
sched_amp_cpu_init_completed(void)718 sched_amp_cpu_init_completed(void)
719 {
720 if (PE_parse_boot_argn("cpus", NULL, 0) || PE_parse_boot_argn("cpumask", NULL, 0)) {
721 /* If number of cpus booted is restricted, these asserts may not be true */
722 return;
723 }
724
725 assert(pset_array[0] != NULL);
726 assert(pset_array[1] != NULL);
727
728 assert(ecore_set != NULL);
729 assert(pcore_set != NULL);
730
731 if (pset_array[0] == ecore_set) {
732 assert(pset_array[1] == pcore_set);
733 } else {
734 assert(pset_array[0] == pcore_set);
735 assert(pset_array[1] == ecore_set);
736 }
737
738 for (processor_t p = processor_list; p != NULL; p = p->processor_list) {
739 processor_set_t pset = p->processor_set;
740 kprintf("%s>cpu_id %02d in pset_id %02d type %s\n", __FUNCTION__, p->cpu_id, pset->pset_id,
741 pct_name[pset->pset_cluster_type]);
742
743 assert(p == processor_array[p->cpu_id]);
744 assert(pset->pset_cluster_type != PSET_SMP);
745 if (pset->pset_cluster_type == PSET_AMP_E) {
746 assert(pset->pset_type == CLUSTER_TYPE_E);
747 assert(pset == ecore_set);
748 } else {
749 assert(pset->pset_cluster_type == PSET_AMP_P);
750 assert(pset->pset_type == CLUSTER_TYPE_P);
751 assert(pset == pcore_set);
752 }
753 }
754 }
755
756 #endif /* __AMP__ && !CONFIG_SCHED_EDGE */
757