1 /*
2 * Copyright (c) 2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <os/atomic_private.h>
30 #include <machine/machine_routines.h>
31 #include <kern/processor.h>
32
33 #include <kern/sched_common.h>
34
35 #if __AMP__
36
37 void
sched_pset_search_order_compute(sched_pset_search_order_t * search_order_out,sched_pset_search_order_sort_data_t * datas,size_t num_datas,sched_pset_search_order_sort_cmpfunc_t cmp)38 sched_pset_search_order_compute(sched_pset_search_order_t *search_order_out,
39 sched_pset_search_order_sort_data_t *datas, size_t num_datas,
40 sched_pset_search_order_sort_cmpfunc_t cmp)
41 {
42 qsort(datas, num_datas, sizeof(sched_pset_search_order_sort_data_t), cmp);
43 sched_pset_search_order_t search_order;
44 for (int i = 0; i < num_datas; i++) {
45 search_order.spso_search_order[i] = datas[i].spsosd_dst_pset_id;
46 }
47 int num_psets = ml_get_cluster_count();
48 for (int i = (int)num_datas; i < num_psets - 1; i++) {
49 /*
50 * If fewer sort datas were passed in than the number of psets minus
51 * 1 (AKA the maximum length of a pset search order), then mark the
52 * remaining slots at the end with an invalid pset id.
53 */
54 search_order.spso_search_order[i] = PSET_ID_INVALID;
55 }
56 os_atomic_store_wide(&search_order_out->spso_packed, search_order.spso_packed, relaxed);
57 }
58
59 void
sched_pset_search_order_init(processor_set_t src_pset,sched_pset_search_order_t * search_order_out)60 sched_pset_search_order_init(processor_set_t src_pset, sched_pset_search_order_t *search_order_out)
61 {
62 pset_id_t other_pset_id = 0;
63 sched_pset_search_order_t spill_order;
64 int num_psets = ml_get_cluster_count();
65 for (int i = 0; i < MAX_PSETS - 1; i++, other_pset_id++) {
66 if (i < num_psets - 1) {
67 if (other_pset_id == src_pset->pset_id) {
68 /* Exclude the source pset */
69 other_pset_id++;
70 }
71 assert3u(other_pset_id, <, num_psets);
72 spill_order.spso_search_order[i] = other_pset_id;
73 } else {
74 /* Mark unneeded slots with an invalid id, as they should not be accessed */
75 spill_order.spso_search_order[i] = PSET_ID_INVALID;
76 }
77 }
78 os_atomic_store_wide(&search_order_out->spso_packed, spill_order.spso_packed, relaxed);
79 }
80
81 bool
sched_iterate_psets_ordered(processor_set_t starting_pset,sched_pset_search_order_t * search_order,uint64_t candidate_map,sched_pset_iterate_state_t * istate)82 sched_iterate_psets_ordered(processor_set_t starting_pset, sched_pset_search_order_t *search_order,
83 uint64_t candidate_map, sched_pset_iterate_state_t *istate)
84 {
85 int num_psets = ml_get_cluster_count();
86 while (istate->spis_search_index < num_psets - 1) {
87 int pset_id;
88 if (istate->spis_search_index == -1) {
89 /* Initial condition */
90 pset_id = starting_pset->pset_id;
91 istate->spis_cached_search_order =
92 (sched_pset_search_order_t)os_atomic_load_wide(&search_order->spso_packed, relaxed);
93 } else {
94 pset_id = istate->spis_cached_search_order.spso_search_order[istate->spis_search_index];
95 if (pset_id == PSET_ID_INVALID) {
96 /* The given search order does not include all psets */
97 break;
98 }
99 assert3u(pset_id, !=, starting_pset->pset_id);
100 }
101 istate->spis_search_index++;
102 if (bit_test(candidate_map, pset_id)) {
103 istate->spis_pset_id = pset_id;
104 return true;
105 }
106 }
107 istate->spis_pset_id = -1;
108 return false;
109 }
110
111 #endif /* __AMP__ */
112