1 /*
2 * Copyright (c) 2007-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach/machine.h>
30 #include <mach/processor.h>
31 #include <kern/kalloc.h>
32 #include <i386/cpu_affinity.h>
33 #include <i386/cpu_topology.h>
34 #include <i386/cpu_threads.h>
35 #include <i386/machine_cpu.h>
36 #include <i386/bit_routines.h>
37 #include <i386/cpu_data.h>
38 #include <i386/lapic.h>
39 #include <i386/machine_routines.h>
40 #include <stddef.h>
41
42 __private_extern__ void qsort(
43 void * array,
44 size_t nmembers,
45 size_t member_size,
46 int (*)(const void *, const void *));
47
48 static int lapicid_cmp(const void *x, const void *y);
49 static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
50
51 x86_affinity_set_t *x86_affinities = NULL;
52 static int x86_affinity_count = 0;
53
54 extern cpu_data_t cpshadows[];
55
56 #if DEVELOPMENT || DEBUG
57 void iotrace_init(void);
58 void traptrace_init(void);
59 #endif /* DEVELOPMENT || DEBUG */
60
61
62 /* Re-sort double-mapped CPU data shadows after topology discovery sorts the
63 * primary CPU data structures by physical/APIC CPU ID.
64 */
65 static void
cpu_shadow_sort(int ncpus)66 cpu_shadow_sort(int ncpus)
67 {
68 for (int i = 0; i < ncpus; i++) {
69 cpu_data_t *cpup = cpu_datap(i);
70 ptrdiff_t coff = cpup - cpu_datap(0);
71
72 cpup->cd_shadow = &cpshadows[coff];
73 }
74 }
75
76 /*
77 * cpu_topology_sort() is called after all processors have been registered but
78 * before any non-boot processor is started. We establish canonical logical
79 * processor numbering - logical cpus must be contiguous, zero-based and
80 * assigned in physical (local apic id) order. This step is required because
81 * the discovery/registration order is non-deterministic - cores are registered
82 * in differing orders over boots. Enforcing canonical numbering simplifies
83 * identification of processors.
84 */
85 void
cpu_topology_sort(int ncpus)86 cpu_topology_sort(int ncpus)
87 {
88 int i;
89 boolean_t istate;
90 processor_t lprim = NULL;
91
92 assert(machine_info.physical_cpu == 1);
93 assert(machine_info.logical_cpu == 1);
94 assert(master_cpu == 0);
95 assert(cpu_number() == 0);
96 assert(cpu_datap(0)->cpu_number == 0);
97
98 uint32_t cpus_per_pset = 0;
99
100 #if DEVELOPMENT || DEBUG
101 PE_parse_boot_argn("cpus_per_pset", &cpus_per_pset, sizeof(cpus_per_pset));
102 #endif
103
104 /* Lights out for this */
105 istate = ml_set_interrupts_enabled(FALSE);
106
107 if (topo_dbg) {
108 TOPO_DBG("cpu_topology_start() %d cpu%s registered\n",
109 ncpus, (ncpus > 1) ? "s" : "");
110 for (i = 0; i < ncpus; i++) {
111 cpu_data_t *cpup = cpu_datap(i);
112 TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
113 i, (void *) cpup, cpup->cpu_phys_number);
114 }
115 }
116
117 /*
118 * Re-order the cpu_data_ptr vector sorting by physical id.
119 * Skip the boot processor, it's required to be correct.
120 */
121 if (ncpus > 1) {
122 qsort((void *) &cpu_data_ptr[1],
123 ncpus - 1,
124 sizeof(cpu_data_t *),
125 lapicid_cmp);
126 }
127 if (topo_dbg) {
128 TOPO_DBG("cpu_topology_start() after sorting:\n");
129 for (i = 0; i < ncpus; i++) {
130 cpu_data_t *cpup = cpu_datap(i);
131 TOPO_DBG("\tcpu_data[%d]:%p local apic 0x%x\n",
132 i, (void *) cpup, cpup->cpu_phys_number);
133 }
134 }
135
136 /*
137 * Finalize logical numbers and map kept by the lapic code.
138 */
139 for (i = 0; i < ncpus; i++) {
140 cpu_data_t *cpup = cpu_datap(i);
141
142 if (cpup->cpu_number != i) {
143 kprintf("cpu_datap(%d):%p local apic id 0x%x "
144 "remapped from %d\n",
145 i, cpup, cpup->cpu_phys_number,
146 cpup->cpu_number);
147 }
148 cpup->cpu_number = i;
149 lapic_cpu_map(cpup->cpu_phys_number, i);
150 x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
151 }
152
153 cpu_shadow_sort(ncpus);
154 x86_validate_topology();
155
156 ml_set_interrupts_enabled(istate);
157 TOPO_DBG("cpu_topology_start() LLC is L%d\n", topoParms.LLCDepth + 1);
158
159 #if DEVELOPMENT || DEBUG
160 iotrace_init();
161 traptrace_init();
162 #endif /* DEVELOPMENT || DEBUG */
163
164 /*
165 * Let the CPU Power Management know that the topology is stable.
166 */
167 topoParms.stable = TRUE;
168 pmCPUStateInit();
169
170 /*
171 * Iterate over all logical cpus finding or creating the affinity set
172 * for their LLC cache. Each affinity set possesses a processor set
173 * into which each logical processor is added.
174 */
175 TOPO_DBG("cpu_topology_start() creating affinity sets:ncpus=%d max_cpus=%d\n", ncpus, machine_info.max_cpus);
176
177 uint32_t pset_cluster_id = 0;
178 for (i = 0; i < machine_info.max_cpus; i++) {
179 cpu_data_t *cpup = cpu_datap(i);
180 x86_lcpu_t *lcpup = cpu_to_lcpu(i);
181 x86_cpu_cache_t *LLC_cachep;
182 x86_affinity_set_t *aset;
183
184 LLC_cachep = lcpup->caches[topoParms.LLCDepth];
185 assert(LLC_cachep->type == CPU_CACHE_TYPE_UNIF);
186 aset = find_cache_affinity(LLC_cachep);
187 if ((aset == NULL) || ((cpus_per_pset != 0) && (i % cpus_per_pset) == 0)) {
188 aset = kalloc_type(x86_affinity_set_t, Z_WAITOK | Z_NOFAIL);
189 aset->next = x86_affinities;
190 x86_affinities = aset;
191 aset->num = x86_affinity_count++;
192 aset->cache = LLC_cachep;
193 if (i == master_cpu) {
194 aset->pset = processor_pset(master_processor);
195 } else {
196 pset_cluster_id++;
197 aset->pset = pset_create(pset_node_root(), PSET_SMP, pset_cluster_id, pset_cluster_id);
198 if (aset->pset == PROCESSOR_SET_NULL) {
199 panic("cpu_topology_start: pset_create");
200 }
201 }
202 TOPO_DBG("\tnew set %p(%d) pset %p for cache %p\n",
203 aset, aset->num, aset->pset, aset->cache);
204 }
205
206 TOPO_DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
207 aset, aset->num, lcpup, lcpup->cpu_num, cpup, cpup->cpu_processor);
208
209 if (i != master_cpu) {
210 processor_init(cpup->cpu_processor, i, aset->pset);
211 }
212
213 if (lcpup->core->num_lcpus > 1) {
214 if (lcpup->lnum == 0) {
215 lprim = cpup->cpu_processor;
216 }
217
218 processor_set_primary(cpup->cpu_processor, lprim);
219 }
220 }
221
222 if (machine_info.max_cpus < machine_info.logical_cpu_max) {
223 /* boot-args cpus=n is set, so adjust max numbers to match */
224 int logical_max = machine_info.max_cpus;
225 int physical_max = logical_max;
226 if (machine_info.logical_cpu_max != machine_info.physical_cpu_max) {
227 physical_max = (logical_max + 1) / 2;
228 }
229 machine_info.logical_cpu_max = logical_max;
230 machine_info.physical_cpu_max = physical_max;
231 }
232 }
233
234 /* We got a request to start a CPU. Check that this CPU is within the
235 * max cpu limit set before we do.
236 */
237 kern_return_t
cpu_topology_start_cpu(int cpunum)238 cpu_topology_start_cpu( int cpunum )
239 {
240 int ncpus = machine_info.max_cpus;
241 int i = cpunum;
242
243 /* Decide whether to start a CPU, and actually start it */
244 TOPO_DBG("cpu_topology_start() processor_start():\n");
245 if (i < ncpus) {
246 TOPO_DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
247 processor_start(cpu_datap(i)->cpu_processor);
248 return KERN_SUCCESS;
249 } else {
250 return KERN_FAILURE;
251 }
252 }
253
254 static int
lapicid_cmp(const void * x,const void * y)255 lapicid_cmp(const void *x, const void *y)
256 {
257 cpu_data_t *cpu_x = *((cpu_data_t **)(uintptr_t)x);
258 cpu_data_t *cpu_y = *((cpu_data_t **)(uintptr_t)y);
259
260 TOPO_DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
261 x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
262 if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number) {
263 return -1;
264 }
265 if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number) {
266 return 0;
267 }
268 return 1;
269 }
270
271 static x86_affinity_set_t *
find_cache_affinity(x86_cpu_cache_t * l2_cachep)272 find_cache_affinity(x86_cpu_cache_t *l2_cachep)
273 {
274 x86_affinity_set_t *aset;
275
276 for (aset = x86_affinities; aset != NULL; aset = aset->next) {
277 if (l2_cachep == aset->cache) {
278 break;
279 }
280 }
281 return aset;
282 }
283
284 int
ml_get_max_affinity_sets(void)285 ml_get_max_affinity_sets(void)
286 {
287 return x86_affinity_count;
288 }
289
290 processor_set_t
ml_affinity_to_pset(uint32_t affinity_num)291 ml_affinity_to_pset(uint32_t affinity_num)
292 {
293 x86_affinity_set_t *aset;
294
295 for (aset = x86_affinities; aset != NULL; aset = aset->next) {
296 if (affinity_num == aset->num) {
297 break;
298 }
299 }
300 return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;
301 }
302
303 uint64_t
ml_cpu_cache_size(unsigned int level)304 ml_cpu_cache_size(unsigned int level)
305 {
306 x86_cpu_cache_t *cachep;
307
308 if (level == 0) {
309 return machine_info.max_mem;
310 } else if (1 <= level && level <= MAX_CACHE_DEPTH) {
311 cachep = current_cpu_datap()->lcpu.caches[level - 1];
312 return cachep ? cachep->cache_size : 0;
313 } else {
314 return 0;
315 }
316 }
317
318 unsigned int
ml_cpu_cache_sharing(unsigned int level,cluster_type_t cluster_type __unused,bool include_all_cpu_types __unused)319 ml_cpu_cache_sharing(unsigned int level, cluster_type_t cluster_type __unused, bool include_all_cpu_types __unused)
320 {
321 x86_cpu_cache_t *cachep;
322
323 if (level == 0) {
324 return machine_info.max_cpus;
325 } else if (1 <= level && level <= MAX_CACHE_DEPTH) {
326 cachep = current_cpu_datap()->lcpu.caches[level - 1];
327 return cachep ? cachep->nlcpus : 0;
328 } else {
329 return 0;
330 }
331 }
332
333 #if DEVELOPMENT || DEBUG
334 volatile int mmiotrace_enabled = 1;
335 uint32_t iotrace_entries_per_cpu = 0;
336 uint32_t PERCPU_DATA(iotrace_next);
337 iotrace_entry_t *PERCPU_DATA(iotrace_ring);
338
339 volatile int traptrace_enabled = 1;
340 uint32_t traptrace_entries_per_cpu = 0;
341 uint32_t PERCPU_DATA(traptrace_next);
342 traptrace_entry_t *PERCPU_DATA(traptrace_ring);
343
344 static void
init_iotrace_bufs(int entries_per_cpu)345 init_iotrace_bufs(int entries_per_cpu)
346 {
347 size_t size = entries_per_cpu * sizeof(iotrace_entry_t);
348
349 percpu_foreach(ring, iotrace_ring) {
350 *ring = zalloc_permanent_tag(size, 63, VM_KERN_MEMORY_DIAG);
351 };
352
353 iotrace_entries_per_cpu = entries_per_cpu;
354 }
355
356 static void
init_traptrace_bufs(int entries_per_cpu)357 init_traptrace_bufs(int entries_per_cpu)
358 {
359 size_t size = entries_per_cpu * sizeof(traptrace_entry_t);
360
361 percpu_foreach(ring, traptrace_ring) {
362 *ring = zalloc_permanent_tag(size, 63, VM_KERN_MEMORY_DIAG);
363 };
364
365 traptrace_entries_per_cpu = entries_per_cpu;
366 }
367
368 static void
gentrace_configure_from_bootargs(const char * ena_prop,int * ena_valp,const char * epc_prop,int * epcp,int max_epc,int def_epc,int override)369 gentrace_configure_from_bootargs(const char *ena_prop, int *ena_valp, const char *epc_prop,
370 int *epcp, int max_epc, int def_epc, int override)
371 {
372 if (kern_feature_override(override)) {
373 *ena_valp = 0;
374 }
375
376 (void) PE_parse_boot_argn(ena_prop, ena_valp, sizeof(*ena_valp));
377
378 if (*ena_valp == 0) {
379 return;
380 }
381
382 if (PE_parse_boot_argn(epc_prop, epcp, sizeof(*epcp)) &&
383 (*epcp < 1 || *epcp > max_epc)) {
384 *epcp = def_epc;
385 }
386 }
387
388 void
iotrace_init(void)389 iotrace_init(void)
390 {
391 int entries_per_cpu = DEFAULT_IOTRACE_ENTRIES_PER_CPU;
392 int enable = mmiotrace_enabled;
393
394 gentrace_configure_from_bootargs("iotrace", &enable, "iotrace_epc", &entries_per_cpu,
395 IOTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_IOTRACE_ENTRIES_PER_CPU, KF_IOTRACE_OVRD);
396
397 mmiotrace_enabled = enable;
398
399 if (mmiotrace_enabled) {
400 init_iotrace_bufs(entries_per_cpu);
401 }
402 }
403
404 void
traptrace_init(void)405 traptrace_init(void)
406 {
407 int entries_per_cpu = DEFAULT_TRAPTRACE_ENTRIES_PER_CPU;
408 int enable = traptrace_enabled;
409
410 gentrace_configure_from_bootargs("traptrace", &enable, "traptrace_epc", &entries_per_cpu,
411 TRAPTRACE_MAX_ENTRIES_PER_CPU, DEFAULT_TRAPTRACE_ENTRIES_PER_CPU, KF_TRAPTRACE_OVRD);
412
413 traptrace_enabled = enable;
414
415 if (traptrace_enabled) {
416 init_traptrace_bufs(entries_per_cpu);
417 }
418 }
419
420 #endif /* DEVELOPMENT || DEBUG */
421