1 /*
2 * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <vm/vm_kern.h>
29 #include <kern/zalloc.h>
30 #include <kern/lock_group.h>
31 #include <kern/timer_queue.h>
32 #include <kern/monotonic.h>
33 #include <mach/machine.h>
34 #include <i386/cpu_threads.h>
35 #include <i386/cpuid.h>
36 #include <i386/machine_cpu.h>
37 #include <i386/pmCPU.h>
38 #include <i386/bit_routines.h>
39
40 #define DIVISOR_GUARD(denom) \
41 if ((denom) == 0) { \
42 kprintf("%s: %d Zero divisor: " #denom, \
43 __FILE__, __LINE__); \
44 }
45
46 static void debug_topology_print(void);
47
48 boolean_t topo_dbg = FALSE;
49
50 x86_pkg_t *x86_pkgs = NULL;
51 uint32_t num_Lx_caches[MAX_CACHE_DEPTH] = { 0 };
52
53 static x86_pkg_t *free_pkgs = NULL;
54 static x86_die_t *free_dies = NULL;
55 static x86_core_t *free_cores = NULL;
56 static uint32_t num_dies = 0;
57
58 static x86_cpu_cache_t *x86_caches = NULL;
59 static uint32_t num_caches = 0;
60
61 static boolean_t topoParmsInited = FALSE;
62 x86_topology_parameters_t topoParms;
63
64 decl_simple_lock_data(, x86_topo_lock);
65
66 static struct cpu_cache {
67 int level; int type;
68 } cpu_caches[LCACHE_MAX] = {
69 [L1D] = { 1, CPU_CACHE_TYPE_DATA },
70 [L1I] = { 1, CPU_CACHE_TYPE_INST },
71 [L2U] = { 2, CPU_CACHE_TYPE_UNIF },
72 [L3U] = { 3, CPU_CACHE_TYPE_UNIF },
73 };
74
75 static boolean_t
cpu_is_hyperthreaded(void)76 cpu_is_hyperthreaded(void)
77 {
78 i386_cpu_info_t *cpuinfo;
79
80 cpuinfo = cpuid_info();
81 return cpuinfo->thread_count > cpuinfo->core_count;
82 }
83
84 static x86_cpu_cache_t *
x86_cache_alloc(void)85 x86_cache_alloc(void)
86 {
87 x86_cpu_cache_t *cache;
88 int i;
89
90 if (x86_caches == NULL) {
91 cache = zalloc_permanent_tag(sizeof(x86_cpu_cache_t) +
92 (MAX_CPUS * sizeof(x86_lcpu_t *)),
93 ZALIGN(x86_cpu_cache_t), VM_KERN_MEMORY_CPU);
94 if (cache == NULL) {
95 return NULL;
96 }
97 } else {
98 cache = x86_caches;
99 x86_caches = cache->next;
100 cache->next = NULL;
101 }
102
103 cache->next = NULL;
104 cache->maxcpus = MAX_CPUS;
105 for (i = 0; i < cache->maxcpus; i += 1) {
106 cache->cpus[i] = NULL;
107 }
108
109 num_caches += 1;
110
111 return cache;
112 }
113
114 static void
x86_LLC_info(void)115 x86_LLC_info(void)
116 {
117 int cache_level = 0;
118 uint32_t nCPUsSharing = 1;
119 i386_cpu_info_t *cpuinfo;
120 struct cpu_cache *cachep;
121 int i;
122
123 cpuinfo = cpuid_info();
124
125 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
126 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) {
127 continue;
128 }
129
130 /*
131 * Only worry about it if it's a deeper level than
132 * what we've seen before.
133 */
134 if (cachep->level > cache_level) {
135 cache_level = cachep->level;
136
137 /*
138 * Save the number of CPUs sharing this cache.
139 */
140 nCPUsSharing = cpuinfo->cache_sharing[i];
141 }
142 }
143
144 /*
145 * Make the level of the LLC be 0 based.
146 */
147 topoParms.LLCDepth = cache_level - 1;
148
149 /*
150 * nCPUsSharing represents the *maximum* number of cores or
151 * logical CPUs sharing the cache.
152 */
153 topoParms.maxSharingLLC = nCPUsSharing;
154
155 topoParms.nCoresSharingLLC = nCPUsSharing / (cpuinfo->thread_count /
156 cpuinfo->core_count);
157 topoParms.nLCPUsSharingLLC = nCPUsSharing;
158
159 /*
160 * nCPUsSharing may not be the number of *active* cores or
161 * threads that are sharing the cache.
162 */
163 if (nCPUsSharing > cpuinfo->core_count) {
164 topoParms.nCoresSharingLLC = cpuinfo->core_count;
165 }
166 if (nCPUsSharing > cpuinfo->thread_count) {
167 topoParms.nLCPUsSharingLLC = cpuinfo->thread_count;
168 }
169 }
170
171 static void
initTopoParms(void)172 initTopoParms(void)
173 {
174 i386_cpu_info_t *cpuinfo;
175
176 topoParms.stable = FALSE;
177
178 cpuinfo = cpuid_info();
179
180 PE_parse_boot_argn("-topo", &topo_dbg, sizeof(topo_dbg));
181
182 /*
183 * We need to start with getting the LLC information correct.
184 */
185 x86_LLC_info();
186
187 /*
188 * Compute the number of threads (logical CPUs) per core.
189 */
190 DIVISOR_GUARD(cpuinfo->core_count);
191 topoParms.nLThreadsPerCore = cpuinfo->thread_count / cpuinfo->core_count;
192 DIVISOR_GUARD(cpuinfo->cpuid_cores_per_package);
193 topoParms.nPThreadsPerCore = cpuinfo->cpuid_logical_per_package / cpuinfo->cpuid_cores_per_package;
194
195 /*
196 * Compute the number of dies per package.
197 */
198 DIVISOR_GUARD(topoParms.nCoresSharingLLC);
199 topoParms.nLDiesPerPackage = cpuinfo->core_count / topoParms.nCoresSharingLLC;
200 DIVISOR_GUARD(topoParms.nPThreadsPerCore);
201 DIVISOR_GUARD(topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
202 topoParms.nPDiesPerPackage = cpuinfo->cpuid_cores_per_package / (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
203
204
205 /*
206 * Compute the number of cores per die.
207 */
208 topoParms.nLCoresPerDie = topoParms.nCoresSharingLLC;
209 topoParms.nPCoresPerDie = (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
210
211 /*
212 * Compute the number of threads per die.
213 */
214 topoParms.nLThreadsPerDie = topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie;
215 topoParms.nPThreadsPerDie = topoParms.nPThreadsPerCore * topoParms.nPCoresPerDie;
216
217 /*
218 * Compute the number of cores per package.
219 */
220 topoParms.nLCoresPerPackage = topoParms.nLCoresPerDie * topoParms.nLDiesPerPackage;
221 topoParms.nPCoresPerPackage = topoParms.nPCoresPerDie * topoParms.nPDiesPerPackage;
222
223 /*
224 * Compute the number of threads per package.
225 */
226 topoParms.nLThreadsPerPackage = topoParms.nLThreadsPerCore * topoParms.nLCoresPerPackage;
227 topoParms.nPThreadsPerPackage = topoParms.nPThreadsPerCore * topoParms.nPCoresPerPackage;
228
229 TOPO_DBG("\nCache Topology Parameters:\n");
230 TOPO_DBG("\tLLC Depth: %d\n", topoParms.LLCDepth);
231 TOPO_DBG("\tCores Sharing LLC: %d\n", topoParms.nCoresSharingLLC);
232 TOPO_DBG("\tThreads Sharing LLC: %d\n", topoParms.nLCPUsSharingLLC);
233 TOPO_DBG("\tmax Sharing of LLC: %d\n", topoParms.maxSharingLLC);
234
235 TOPO_DBG("\nLogical Topology Parameters:\n");
236 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore);
237 TOPO_DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie);
238 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie);
239 TOPO_DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage);
240 TOPO_DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage);
241 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage);
242
243 TOPO_DBG("\nPhysical Topology Parameters:\n");
244 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore);
245 TOPO_DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie);
246 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie);
247 TOPO_DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage);
248 TOPO_DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage);
249 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage);
250
251 topoParmsInited = TRUE;
252 }
253
254 static void
x86_cache_free(x86_cpu_cache_t * cache)255 x86_cache_free(x86_cpu_cache_t *cache)
256 {
257 num_caches -= 1;
258 if (cache->level > 0 && cache->level <= MAX_CACHE_DEPTH) {
259 num_Lx_caches[cache->level - 1] -= 1;
260 }
261 cache->next = x86_caches;
262 x86_caches = cache;
263 }
264
265 /*
266 * This returns a list of cache structures that represent the
267 * caches for a CPU. Some of the structures may have to be
268 * "freed" if they are actually shared between CPUs.
269 */
270 static x86_cpu_cache_t *
x86_cache_list(void)271 x86_cache_list(void)
272 {
273 x86_cpu_cache_t *root = NULL;
274 x86_cpu_cache_t *cur = NULL;
275 x86_cpu_cache_t *last = NULL;
276 struct cpu_cache *cachep;
277 int i;
278
279 /*
280 * Cons up a list driven not by CPUID leaf 4 (deterministic cache params)
281 * but by the table above plus parameters already cracked from cpuid...
282 */
283 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
284 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) {
285 continue;
286 }
287
288 cur = x86_cache_alloc();
289 if (cur == NULL) {
290 break;
291 }
292
293 cur->type = cachep->type;
294 cur->level = cachep->level;
295 cur->nlcpus = 0;
296 cur->maxcpus = cpuid_info()->cache_sharing[i];
297 cur->partitions = cpuid_info()->cache_partitions[i];
298 cur->cache_size = cpuid_info()->cache_size[i];
299 cur->line_size = cpuid_info()->cache_linesize;
300
301 if (last == NULL) {
302 root = cur;
303 last = cur;
304 } else {
305 last->next = cur;
306 last = cur;
307 }
308 num_Lx_caches[cur->level - 1] += 1;
309 }
310 return root;
311 }
312
313
314 static x86_cpu_cache_t *
x86_match_cache(x86_cpu_cache_t * list,x86_cpu_cache_t * matcher)315 x86_match_cache(x86_cpu_cache_t *list, x86_cpu_cache_t *matcher)
316 {
317 x86_cpu_cache_t *cur_cache;
318
319 cur_cache = list;
320 while (cur_cache != NULL) {
321 if (cur_cache->maxcpus == matcher->maxcpus
322 && cur_cache->type == matcher->type
323 && cur_cache->level == matcher->level
324 && cur_cache->partitions == matcher->partitions
325 && cur_cache->line_size == matcher->line_size
326 && cur_cache->cache_size == matcher->cache_size) {
327 break;
328 }
329
330 cur_cache = cur_cache->next;
331 }
332
333 return cur_cache;
334 }
335
336 static void
x86_lcpu_init(int cpu)337 x86_lcpu_init(int cpu)
338 {
339 cpu_data_t *cpup;
340 x86_lcpu_t *lcpu;
341 int i;
342
343 cpup = cpu_datap(cpu);
344
345 lcpu = &cpup->lcpu;
346 lcpu->lcpu = lcpu;
347 lcpu->cpu = cpup;
348 lcpu->next_in_core = NULL;
349 lcpu->next_in_die = NULL;
350 lcpu->next_in_pkg = NULL;
351 lcpu->core = NULL;
352 lcpu->die = NULL;
353 lcpu->package = NULL;
354 lcpu->cpu_num = cpu;
355 lcpu->lnum = cpu;
356 lcpu->pnum = cpup->cpu_phys_number;
357 lcpu->state = LCPU_OFF;
358 for (i = 0; i < MAX_CACHE_DEPTH; i += 1) {
359 lcpu->caches[i] = NULL;
360 }
361 }
362
363 static x86_core_t *
x86_core_alloc(int cpu)364 x86_core_alloc(int cpu)
365 {
366 x86_core_t *core;
367 cpu_data_t *cpup;
368
369 cpup = cpu_datap(cpu);
370
371 mp_safe_spin_lock(&x86_topo_lock);
372 if (free_cores != NULL) {
373 core = free_cores;
374 free_cores = core->next_in_die;
375 core->next_in_die = NULL;
376 simple_unlock(&x86_topo_lock);
377 } else {
378 simple_unlock(&x86_topo_lock);
379 core = zalloc_permanent_type(x86_core_t);
380 if (core == NULL) {
381 panic("x86_core_alloc() alloc of x86_core_t failed!");
382 }
383 }
384
385 core->pcore_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
386 core->lcore_num = core->pcore_num % topoParms.nPCoresPerPackage;
387
388 core->flags = X86CORE_FL_PRESENT | X86CORE_FL_READY
389 | X86CORE_FL_HALTED | X86CORE_FL_IDLE;
390
391 return core;
392 }
393
394 static void
x86_core_free(x86_core_t * core)395 x86_core_free(x86_core_t *core)
396 {
397 mp_safe_spin_lock(&x86_topo_lock);
398 core->next_in_die = free_cores;
399 free_cores = core;
400 simple_unlock(&x86_topo_lock);
401 }
402
403 static x86_pkg_t *
x86_package_find(int cpu)404 x86_package_find(int cpu)
405 {
406 x86_pkg_t *pkg;
407 cpu_data_t *cpup;
408 uint32_t pkg_num;
409
410 cpup = cpu_datap(cpu);
411
412 pkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
413
414 pkg = x86_pkgs;
415 while (pkg != NULL) {
416 if (pkg->ppkg_num == pkg_num) {
417 break;
418 }
419 pkg = pkg->next;
420 }
421
422 return pkg;
423 }
424
425 static x86_die_t *
x86_die_find(int cpu)426 x86_die_find(int cpu)
427 {
428 x86_die_t *die;
429 x86_pkg_t *pkg;
430 cpu_data_t *cpup;
431 uint32_t die_num;
432
433 cpup = cpu_datap(cpu);
434
435 die_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
436
437 pkg = x86_package_find(cpu);
438 if (pkg == NULL) {
439 return NULL;
440 }
441
442 die = pkg->dies;
443 while (die != NULL) {
444 if (die->pdie_num == die_num) {
445 break;
446 }
447 die = die->next_in_pkg;
448 }
449
450 return die;
451 }
452
453 static x86_core_t *
x86_core_find(int cpu)454 x86_core_find(int cpu)
455 {
456 x86_core_t *core;
457 x86_die_t *die;
458 cpu_data_t *cpup;
459 uint32_t core_num;
460
461 cpup = cpu_datap(cpu);
462
463 core_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
464
465 die = x86_die_find(cpu);
466 if (die == NULL) {
467 return NULL;
468 }
469
470 core = die->cores;
471 while (core != NULL) {
472 if (core->pcore_num == core_num) {
473 break;
474 }
475 core = core->next_in_die;
476 }
477
478 return core;
479 }
480
481 void
x86_set_logical_topology(x86_lcpu_t * lcpu,int pnum,int lnum)482 x86_set_logical_topology(x86_lcpu_t *lcpu, int pnum, int lnum)
483 {
484 x86_core_t *core = lcpu->core;
485 x86_die_t *die = lcpu->die;
486 x86_pkg_t *pkg = lcpu->package;
487
488 assert(core != NULL);
489 assert(die != NULL);
490 assert(pkg != NULL);
491
492 lcpu->cpu_num = lnum;
493 lcpu->pnum = pnum;
494 lcpu->master = (lnum == master_cpu);
495 lcpu->primary = (lnum % topoParms.nLThreadsPerPackage) == 0;
496
497 lcpu->lnum = lnum % topoParms.nLThreadsPerCore;
498
499 core->pcore_num = lnum / topoParms.nLThreadsPerCore;
500 core->lcore_num = core->pcore_num % topoParms.nLCoresPerDie;
501
502 die->pdie_num = lnum / (topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie);
503 die->ldie_num = die->pdie_num % topoParms.nLDiesPerPackage;
504
505 pkg->ppkg_num = lnum / topoParms.nLThreadsPerPackage;
506 pkg->lpkg_num = pkg->ppkg_num;
507 }
508
509 static x86_die_t *
x86_die_alloc(int cpu)510 x86_die_alloc(int cpu)
511 {
512 x86_die_t *die;
513 cpu_data_t *cpup;
514
515 cpup = cpu_datap(cpu);
516
517 mp_safe_spin_lock(&x86_topo_lock);
518 if (free_dies != NULL) {
519 die = free_dies;
520 free_dies = die->next_in_pkg;
521 die->next_in_pkg = NULL;
522 simple_unlock(&x86_topo_lock);
523 } else {
524 simple_unlock(&x86_topo_lock);
525 die = zalloc_permanent_type(x86_die_t);
526 if (die == NULL) {
527 panic("x86_die_alloc() alloc of x86_die_t failed!");
528 }
529 }
530
531 die->pdie_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
532
533 die->ldie_num = num_dies;
534 atomic_incl((long *) &num_dies, 1);
535
536 die->flags = X86DIE_FL_PRESENT;
537 return die;
538 }
539
540 static void
x86_die_free(x86_die_t * die)541 x86_die_free(x86_die_t *die)
542 {
543 mp_safe_spin_lock(&x86_topo_lock);
544 die->next_in_pkg = free_dies;
545 free_dies = die;
546 atomic_decl((long *) &num_dies, 1);
547 simple_unlock(&x86_topo_lock);
548 }
549
550 static x86_pkg_t *
x86_package_alloc(int cpu)551 x86_package_alloc(int cpu)
552 {
553 x86_pkg_t *pkg;
554 cpu_data_t *cpup;
555
556 cpup = cpu_datap(cpu);
557
558 mp_safe_spin_lock(&x86_topo_lock);
559 if (free_pkgs != NULL) {
560 pkg = free_pkgs;
561 free_pkgs = pkg->next;
562 pkg->next = NULL;
563 simple_unlock(&x86_topo_lock);
564 } else {
565 simple_unlock(&x86_topo_lock);
566 pkg = zalloc_permanent_type(x86_pkg_t);
567 if (pkg == NULL) {
568 panic("x86_package_alloc() alloc of x86_pkg_t failed!");
569 }
570 }
571
572 pkg->ppkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
573
574 pkg->lpkg_num = topoParms.nPackages;
575 atomic_incl((long *) &topoParms.nPackages, 1);
576
577 pkg->flags = X86PKG_FL_PRESENT | X86PKG_FL_READY;
578 return pkg;
579 }
580
581 static void
x86_package_free(x86_pkg_t * pkg)582 x86_package_free(x86_pkg_t *pkg)
583 {
584 mp_safe_spin_lock(&x86_topo_lock);
585 pkg->next = free_pkgs;
586 free_pkgs = pkg;
587 atomic_decl((long *) &topoParms.nPackages, 1);
588 simple_unlock(&x86_topo_lock);
589 }
590
591 static void
x86_cache_add_lcpu(x86_cpu_cache_t * cache,x86_lcpu_t * lcpu)592 x86_cache_add_lcpu(x86_cpu_cache_t *cache, x86_lcpu_t *lcpu)
593 {
594 x86_cpu_cache_t *cur_cache;
595 int i;
596
597 /*
598 * Put the new CPU into the list of the cache.
599 */
600 cur_cache = lcpu->caches[cache->level - 1];
601 lcpu->caches[cache->level - 1] = cache;
602 cache->next = cur_cache;
603 cache->nlcpus += 1;
604 for (i = 0; i < cache->nlcpus; i += 1) {
605 if (cache->cpus[i] == NULL) {
606 cache->cpus[i] = lcpu;
607 break;
608 }
609 }
610 }
611
612 static void
x86_lcpu_add_caches(x86_lcpu_t * lcpu)613 x86_lcpu_add_caches(x86_lcpu_t *lcpu)
614 {
615 x86_cpu_cache_t *list;
616 x86_cpu_cache_t *cur;
617 x86_cpu_cache_t *match;
618 x86_die_t *die;
619 x86_core_t *core;
620 x86_lcpu_t *cur_lcpu;
621 uint32_t level;
622 boolean_t found = FALSE;
623
624 assert(lcpu != NULL);
625
626 /*
627 * Add the cache data to the topology.
628 */
629 list = x86_cache_list();
630
631 mp_safe_spin_lock(&x86_topo_lock);
632
633 while (list != NULL) {
634 /*
635 * Remove the cache from the front of the list.
636 */
637 cur = list;
638 list = cur->next;
639 cur->next = NULL;
640 level = cur->level - 1;
641
642 /*
643 * If the cache isn't shared then just put it where it
644 * belongs.
645 */
646 if (cur->maxcpus == 1) {
647 x86_cache_add_lcpu(cur, lcpu);
648 continue;
649 }
650
651 /*
652 * We'll assume that all of the caches at a particular level
653 * have the same sharing. So if we have a cache already at
654 * this level, we'll just skip looking for the match.
655 */
656 if (lcpu->caches[level] != NULL) {
657 x86_cache_free(cur);
658 continue;
659 }
660
661 /*
662 * This is a shared cache, so we have to figure out if
663 * this is the first time we've seen this cache. We do
664 * this by searching through the topology and seeing if
665 * this cache is already described.
666 *
667 * Assume that L{LLC-1} are all at the core level and that
668 * LLC is shared at the die level.
669 */
670 if (level < topoParms.LLCDepth) {
671 /*
672 * Shared at the core.
673 */
674 core = lcpu->core;
675 cur_lcpu = core->lcpus;
676 while (cur_lcpu != NULL) {
677 /*
678 * Skip ourselves.
679 */
680 if (cur_lcpu == lcpu) {
681 cur_lcpu = cur_lcpu->next_in_core;
682 continue;
683 }
684
685 /*
686 * If there's a cache on this logical CPU,
687 * then use that one.
688 */
689 match = x86_match_cache(cur_lcpu->caches[level], cur);
690 if (match != NULL) {
691 x86_cache_free(cur);
692 x86_cache_add_lcpu(match, lcpu);
693 found = TRUE;
694 break;
695 }
696
697 cur_lcpu = cur_lcpu->next_in_core;
698 }
699 } else {
700 /*
701 * Shared at the die.
702 */
703 die = lcpu->die;
704 cur_lcpu = die->lcpus;
705 while (cur_lcpu != NULL) {
706 /*
707 * Skip ourselves.
708 */
709 if (cur_lcpu == lcpu) {
710 cur_lcpu = cur_lcpu->next_in_die;
711 continue;
712 }
713
714 /*
715 * If there's a cache on this logical CPU,
716 * then use that one.
717 */
718 match = x86_match_cache(cur_lcpu->caches[level], cur);
719 if (match != NULL) {
720 x86_cache_free(cur);
721 x86_cache_add_lcpu(match, lcpu);
722 found = TRUE;
723 break;
724 }
725
726 cur_lcpu = cur_lcpu->next_in_die;
727 }
728 }
729
730 /*
731 * If a shared cache wasn't found, then this logical CPU must
732 * be the first one encountered.
733 */
734 if (!found) {
735 x86_cache_add_lcpu(cur, lcpu);
736 }
737 }
738
739 simple_unlock(&x86_topo_lock);
740 }
741
742 static void
x86_core_add_lcpu(x86_core_t * core,x86_lcpu_t * lcpu)743 x86_core_add_lcpu(x86_core_t *core, x86_lcpu_t *lcpu)
744 {
745 assert(core != NULL);
746 assert(lcpu != NULL);
747
748 mp_safe_spin_lock(&x86_topo_lock);
749
750 lcpu->next_in_core = core->lcpus;
751 lcpu->core = core;
752 core->lcpus = lcpu;
753 core->num_lcpus += 1;
754 simple_unlock(&x86_topo_lock);
755 }
756
757 static void
x86_die_add_lcpu(x86_die_t * die,x86_lcpu_t * lcpu)758 x86_die_add_lcpu(x86_die_t *die, x86_lcpu_t *lcpu)
759 {
760 assert(die != NULL);
761 assert(lcpu != NULL);
762
763 lcpu->next_in_die = die->lcpus;
764 lcpu->die = die;
765 die->lcpus = lcpu;
766 }
767
768 static void
x86_die_add_core(x86_die_t * die,x86_core_t * core)769 x86_die_add_core(x86_die_t *die, x86_core_t *core)
770 {
771 assert(die != NULL);
772 assert(core != NULL);
773
774 core->next_in_die = die->cores;
775 core->die = die;
776 die->cores = core;
777 die->num_cores += 1;
778 }
779
780 static void
x86_package_add_lcpu(x86_pkg_t * pkg,x86_lcpu_t * lcpu)781 x86_package_add_lcpu(x86_pkg_t *pkg, x86_lcpu_t *lcpu)
782 {
783 assert(pkg != NULL);
784 assert(lcpu != NULL);
785
786 lcpu->next_in_pkg = pkg->lcpus;
787 lcpu->package = pkg;
788 pkg->lcpus = lcpu;
789 }
790
791 static void
x86_package_add_core(x86_pkg_t * pkg,x86_core_t * core)792 x86_package_add_core(x86_pkg_t *pkg, x86_core_t *core)
793 {
794 assert(pkg != NULL);
795 assert(core != NULL);
796
797 core->next_in_pkg = pkg->cores;
798 core->package = pkg;
799 pkg->cores = core;
800 }
801
802 static void
x86_package_add_die(x86_pkg_t * pkg,x86_die_t * die)803 x86_package_add_die(x86_pkg_t *pkg, x86_die_t *die)
804 {
805 assert(pkg != NULL);
806 assert(die != NULL);
807
808 die->next_in_pkg = pkg->dies;
809 die->package = pkg;
810 pkg->dies = die;
811 pkg->num_dies += 1;
812 }
813
814 void *
cpu_thread_alloc(int cpu)815 cpu_thread_alloc(int cpu)
816 {
817 x86_core_t *core = NULL;
818 x86_die_t *die = NULL;
819 x86_pkg_t *pkg = NULL;
820 cpu_data_t *cpup;
821 uint32_t phys_cpu;
822
823 /*
824 * Only allow one to manipulate the topology at a time.
825 */
826 mp_safe_spin_lock(&x86_topo_lock);
827
828 /*
829 * Make sure all of the topology parameters have been initialized.
830 */
831 if (!topoParmsInited) {
832 initTopoParms();
833 }
834
835 cpup = cpu_datap(cpu);
836
837 phys_cpu = cpup->cpu_phys_number;
838
839 x86_lcpu_init(cpu);
840
841 /*
842 * Assume that all cpus have the same features.
843 */
844 if (cpu_is_hyperthreaded()) {
845 cpup->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
846 } else {
847 cpup->cpu_threadtype = CPU_THREADTYPE_NONE;
848 }
849
850 /*
851 * Get the package that the logical CPU is in.
852 */
853 do {
854 pkg = x86_package_find(cpu);
855 if (pkg == NULL) {
856 /*
857 * Package structure hasn't been created yet, do it now.
858 */
859 simple_unlock(&x86_topo_lock);
860 pkg = x86_package_alloc(cpu);
861 mp_safe_spin_lock(&x86_topo_lock);
862 if (x86_package_find(cpu) != NULL) {
863 x86_package_free(pkg);
864 continue;
865 }
866
867 /*
868 * Add the new package to the global list of packages.
869 */
870 pkg->next = x86_pkgs;
871 x86_pkgs = pkg;
872 }
873 } while (pkg == NULL);
874
875 /*
876 * Get the die that the logical CPU is in.
877 */
878 do {
879 die = x86_die_find(cpu);
880 if (die == NULL) {
881 /*
882 * Die structure hasn't been created yet, do it now.
883 */
884 simple_unlock(&x86_topo_lock);
885 die = x86_die_alloc(cpu);
886 mp_safe_spin_lock(&x86_topo_lock);
887 if (x86_die_find(cpu) != NULL) {
888 x86_die_free(die);
889 continue;
890 }
891
892 /*
893 * Add the die to the package.
894 */
895 x86_package_add_die(pkg, die);
896 }
897 } while (die == NULL);
898
899 /*
900 * Get the core for this logical CPU.
901 */
902 do {
903 core = x86_core_find(cpu);
904 if (core == NULL) {
905 /*
906 * Allocate the core structure now.
907 */
908 simple_unlock(&x86_topo_lock);
909 core = x86_core_alloc(cpu);
910 mp_safe_spin_lock(&x86_topo_lock);
911 if (x86_core_find(cpu) != NULL) {
912 x86_core_free(core);
913 continue;
914 }
915
916 /*
917 * Add the core to the die & package.
918 */
919 x86_die_add_core(die, core);
920 x86_package_add_core(pkg, core);
921 machine_info.physical_cpu_max += 1;
922 }
923 } while (core == NULL);
924
925
926 /*
927 * Done manipulating the topology, so others can get in.
928 */
929 machine_info.logical_cpu_max += 1;
930 simple_unlock(&x86_topo_lock);
931
932 /*
933 * Add the logical CPU to the other topology structures.
934 */
935 x86_core_add_lcpu(core, &cpup->lcpu);
936 x86_die_add_lcpu(core->die, &cpup->lcpu);
937 x86_package_add_lcpu(core->package, &cpup->lcpu);
938 x86_lcpu_add_caches(&cpup->lcpu);
939
940 return (void *) core;
941 }
942
943 void
cpu_thread_init(void)944 cpu_thread_init(void)
945 {
946 int my_cpu = get_cpu_number();
947 cpu_data_t *cpup = current_cpu_datap();
948 x86_core_t *core;
949 static int initialized = 0;
950
951 /*
952 * If we're the boot processor, we do all of the initialization of
953 * the CPU topology infrastructure.
954 */
955 if (my_cpu == master_cpu && !initialized) {
956 simple_lock_init(&x86_topo_lock, 0);
957
958 /*
959 * Put this logical CPU into the physical CPU topology.
960 */
961 cpup->lcpu.core = cpu_thread_alloc(my_cpu);
962
963 initialized = 1;
964 }
965
966 /*
967 * Do the CPU accounting.
968 */
969 core = cpup->lcpu.core;
970 mp_safe_spin_lock(&x86_topo_lock);
971 machine_info.logical_cpu += 1;
972 if (core->active_lcpus == 0) {
973 machine_info.physical_cpu += 1;
974 }
975 core->active_lcpus += 1;
976 simple_unlock(&x86_topo_lock);
977
978 pmCPUMarkRunning(cpup);
979 timer_resync_deadlines();
980 }
981
982 /*
983 * Called for a cpu to halt permanently
984 * (as opposed to halting and expecting an interrupt to awaken it).
985 */
986 __attribute__((noreturn))
987 void
cpu_thread_halt(void)988 cpu_thread_halt(void)
989 {
990 x86_core_t *core;
991 cpu_data_t *cpup = current_cpu_datap();
992
993 mp_safe_spin_lock(&x86_topo_lock);
994 machine_info.logical_cpu -= 1;
995 core = cpup->lcpu.core;
996 core->active_lcpus -= 1;
997 if (core->active_lcpus == 0) {
998 machine_info.physical_cpu -= 1;
999 }
1000 simple_unlock(&x86_topo_lock);
1001
1002 /*
1003 * Let the power management code determine the best way to "stop"
1004 * the processor.
1005 */
1006 ml_set_interrupts_enabled(FALSE);
1007 while (1) {
1008 pmCPUHalt(PM_HALT_NORMAL);
1009 }
1010 /* NOT REACHED */
1011 }
1012
1013 /*
1014 * Validates that the topology was built correctly. Must be called only
1015 * after the complete topology is built and no other changes are being made.
1016 */
1017 void
x86_validate_topology(void)1018 x86_validate_topology(void)
1019 {
1020 x86_pkg_t *pkg;
1021 x86_die_t *die;
1022 x86_core_t *core;
1023 x86_lcpu_t *lcpu;
1024 uint32_t nDies;
1025 uint32_t nCores;
1026 uint32_t nCPUs;
1027
1028 if (topo_dbg) {
1029 debug_topology_print();
1030 }
1031
1032 /*
1033 * Called after processors are registered but before non-boot processors
1034 * are started:
1035 * - real_ncpus: number of registered processors driven from MADT
1036 * - max_ncpus: max number of processors that will be started
1037 */
1038 nCPUs = topoParms.nPackages * topoParms.nLThreadsPerPackage;
1039 if (nCPUs != real_ncpus) {
1040 panic("x86_validate_topology() %d threads but %d registered from MADT",
1041 nCPUs, real_ncpus);
1042 }
1043
1044 pkg = x86_pkgs;
1045 while (pkg != NULL) {
1046 /*
1047 * Make sure that the package has the correct number of dies.
1048 */
1049 nDies = 0;
1050 die = pkg->dies;
1051 while (die != NULL) {
1052 if (die->package == NULL) {
1053 panic("Die(%d)->package is NULL",
1054 die->pdie_num);
1055 }
1056 if (die->package != pkg) {
1057 panic("Die %d points to package %d, should be %d",
1058 die->pdie_num, die->package->lpkg_num, pkg->lpkg_num);
1059 }
1060
1061 TOPO_DBG("Die(%d)->package %d\n",
1062 die->pdie_num, pkg->lpkg_num);
1063
1064 /*
1065 * Make sure that the die has the correct number of cores.
1066 */
1067 TOPO_DBG("Die(%d)->cores: ", die->pdie_num);
1068 nCores = 0;
1069 core = die->cores;
1070 while (core != NULL) {
1071 if (core->die == NULL) {
1072 panic("Core(%d)->die is NULL",
1073 core->pcore_num);
1074 }
1075 if (core->die != die) {
1076 panic("Core %d points to die %d, should be %d",
1077 core->pcore_num, core->die->pdie_num, die->pdie_num);
1078 }
1079 nCores += 1;
1080 TOPO_DBG("%d ", core->pcore_num);
1081 core = core->next_in_die;
1082 }
1083 TOPO_DBG("\n");
1084
1085 if (nCores != topoParms.nLCoresPerDie) {
1086 panic("Should have %d Cores, but only found %d for Die %d",
1087 topoParms.nLCoresPerDie, nCores, die->pdie_num);
1088 }
1089
1090 /*
1091 * Make sure that the die has the correct number of CPUs.
1092 */
1093 TOPO_DBG("Die(%d)->lcpus: ", die->pdie_num);
1094 nCPUs = 0;
1095 lcpu = die->lcpus;
1096 while (lcpu != NULL) {
1097 if (lcpu->die == NULL) {
1098 panic("CPU(%d)->die is NULL",
1099 lcpu->cpu_num);
1100 }
1101 if (lcpu->die != die) {
1102 panic("CPU %d points to die %d, should be %d",
1103 lcpu->cpu_num, lcpu->die->pdie_num, die->pdie_num);
1104 }
1105 nCPUs += 1;
1106 TOPO_DBG("%d ", lcpu->cpu_num);
1107 lcpu = lcpu->next_in_die;
1108 }
1109 TOPO_DBG("\n");
1110
1111 if (nCPUs != topoParms.nLThreadsPerDie) {
1112 panic("Should have %d Threads, but only found %d for Die %d",
1113 topoParms.nLThreadsPerDie, nCPUs, die->pdie_num);
1114 }
1115
1116 nDies += 1;
1117 die = die->next_in_pkg;
1118 }
1119
1120 if (nDies != topoParms.nLDiesPerPackage) {
1121 panic("Should have %d Dies, but only found %d for package %d",
1122 topoParms.nLDiesPerPackage, nDies, pkg->lpkg_num);
1123 }
1124
1125 /*
1126 * Make sure that the package has the correct number of cores.
1127 */
1128 nCores = 0;
1129 core = pkg->cores;
1130 while (core != NULL) {
1131 if (core->package == NULL) {
1132 panic("Core(%d)->package is NULL",
1133 core->pcore_num);
1134 }
1135 if (core->package != pkg) {
1136 panic("Core %d points to package %d, should be %d",
1137 core->pcore_num, core->package->lpkg_num, pkg->lpkg_num);
1138 }
1139 TOPO_DBG("Core(%d)->package %d\n",
1140 core->pcore_num, pkg->lpkg_num);
1141
1142 /*
1143 * Make sure that the core has the correct number of CPUs.
1144 */
1145 nCPUs = 0;
1146 lcpu = core->lcpus;
1147 TOPO_DBG("Core(%d)->lcpus: ", core->pcore_num);
1148 while (lcpu != NULL) {
1149 if (lcpu->core == NULL) {
1150 panic("CPU(%d)->core is NULL",
1151 lcpu->cpu_num);
1152 }
1153 if (lcpu->core != core) {
1154 panic("CPU %d points to core %d, should be %d",
1155 lcpu->cpu_num, lcpu->core->pcore_num, core->pcore_num);
1156 }
1157 TOPO_DBG("%d ", lcpu->cpu_num);
1158 nCPUs += 1;
1159 lcpu = lcpu->next_in_core;
1160 }
1161 TOPO_DBG("\n");
1162
1163 if (nCPUs != topoParms.nLThreadsPerCore) {
1164 panic("Should have %d Threads, but only found %d for Core %d",
1165 topoParms.nLThreadsPerCore, nCPUs, core->pcore_num);
1166 }
1167 nCores += 1;
1168 core = core->next_in_pkg;
1169 }
1170
1171 if (nCores != topoParms.nLCoresPerPackage) {
1172 panic("Should have %d Cores, but only found %d for package %d",
1173 topoParms.nLCoresPerPackage, nCores, pkg->lpkg_num);
1174 }
1175
1176 /*
1177 * Make sure that the package has the correct number of CPUs.
1178 */
1179 nCPUs = 0;
1180 lcpu = pkg->lcpus;
1181 while (lcpu != NULL) {
1182 if (lcpu->package == NULL) {
1183 panic("CPU(%d)->package is NULL",
1184 lcpu->cpu_num);
1185 }
1186 if (lcpu->package != pkg) {
1187 panic("CPU %d points to package %d, should be %d",
1188 lcpu->cpu_num, lcpu->package->lpkg_num, pkg->lpkg_num);
1189 }
1190 TOPO_DBG("CPU(%d)->package %d\n",
1191 lcpu->cpu_num, pkg->lpkg_num);
1192 nCPUs += 1;
1193 lcpu = lcpu->next_in_pkg;
1194 }
1195
1196 if (nCPUs != topoParms.nLThreadsPerPackage) {
1197 panic("Should have %d Threads, but only found %d for package %d",
1198 topoParms.nLThreadsPerPackage, nCPUs, pkg->lpkg_num);
1199 }
1200
1201 pkg = pkg->next;
1202 }
1203 }
1204
1205 /*
1206 * Prints out the topology
1207 */
1208 static void
debug_topology_print(void)1209 debug_topology_print(void)
1210 {
1211 x86_pkg_t *pkg;
1212 x86_die_t *die;
1213 x86_core_t *core;
1214 x86_lcpu_t *cpu;
1215
1216 pkg = x86_pkgs;
1217 while (pkg != NULL) {
1218 kprintf("Package:\n");
1219 kprintf(" Physical: %d\n", pkg->ppkg_num);
1220 kprintf(" Logical: %d\n", pkg->lpkg_num);
1221
1222 die = pkg->dies;
1223 while (die != NULL) {
1224 kprintf(" Die:\n");
1225 kprintf(" Physical: %d\n", die->pdie_num);
1226 kprintf(" Logical: %d\n", die->ldie_num);
1227
1228 core = die->cores;
1229 while (core != NULL) {
1230 kprintf(" Core:\n");
1231 kprintf(" Physical: %d\n", core->pcore_num);
1232 kprintf(" Logical: %d\n", core->lcore_num);
1233
1234 cpu = core->lcpus;
1235 while (cpu != NULL) {
1236 kprintf(" LCPU:\n");
1237 kprintf(" CPU #: %d\n", cpu->cpu_num);
1238 kprintf(" Physical: %d\n", cpu->pnum);
1239 kprintf(" Logical: %d\n", cpu->lnum);
1240 kprintf(" Flags: ");
1241 if (cpu->master) {
1242 kprintf("MASTER ");
1243 }
1244 if (cpu->primary) {
1245 kprintf("PRIMARY");
1246 }
1247 if (!cpu->master && !cpu->primary) {
1248 kprintf("(NONE)");
1249 }
1250 kprintf("\n");
1251
1252 cpu = cpu->next_in_core;
1253 }
1254
1255 core = core->next_in_die;
1256 }
1257
1258 die = die->next_in_pkg;
1259 }
1260
1261 pkg = pkg->next;
1262 }
1263 }
1264