1 /*
2 * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <vm/vm_kern.h>
29 #include <kern/zalloc.h>
30 #include <kern/lock_group.h>
31 #include <kern/timer_queue.h>
32 #include <mach/machine.h>
33 #include <i386/cpu_threads.h>
34 #include <i386/cpuid.h>
35 #include <i386/machine_cpu.h>
36 #include <i386/pmCPU.h>
37 #include <i386/bit_routines.h>
38
39 #if MONOTONIC
40 #include <kern/monotonic.h>
41 #endif /* MONOTONIC */
42
43 #define DIVISOR_GUARD(denom) \
44 if ((denom) == 0) { \
45 kprintf("%s: %d Zero divisor: " #denom, \
46 __FILE__, __LINE__); \
47 }
48
49 static void debug_topology_print(void);
50
51 boolean_t topo_dbg = FALSE;
52
53 x86_pkg_t *x86_pkgs = NULL;
54 uint32_t num_Lx_caches[MAX_CACHE_DEPTH] = { 0 };
55
56 static x86_pkg_t *free_pkgs = NULL;
57 static x86_die_t *free_dies = NULL;
58 static x86_core_t *free_cores = NULL;
59 static uint32_t num_dies = 0;
60
61 static x86_cpu_cache_t *x86_caches = NULL;
62 static uint32_t num_caches = 0;
63
64 static boolean_t topoParmsInited = FALSE;
65 x86_topology_parameters_t topoParms;
66
67 decl_simple_lock_data(, x86_topo_lock);
68
69 static struct cpu_cache {
70 int level; int type;
71 } cpu_caches[LCACHE_MAX] = {
72 [L1D] = { 1, CPU_CACHE_TYPE_DATA },
73 [L1I] = { 1, CPU_CACHE_TYPE_INST },
74 [L2U] = { 2, CPU_CACHE_TYPE_UNIF },
75 [L3U] = { 3, CPU_CACHE_TYPE_UNIF },
76 };
77
78 static boolean_t
cpu_is_hyperthreaded(void)79 cpu_is_hyperthreaded(void)
80 {
81 i386_cpu_info_t *cpuinfo;
82
83 cpuinfo = cpuid_info();
84 return cpuinfo->thread_count > cpuinfo->core_count;
85 }
86
87 static x86_cpu_cache_t *
x86_cache_alloc(void)88 x86_cache_alloc(void)
89 {
90 x86_cpu_cache_t *cache;
91 int i;
92
93 if (x86_caches == NULL) {
94 cache = zalloc_permanent_tag(sizeof(x86_cpu_cache_t) +
95 (MAX_CPUS * sizeof(x86_lcpu_t *)),
96 ZALIGN(x86_cpu_cache_t), VM_KERN_MEMORY_CPU);
97 if (cache == NULL) {
98 return NULL;
99 }
100 } else {
101 cache = x86_caches;
102 x86_caches = cache->next;
103 cache->next = NULL;
104 }
105
106 cache->next = NULL;
107 cache->maxcpus = MAX_CPUS;
108 for (i = 0; i < cache->maxcpus; i += 1) {
109 cache->cpus[i] = NULL;
110 }
111
112 num_caches += 1;
113
114 return cache;
115 }
116
117 static void
x86_LLC_info(void)118 x86_LLC_info(void)
119 {
120 int cache_level = 0;
121 uint32_t nCPUsSharing = 1;
122 i386_cpu_info_t *cpuinfo;
123 struct cpu_cache *cachep;
124 int i;
125
126 cpuinfo = cpuid_info();
127
128 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
129 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) {
130 continue;
131 }
132
133 /*
134 * Only worry about it if it's a deeper level than
135 * what we've seen before.
136 */
137 if (cachep->level > cache_level) {
138 cache_level = cachep->level;
139
140 /*
141 * Save the number of CPUs sharing this cache.
142 */
143 nCPUsSharing = cpuinfo->cache_sharing[i];
144 }
145 }
146
147 /*
148 * Make the level of the LLC be 0 based.
149 */
150 topoParms.LLCDepth = cache_level - 1;
151
152 /*
153 * nCPUsSharing represents the *maximum* number of cores or
154 * logical CPUs sharing the cache.
155 */
156 topoParms.maxSharingLLC = nCPUsSharing;
157
158 topoParms.nCoresSharingLLC = nCPUsSharing / (cpuinfo->thread_count /
159 cpuinfo->core_count);
160 topoParms.nLCPUsSharingLLC = nCPUsSharing;
161
162 /*
163 * nCPUsSharing may not be the number of *active* cores or
164 * threads that are sharing the cache.
165 */
166 if (nCPUsSharing > cpuinfo->core_count) {
167 topoParms.nCoresSharingLLC = cpuinfo->core_count;
168 }
169 if (nCPUsSharing > cpuinfo->thread_count) {
170 topoParms.nLCPUsSharingLLC = cpuinfo->thread_count;
171 }
172 }
173
174 static void
initTopoParms(void)175 initTopoParms(void)
176 {
177 i386_cpu_info_t *cpuinfo;
178
179 topoParms.stable = FALSE;
180
181 cpuinfo = cpuid_info();
182
183 PE_parse_boot_argn("-topo", &topo_dbg, sizeof(topo_dbg));
184
185 /*
186 * We need to start with getting the LLC information correct.
187 */
188 x86_LLC_info();
189
190 /*
191 * Compute the number of threads (logical CPUs) per core.
192 */
193 DIVISOR_GUARD(cpuinfo->core_count);
194 topoParms.nLThreadsPerCore = cpuinfo->thread_count / cpuinfo->core_count;
195 DIVISOR_GUARD(cpuinfo->cpuid_cores_per_package);
196 topoParms.nPThreadsPerCore = cpuinfo->cpuid_logical_per_package / cpuinfo->cpuid_cores_per_package;
197
198 /*
199 * Compute the number of dies per package.
200 */
201 DIVISOR_GUARD(topoParms.nCoresSharingLLC);
202 topoParms.nLDiesPerPackage = cpuinfo->core_count / topoParms.nCoresSharingLLC;
203 DIVISOR_GUARD(topoParms.nPThreadsPerCore);
204 DIVISOR_GUARD(topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
205 topoParms.nPDiesPerPackage = cpuinfo->cpuid_cores_per_package / (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
206
207
208 /*
209 * Compute the number of cores per die.
210 */
211 topoParms.nLCoresPerDie = topoParms.nCoresSharingLLC;
212 topoParms.nPCoresPerDie = (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
213
214 /*
215 * Compute the number of threads per die.
216 */
217 topoParms.nLThreadsPerDie = topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie;
218 topoParms.nPThreadsPerDie = topoParms.nPThreadsPerCore * topoParms.nPCoresPerDie;
219
220 /*
221 * Compute the number of cores per package.
222 */
223 topoParms.nLCoresPerPackage = topoParms.nLCoresPerDie * topoParms.nLDiesPerPackage;
224 topoParms.nPCoresPerPackage = topoParms.nPCoresPerDie * topoParms.nPDiesPerPackage;
225
226 /*
227 * Compute the number of threads per package.
228 */
229 topoParms.nLThreadsPerPackage = topoParms.nLThreadsPerCore * topoParms.nLCoresPerPackage;
230 topoParms.nPThreadsPerPackage = topoParms.nPThreadsPerCore * topoParms.nPCoresPerPackage;
231
232 TOPO_DBG("\nCache Topology Parameters:\n");
233 TOPO_DBG("\tLLC Depth: %d\n", topoParms.LLCDepth);
234 TOPO_DBG("\tCores Sharing LLC: %d\n", topoParms.nCoresSharingLLC);
235 TOPO_DBG("\tThreads Sharing LLC: %d\n", topoParms.nLCPUsSharingLLC);
236 TOPO_DBG("\tmax Sharing of LLC: %d\n", topoParms.maxSharingLLC);
237
238 TOPO_DBG("\nLogical Topology Parameters:\n");
239 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore);
240 TOPO_DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie);
241 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie);
242 TOPO_DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage);
243 TOPO_DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage);
244 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage);
245
246 TOPO_DBG("\nPhysical Topology Parameters:\n");
247 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore);
248 TOPO_DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie);
249 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie);
250 TOPO_DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage);
251 TOPO_DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage);
252 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage);
253
254 topoParmsInited = TRUE;
255 }
256
257 static void
x86_cache_free(x86_cpu_cache_t * cache)258 x86_cache_free(x86_cpu_cache_t *cache)
259 {
260 num_caches -= 1;
261 if (cache->level > 0 && cache->level <= MAX_CACHE_DEPTH) {
262 num_Lx_caches[cache->level - 1] -= 1;
263 }
264 cache->next = x86_caches;
265 x86_caches = cache;
266 }
267
268 /*
269 * This returns a list of cache structures that represent the
270 * caches for a CPU. Some of the structures may have to be
271 * "freed" if they are actually shared between CPUs.
272 */
273 static x86_cpu_cache_t *
x86_cache_list(void)274 x86_cache_list(void)
275 {
276 x86_cpu_cache_t *root = NULL;
277 x86_cpu_cache_t *cur = NULL;
278 x86_cpu_cache_t *last = NULL;
279 struct cpu_cache *cachep;
280 int i;
281
282 /*
283 * Cons up a list driven not by CPUID leaf 4 (deterministic cache params)
284 * but by the table above plus parameters already cracked from cpuid...
285 */
286 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
287 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) {
288 continue;
289 }
290
291 cur = x86_cache_alloc();
292 if (cur == NULL) {
293 break;
294 }
295
296 cur->type = cachep->type;
297 cur->level = cachep->level;
298 cur->nlcpus = 0;
299 cur->maxcpus = cpuid_info()->cache_sharing[i];
300 cur->partitions = cpuid_info()->cache_partitions[i];
301 cur->cache_size = cpuid_info()->cache_size[i];
302 cur->line_size = cpuid_info()->cache_linesize;
303
304 if (last == NULL) {
305 root = cur;
306 last = cur;
307 } else {
308 last->next = cur;
309 last = cur;
310 }
311 num_Lx_caches[cur->level - 1] += 1;
312 }
313 return root;
314 }
315
316
317 static x86_cpu_cache_t *
x86_match_cache(x86_cpu_cache_t * list,x86_cpu_cache_t * matcher)318 x86_match_cache(x86_cpu_cache_t *list, x86_cpu_cache_t *matcher)
319 {
320 x86_cpu_cache_t *cur_cache;
321
322 cur_cache = list;
323 while (cur_cache != NULL) {
324 if (cur_cache->maxcpus == matcher->maxcpus
325 && cur_cache->type == matcher->type
326 && cur_cache->level == matcher->level
327 && cur_cache->partitions == matcher->partitions
328 && cur_cache->line_size == matcher->line_size
329 && cur_cache->cache_size == matcher->cache_size) {
330 break;
331 }
332
333 cur_cache = cur_cache->next;
334 }
335
336 return cur_cache;
337 }
338
339 static void
x86_lcpu_init(int cpu)340 x86_lcpu_init(int cpu)
341 {
342 cpu_data_t *cpup;
343 x86_lcpu_t *lcpu;
344 int i;
345
346 cpup = cpu_datap(cpu);
347
348 lcpu = &cpup->lcpu;
349 lcpu->lcpu = lcpu;
350 lcpu->cpu = cpup;
351 lcpu->next_in_core = NULL;
352 lcpu->next_in_die = NULL;
353 lcpu->next_in_pkg = NULL;
354 lcpu->core = NULL;
355 lcpu->die = NULL;
356 lcpu->package = NULL;
357 lcpu->cpu_num = cpu;
358 lcpu->lnum = cpu;
359 lcpu->pnum = cpup->cpu_phys_number;
360 lcpu->state = LCPU_OFF;
361 for (i = 0; i < MAX_CACHE_DEPTH; i += 1) {
362 lcpu->caches[i] = NULL;
363 }
364 }
365
366 static x86_core_t *
x86_core_alloc(int cpu)367 x86_core_alloc(int cpu)
368 {
369 x86_core_t *core;
370 cpu_data_t *cpup;
371
372 cpup = cpu_datap(cpu);
373
374 mp_safe_spin_lock(&x86_topo_lock);
375 if (free_cores != NULL) {
376 core = free_cores;
377 free_cores = core->next_in_die;
378 core->next_in_die = NULL;
379 simple_unlock(&x86_topo_lock);
380 } else {
381 simple_unlock(&x86_topo_lock);
382 core = zalloc_permanent_type(x86_core_t);
383 if (core == NULL) {
384 panic("x86_core_alloc() alloc of x86_core_t failed!");
385 }
386 }
387
388 core->pcore_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
389 core->lcore_num = core->pcore_num % topoParms.nPCoresPerPackage;
390
391 core->flags = X86CORE_FL_PRESENT | X86CORE_FL_READY
392 | X86CORE_FL_HALTED | X86CORE_FL_IDLE;
393
394 return core;
395 }
396
397 static void
x86_core_free(x86_core_t * core)398 x86_core_free(x86_core_t *core)
399 {
400 mp_safe_spin_lock(&x86_topo_lock);
401 core->next_in_die = free_cores;
402 free_cores = core;
403 simple_unlock(&x86_topo_lock);
404 }
405
406 static x86_pkg_t *
x86_package_find(int cpu)407 x86_package_find(int cpu)
408 {
409 x86_pkg_t *pkg;
410 cpu_data_t *cpup;
411 uint32_t pkg_num;
412
413 cpup = cpu_datap(cpu);
414
415 pkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
416
417 pkg = x86_pkgs;
418 while (pkg != NULL) {
419 if (pkg->ppkg_num == pkg_num) {
420 break;
421 }
422 pkg = pkg->next;
423 }
424
425 return pkg;
426 }
427
428 static x86_die_t *
x86_die_find(int cpu)429 x86_die_find(int cpu)
430 {
431 x86_die_t *die;
432 x86_pkg_t *pkg;
433 cpu_data_t *cpup;
434 uint32_t die_num;
435
436 cpup = cpu_datap(cpu);
437
438 die_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
439
440 pkg = x86_package_find(cpu);
441 if (pkg == NULL) {
442 return NULL;
443 }
444
445 die = pkg->dies;
446 while (die != NULL) {
447 if (die->pdie_num == die_num) {
448 break;
449 }
450 die = die->next_in_pkg;
451 }
452
453 return die;
454 }
455
456 static x86_core_t *
x86_core_find(int cpu)457 x86_core_find(int cpu)
458 {
459 x86_core_t *core;
460 x86_die_t *die;
461 cpu_data_t *cpup;
462 uint32_t core_num;
463
464 cpup = cpu_datap(cpu);
465
466 core_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
467
468 die = x86_die_find(cpu);
469 if (die == NULL) {
470 return NULL;
471 }
472
473 core = die->cores;
474 while (core != NULL) {
475 if (core->pcore_num == core_num) {
476 break;
477 }
478 core = core->next_in_die;
479 }
480
481 return core;
482 }
483
484 void
x86_set_logical_topology(x86_lcpu_t * lcpu,int pnum,int lnum)485 x86_set_logical_topology(x86_lcpu_t *lcpu, int pnum, int lnum)
486 {
487 x86_core_t *core = lcpu->core;
488 x86_die_t *die = lcpu->die;
489 x86_pkg_t *pkg = lcpu->package;
490
491 assert(core != NULL);
492 assert(die != NULL);
493 assert(pkg != NULL);
494
495 lcpu->cpu_num = lnum;
496 lcpu->pnum = pnum;
497 lcpu->master = (lnum == master_cpu);
498 lcpu->primary = (lnum % topoParms.nLThreadsPerPackage) == 0;
499
500 lcpu->lnum = lnum % topoParms.nLThreadsPerCore;
501
502 core->pcore_num = lnum / topoParms.nLThreadsPerCore;
503 core->lcore_num = core->pcore_num % topoParms.nLCoresPerDie;
504
505 die->pdie_num = lnum / (topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie);
506 die->ldie_num = die->pdie_num % topoParms.nLDiesPerPackage;
507
508 pkg->ppkg_num = lnum / topoParms.nLThreadsPerPackage;
509 pkg->lpkg_num = pkg->ppkg_num;
510 }
511
512 static x86_die_t *
x86_die_alloc(int cpu)513 x86_die_alloc(int cpu)
514 {
515 x86_die_t *die;
516 cpu_data_t *cpup;
517
518 cpup = cpu_datap(cpu);
519
520 mp_safe_spin_lock(&x86_topo_lock);
521 if (free_dies != NULL) {
522 die = free_dies;
523 free_dies = die->next_in_pkg;
524 die->next_in_pkg = NULL;
525 simple_unlock(&x86_topo_lock);
526 } else {
527 simple_unlock(&x86_topo_lock);
528 die = zalloc_permanent_type(x86_die_t);
529 if (die == NULL) {
530 panic("x86_die_alloc() alloc of x86_die_t failed!");
531 }
532 }
533
534 die->pdie_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
535
536 die->ldie_num = num_dies;
537 atomic_incl((long *) &num_dies, 1);
538
539 die->flags = X86DIE_FL_PRESENT;
540 return die;
541 }
542
543 static void
x86_die_free(x86_die_t * die)544 x86_die_free(x86_die_t *die)
545 {
546 mp_safe_spin_lock(&x86_topo_lock);
547 die->next_in_pkg = free_dies;
548 free_dies = die;
549 atomic_decl((long *) &num_dies, 1);
550 simple_unlock(&x86_topo_lock);
551 }
552
553 static x86_pkg_t *
x86_package_alloc(int cpu)554 x86_package_alloc(int cpu)
555 {
556 x86_pkg_t *pkg;
557 cpu_data_t *cpup;
558
559 cpup = cpu_datap(cpu);
560
561 mp_safe_spin_lock(&x86_topo_lock);
562 if (free_pkgs != NULL) {
563 pkg = free_pkgs;
564 free_pkgs = pkg->next;
565 pkg->next = NULL;
566 simple_unlock(&x86_topo_lock);
567 } else {
568 simple_unlock(&x86_topo_lock);
569 pkg = zalloc_permanent_type(x86_pkg_t);
570 if (pkg == NULL) {
571 panic("x86_package_alloc() alloc of x86_pkg_t failed!");
572 }
573 }
574
575 pkg->ppkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
576
577 pkg->lpkg_num = topoParms.nPackages;
578 atomic_incl((long *) &topoParms.nPackages, 1);
579
580 pkg->flags = X86PKG_FL_PRESENT | X86PKG_FL_READY;
581 return pkg;
582 }
583
584 static void
x86_package_free(x86_pkg_t * pkg)585 x86_package_free(x86_pkg_t *pkg)
586 {
587 mp_safe_spin_lock(&x86_topo_lock);
588 pkg->next = free_pkgs;
589 free_pkgs = pkg;
590 atomic_decl((long *) &topoParms.nPackages, 1);
591 simple_unlock(&x86_topo_lock);
592 }
593
594 static void
x86_cache_add_lcpu(x86_cpu_cache_t * cache,x86_lcpu_t * lcpu)595 x86_cache_add_lcpu(x86_cpu_cache_t *cache, x86_lcpu_t *lcpu)
596 {
597 x86_cpu_cache_t *cur_cache;
598 int i;
599
600 /*
601 * Put the new CPU into the list of the cache.
602 */
603 cur_cache = lcpu->caches[cache->level - 1];
604 lcpu->caches[cache->level - 1] = cache;
605 cache->next = cur_cache;
606 cache->nlcpus += 1;
607 for (i = 0; i < cache->nlcpus; i += 1) {
608 if (cache->cpus[i] == NULL) {
609 cache->cpus[i] = lcpu;
610 break;
611 }
612 }
613 }
614
615 static void
x86_lcpu_add_caches(x86_lcpu_t * lcpu)616 x86_lcpu_add_caches(x86_lcpu_t *lcpu)
617 {
618 x86_cpu_cache_t *list;
619 x86_cpu_cache_t *cur;
620 x86_cpu_cache_t *match;
621 x86_die_t *die;
622 x86_core_t *core;
623 x86_lcpu_t *cur_lcpu;
624 uint32_t level;
625 boolean_t found = FALSE;
626
627 assert(lcpu != NULL);
628
629 /*
630 * Add the cache data to the topology.
631 */
632 list = x86_cache_list();
633
634 mp_safe_spin_lock(&x86_topo_lock);
635
636 while (list != NULL) {
637 /*
638 * Remove the cache from the front of the list.
639 */
640 cur = list;
641 list = cur->next;
642 cur->next = NULL;
643 level = cur->level - 1;
644
645 /*
646 * If the cache isn't shared then just put it where it
647 * belongs.
648 */
649 if (cur->maxcpus == 1) {
650 x86_cache_add_lcpu(cur, lcpu);
651 continue;
652 }
653
654 /*
655 * We'll assume that all of the caches at a particular level
656 * have the same sharing. So if we have a cache already at
657 * this level, we'll just skip looking for the match.
658 */
659 if (lcpu->caches[level] != NULL) {
660 x86_cache_free(cur);
661 continue;
662 }
663
664 /*
665 * This is a shared cache, so we have to figure out if
666 * this is the first time we've seen this cache. We do
667 * this by searching through the topology and seeing if
668 * this cache is already described.
669 *
670 * Assume that L{LLC-1} are all at the core level and that
671 * LLC is shared at the die level.
672 */
673 if (level < topoParms.LLCDepth) {
674 /*
675 * Shared at the core.
676 */
677 core = lcpu->core;
678 cur_lcpu = core->lcpus;
679 while (cur_lcpu != NULL) {
680 /*
681 * Skip ourselves.
682 */
683 if (cur_lcpu == lcpu) {
684 cur_lcpu = cur_lcpu->next_in_core;
685 continue;
686 }
687
688 /*
689 * If there's a cache on this logical CPU,
690 * then use that one.
691 */
692 match = x86_match_cache(cur_lcpu->caches[level], cur);
693 if (match != NULL) {
694 x86_cache_free(cur);
695 x86_cache_add_lcpu(match, lcpu);
696 found = TRUE;
697 break;
698 }
699
700 cur_lcpu = cur_lcpu->next_in_core;
701 }
702 } else {
703 /*
704 * Shared at the die.
705 */
706 die = lcpu->die;
707 cur_lcpu = die->lcpus;
708 while (cur_lcpu != NULL) {
709 /*
710 * Skip ourselves.
711 */
712 if (cur_lcpu == lcpu) {
713 cur_lcpu = cur_lcpu->next_in_die;
714 continue;
715 }
716
717 /*
718 * If there's a cache on this logical CPU,
719 * then use that one.
720 */
721 match = x86_match_cache(cur_lcpu->caches[level], cur);
722 if (match != NULL) {
723 x86_cache_free(cur);
724 x86_cache_add_lcpu(match, lcpu);
725 found = TRUE;
726 break;
727 }
728
729 cur_lcpu = cur_lcpu->next_in_die;
730 }
731 }
732
733 /*
734 * If a shared cache wasn't found, then this logical CPU must
735 * be the first one encountered.
736 */
737 if (!found) {
738 x86_cache_add_lcpu(cur, lcpu);
739 }
740 }
741
742 simple_unlock(&x86_topo_lock);
743 }
744
745 static void
x86_core_add_lcpu(x86_core_t * core,x86_lcpu_t * lcpu)746 x86_core_add_lcpu(x86_core_t *core, x86_lcpu_t *lcpu)
747 {
748 assert(core != NULL);
749 assert(lcpu != NULL);
750
751 mp_safe_spin_lock(&x86_topo_lock);
752
753 lcpu->next_in_core = core->lcpus;
754 lcpu->core = core;
755 core->lcpus = lcpu;
756 core->num_lcpus += 1;
757 simple_unlock(&x86_topo_lock);
758 }
759
760 static void
x86_die_add_lcpu(x86_die_t * die,x86_lcpu_t * lcpu)761 x86_die_add_lcpu(x86_die_t *die, x86_lcpu_t *lcpu)
762 {
763 assert(die != NULL);
764 assert(lcpu != NULL);
765
766 lcpu->next_in_die = die->lcpus;
767 lcpu->die = die;
768 die->lcpus = lcpu;
769 }
770
771 static void
x86_die_add_core(x86_die_t * die,x86_core_t * core)772 x86_die_add_core(x86_die_t *die, x86_core_t *core)
773 {
774 assert(die != NULL);
775 assert(core != NULL);
776
777 core->next_in_die = die->cores;
778 core->die = die;
779 die->cores = core;
780 die->num_cores += 1;
781 }
782
783 static void
x86_package_add_lcpu(x86_pkg_t * pkg,x86_lcpu_t * lcpu)784 x86_package_add_lcpu(x86_pkg_t *pkg, x86_lcpu_t *lcpu)
785 {
786 assert(pkg != NULL);
787 assert(lcpu != NULL);
788
789 lcpu->next_in_pkg = pkg->lcpus;
790 lcpu->package = pkg;
791 pkg->lcpus = lcpu;
792 }
793
794 static void
x86_package_add_core(x86_pkg_t * pkg,x86_core_t * core)795 x86_package_add_core(x86_pkg_t *pkg, x86_core_t *core)
796 {
797 assert(pkg != NULL);
798 assert(core != NULL);
799
800 core->next_in_pkg = pkg->cores;
801 core->package = pkg;
802 pkg->cores = core;
803 }
804
805 static void
x86_package_add_die(x86_pkg_t * pkg,x86_die_t * die)806 x86_package_add_die(x86_pkg_t *pkg, x86_die_t *die)
807 {
808 assert(pkg != NULL);
809 assert(die != NULL);
810
811 die->next_in_pkg = pkg->dies;
812 die->package = pkg;
813 pkg->dies = die;
814 pkg->num_dies += 1;
815 }
816
817 void *
cpu_thread_alloc(int cpu)818 cpu_thread_alloc(int cpu)
819 {
820 x86_core_t *core = NULL;
821 x86_die_t *die = NULL;
822 x86_pkg_t *pkg = NULL;
823 cpu_data_t *cpup;
824 uint32_t phys_cpu;
825
826 /*
827 * Only allow one to manipulate the topology at a time.
828 */
829 mp_safe_spin_lock(&x86_topo_lock);
830
831 /*
832 * Make sure all of the topology parameters have been initialized.
833 */
834 if (!topoParmsInited) {
835 initTopoParms();
836 }
837
838 cpup = cpu_datap(cpu);
839
840 phys_cpu = cpup->cpu_phys_number;
841
842 x86_lcpu_init(cpu);
843
844 /*
845 * Assume that all cpus have the same features.
846 */
847 if (cpu_is_hyperthreaded()) {
848 cpup->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
849 } else {
850 cpup->cpu_threadtype = CPU_THREADTYPE_NONE;
851 }
852
853 /*
854 * Get the package that the logical CPU is in.
855 */
856 do {
857 pkg = x86_package_find(cpu);
858 if (pkg == NULL) {
859 /*
860 * Package structure hasn't been created yet, do it now.
861 */
862 simple_unlock(&x86_topo_lock);
863 pkg = x86_package_alloc(cpu);
864 mp_safe_spin_lock(&x86_topo_lock);
865 if (x86_package_find(cpu) != NULL) {
866 x86_package_free(pkg);
867 continue;
868 }
869
870 /*
871 * Add the new package to the global list of packages.
872 */
873 pkg->next = x86_pkgs;
874 x86_pkgs = pkg;
875 }
876 } while (pkg == NULL);
877
878 /*
879 * Get the die that the logical CPU is in.
880 */
881 do {
882 die = x86_die_find(cpu);
883 if (die == NULL) {
884 /*
885 * Die structure hasn't been created yet, do it now.
886 */
887 simple_unlock(&x86_topo_lock);
888 die = x86_die_alloc(cpu);
889 mp_safe_spin_lock(&x86_topo_lock);
890 if (x86_die_find(cpu) != NULL) {
891 x86_die_free(die);
892 continue;
893 }
894
895 /*
896 * Add the die to the package.
897 */
898 x86_package_add_die(pkg, die);
899 }
900 } while (die == NULL);
901
902 /*
903 * Get the core for this logical CPU.
904 */
905 do {
906 core = x86_core_find(cpu);
907 if (core == NULL) {
908 /*
909 * Allocate the core structure now.
910 */
911 simple_unlock(&x86_topo_lock);
912 core = x86_core_alloc(cpu);
913 mp_safe_spin_lock(&x86_topo_lock);
914 if (x86_core_find(cpu) != NULL) {
915 x86_core_free(core);
916 continue;
917 }
918
919 /*
920 * Add the core to the die & package.
921 */
922 x86_die_add_core(die, core);
923 x86_package_add_core(pkg, core);
924 machine_info.physical_cpu_max += 1;
925 }
926 } while (core == NULL);
927
928
929 /*
930 * Done manipulating the topology, so others can get in.
931 */
932 machine_info.logical_cpu_max += 1;
933 simple_unlock(&x86_topo_lock);
934
935 /*
936 * Add the logical CPU to the other topology structures.
937 */
938 x86_core_add_lcpu(core, &cpup->lcpu);
939 x86_die_add_lcpu(core->die, &cpup->lcpu);
940 x86_package_add_lcpu(core->package, &cpup->lcpu);
941 x86_lcpu_add_caches(&cpup->lcpu);
942
943 return (void *) core;
944 }
945
946 void
cpu_thread_init(void)947 cpu_thread_init(void)
948 {
949 int my_cpu = get_cpu_number();
950 cpu_data_t *cpup = current_cpu_datap();
951 x86_core_t *core;
952 static int initialized = 0;
953
954 /*
955 * If we're the boot processor, we do all of the initialization of
956 * the CPU topology infrastructure.
957 */
958 if (my_cpu == master_cpu && !initialized) {
959 simple_lock_init(&x86_topo_lock, 0);
960
961 /*
962 * Put this logical CPU into the physical CPU topology.
963 */
964 cpup->lcpu.core = cpu_thread_alloc(my_cpu);
965
966 initialized = 1;
967 }
968
969 /*
970 * Do the CPU accounting.
971 */
972 core = cpup->lcpu.core;
973 mp_safe_spin_lock(&x86_topo_lock);
974 machine_info.logical_cpu += 1;
975 if (core->active_lcpus == 0) {
976 machine_info.physical_cpu += 1;
977 }
978 core->active_lcpus += 1;
979 simple_unlock(&x86_topo_lock);
980
981 pmCPUMarkRunning(cpup);
982 timer_resync_deadlines();
983 }
984
985 /*
986 * Called for a cpu to halt permanently
987 * (as opposed to halting and expecting an interrupt to awaken it).
988 */
989 __attribute__((noreturn))
990 void
cpu_thread_halt(void)991 cpu_thread_halt(void)
992 {
993 x86_core_t *core;
994 cpu_data_t *cpup = current_cpu_datap();
995
996 mp_safe_spin_lock(&x86_topo_lock);
997 machine_info.logical_cpu -= 1;
998 core = cpup->lcpu.core;
999 core->active_lcpus -= 1;
1000 if (core->active_lcpus == 0) {
1001 machine_info.physical_cpu -= 1;
1002 }
1003 simple_unlock(&x86_topo_lock);
1004
1005 /*
1006 * Let the power management code determine the best way to "stop"
1007 * the processor.
1008 */
1009 ml_set_interrupts_enabled(FALSE);
1010 while (1) {
1011 pmCPUHalt(PM_HALT_NORMAL);
1012 }
1013 /* NOT REACHED */
1014 }
1015
1016 /*
1017 * Validates that the topology was built correctly. Must be called only
1018 * after the complete topology is built and no other changes are being made.
1019 */
1020 void
x86_validate_topology(void)1021 x86_validate_topology(void)
1022 {
1023 x86_pkg_t *pkg;
1024 x86_die_t *die;
1025 x86_core_t *core;
1026 x86_lcpu_t *lcpu;
1027 uint32_t nDies;
1028 uint32_t nCores;
1029 uint32_t nCPUs;
1030
1031 if (topo_dbg) {
1032 debug_topology_print();
1033 }
1034
1035 /*
1036 * Called after processors are registered but before non-boot processors
1037 * are started:
1038 * - real_ncpus: number of registered processors driven from MADT
1039 * - max_ncpus: max number of processors that will be started
1040 */
1041 nCPUs = topoParms.nPackages * topoParms.nLThreadsPerPackage;
1042 if (nCPUs != real_ncpus) {
1043 panic("x86_validate_topology() %d threads but %d registered from MADT",
1044 nCPUs, real_ncpus);
1045 }
1046
1047 pkg = x86_pkgs;
1048 while (pkg != NULL) {
1049 /*
1050 * Make sure that the package has the correct number of dies.
1051 */
1052 nDies = 0;
1053 die = pkg->dies;
1054 while (die != NULL) {
1055 if (die->package == NULL) {
1056 panic("Die(%d)->package is NULL",
1057 die->pdie_num);
1058 }
1059 if (die->package != pkg) {
1060 panic("Die %d points to package %d, should be %d",
1061 die->pdie_num, die->package->lpkg_num, pkg->lpkg_num);
1062 }
1063
1064 TOPO_DBG("Die(%d)->package %d\n",
1065 die->pdie_num, pkg->lpkg_num);
1066
1067 /*
1068 * Make sure that the die has the correct number of cores.
1069 */
1070 TOPO_DBG("Die(%d)->cores: ", die->pdie_num);
1071 nCores = 0;
1072 core = die->cores;
1073 while (core != NULL) {
1074 if (core->die == NULL) {
1075 panic("Core(%d)->die is NULL",
1076 core->pcore_num);
1077 }
1078 if (core->die != die) {
1079 panic("Core %d points to die %d, should be %d",
1080 core->pcore_num, core->die->pdie_num, die->pdie_num);
1081 }
1082 nCores += 1;
1083 TOPO_DBG("%d ", core->pcore_num);
1084 core = core->next_in_die;
1085 }
1086 TOPO_DBG("\n");
1087
1088 if (nCores != topoParms.nLCoresPerDie) {
1089 panic("Should have %d Cores, but only found %d for Die %d",
1090 topoParms.nLCoresPerDie, nCores, die->pdie_num);
1091 }
1092
1093 /*
1094 * Make sure that the die has the correct number of CPUs.
1095 */
1096 TOPO_DBG("Die(%d)->lcpus: ", die->pdie_num);
1097 nCPUs = 0;
1098 lcpu = die->lcpus;
1099 while (lcpu != NULL) {
1100 if (lcpu->die == NULL) {
1101 panic("CPU(%d)->die is NULL",
1102 lcpu->cpu_num);
1103 }
1104 if (lcpu->die != die) {
1105 panic("CPU %d points to die %d, should be %d",
1106 lcpu->cpu_num, lcpu->die->pdie_num, die->pdie_num);
1107 }
1108 nCPUs += 1;
1109 TOPO_DBG("%d ", lcpu->cpu_num);
1110 lcpu = lcpu->next_in_die;
1111 }
1112 TOPO_DBG("\n");
1113
1114 if (nCPUs != topoParms.nLThreadsPerDie) {
1115 panic("Should have %d Threads, but only found %d for Die %d",
1116 topoParms.nLThreadsPerDie, nCPUs, die->pdie_num);
1117 }
1118
1119 nDies += 1;
1120 die = die->next_in_pkg;
1121 }
1122
1123 if (nDies != topoParms.nLDiesPerPackage) {
1124 panic("Should have %d Dies, but only found %d for package %d",
1125 topoParms.nLDiesPerPackage, nDies, pkg->lpkg_num);
1126 }
1127
1128 /*
1129 * Make sure that the package has the correct number of cores.
1130 */
1131 nCores = 0;
1132 core = pkg->cores;
1133 while (core != NULL) {
1134 if (core->package == NULL) {
1135 panic("Core(%d)->package is NULL",
1136 core->pcore_num);
1137 }
1138 if (core->package != pkg) {
1139 panic("Core %d points to package %d, should be %d",
1140 core->pcore_num, core->package->lpkg_num, pkg->lpkg_num);
1141 }
1142 TOPO_DBG("Core(%d)->package %d\n",
1143 core->pcore_num, pkg->lpkg_num);
1144
1145 /*
1146 * Make sure that the core has the correct number of CPUs.
1147 */
1148 nCPUs = 0;
1149 lcpu = core->lcpus;
1150 TOPO_DBG("Core(%d)->lcpus: ", core->pcore_num);
1151 while (lcpu != NULL) {
1152 if (lcpu->core == NULL) {
1153 panic("CPU(%d)->core is NULL",
1154 lcpu->cpu_num);
1155 }
1156 if (lcpu->core != core) {
1157 panic("CPU %d points to core %d, should be %d",
1158 lcpu->cpu_num, lcpu->core->pcore_num, core->pcore_num);
1159 }
1160 TOPO_DBG("%d ", lcpu->cpu_num);
1161 nCPUs += 1;
1162 lcpu = lcpu->next_in_core;
1163 }
1164 TOPO_DBG("\n");
1165
1166 if (nCPUs != topoParms.nLThreadsPerCore) {
1167 panic("Should have %d Threads, but only found %d for Core %d",
1168 topoParms.nLThreadsPerCore, nCPUs, core->pcore_num);
1169 }
1170 nCores += 1;
1171 core = core->next_in_pkg;
1172 }
1173
1174 if (nCores != topoParms.nLCoresPerPackage) {
1175 panic("Should have %d Cores, but only found %d for package %d",
1176 topoParms.nLCoresPerPackage, nCores, pkg->lpkg_num);
1177 }
1178
1179 /*
1180 * Make sure that the package has the correct number of CPUs.
1181 */
1182 nCPUs = 0;
1183 lcpu = pkg->lcpus;
1184 while (lcpu != NULL) {
1185 if (lcpu->package == NULL) {
1186 panic("CPU(%d)->package is NULL",
1187 lcpu->cpu_num);
1188 }
1189 if (lcpu->package != pkg) {
1190 panic("CPU %d points to package %d, should be %d",
1191 lcpu->cpu_num, lcpu->package->lpkg_num, pkg->lpkg_num);
1192 }
1193 TOPO_DBG("CPU(%d)->package %d\n",
1194 lcpu->cpu_num, pkg->lpkg_num);
1195 nCPUs += 1;
1196 lcpu = lcpu->next_in_pkg;
1197 }
1198
1199 if (nCPUs != topoParms.nLThreadsPerPackage) {
1200 panic("Should have %d Threads, but only found %d for package %d",
1201 topoParms.nLThreadsPerPackage, nCPUs, pkg->lpkg_num);
1202 }
1203
1204 pkg = pkg->next;
1205 }
1206 }
1207
1208 /*
1209 * Prints out the topology
1210 */
1211 static void
debug_topology_print(void)1212 debug_topology_print(void)
1213 {
1214 x86_pkg_t *pkg;
1215 x86_die_t *die;
1216 x86_core_t *core;
1217 x86_lcpu_t *cpu;
1218
1219 pkg = x86_pkgs;
1220 while (pkg != NULL) {
1221 kprintf("Package:\n");
1222 kprintf(" Physical: %d\n", pkg->ppkg_num);
1223 kprintf(" Logical: %d\n", pkg->lpkg_num);
1224
1225 die = pkg->dies;
1226 while (die != NULL) {
1227 kprintf(" Die:\n");
1228 kprintf(" Physical: %d\n", die->pdie_num);
1229 kprintf(" Logical: %d\n", die->ldie_num);
1230
1231 core = die->cores;
1232 while (core != NULL) {
1233 kprintf(" Core:\n");
1234 kprintf(" Physical: %d\n", core->pcore_num);
1235 kprintf(" Logical: %d\n", core->lcore_num);
1236
1237 cpu = core->lcpus;
1238 while (cpu != NULL) {
1239 kprintf(" LCPU:\n");
1240 kprintf(" CPU #: %d\n", cpu->cpu_num);
1241 kprintf(" Physical: %d\n", cpu->pnum);
1242 kprintf(" Logical: %d\n", cpu->lnum);
1243 kprintf(" Flags: ");
1244 if (cpu->master) {
1245 kprintf("MASTER ");
1246 }
1247 if (cpu->primary) {
1248 kprintf("PRIMARY");
1249 }
1250 if (!cpu->master && !cpu->primary) {
1251 kprintf("(NONE)");
1252 }
1253 kprintf("\n");
1254
1255 cpu = cpu->next_in_core;
1256 }
1257
1258 core = core->next_in_die;
1259 }
1260
1261 die = die->next_in_pkg;
1262 }
1263
1264 pkg = pkg->next;
1265 }
1266 }
1267