1 /*
2 * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #include <vm/vm_kern.h>
29 #include <kern/zalloc.h>
30 #include <kern/lock_group.h>
31 #include <kern/timer_queue.h>
32 #include <mach/machine.h>
33 #include <i386/cpu_threads.h>
34 #include <i386/cpuid.h>
35 #include <i386/machine_cpu.h>
36 #include <i386/pmCPU.h>
37 #include <i386/bit_routines.h>
38
39 #if MONOTONIC
40 #include <kern/monotonic.h>
41 #endif /* MONOTONIC */
42
43 #define DIVISOR_GUARD(denom) \
44 if ((denom) == 0) { \
45 kprintf("%s: %d Zero divisor: " #denom, \
46 __FILE__, __LINE__); \
47 }
48
49 static void debug_topology_print(void);
50
51 boolean_t topo_dbg = FALSE;
52
53 x86_pkg_t *x86_pkgs = NULL;
54 uint32_t num_Lx_caches[MAX_CACHE_DEPTH] = { 0 };
55
56 static x86_pkg_t *free_pkgs = NULL;
57 static x86_die_t *free_dies = NULL;
58 static x86_core_t *free_cores = NULL;
59 static uint32_t num_dies = 0;
60
61 static x86_cpu_cache_t *x86_caches = NULL;
62 static uint32_t num_caches = 0;
63
64 static boolean_t topoParmsInited = FALSE;
65 x86_topology_parameters_t topoParms;
66
67 decl_simple_lock_data(, x86_topo_lock);
68
69 static struct cpu_cache {
70 int level; int type;
71 } cpu_caches[LCACHE_MAX] = {
72 [L1D] = { 1, CPU_CACHE_TYPE_DATA },
73 [L1I] = { 1, CPU_CACHE_TYPE_INST },
74 [L2U] = { 2, CPU_CACHE_TYPE_UNIF },
75 [L3U] = { 3, CPU_CACHE_TYPE_UNIF },
76 };
77
78 static boolean_t
cpu_is_hyperthreaded(void)79 cpu_is_hyperthreaded(void)
80 {
81 i386_cpu_info_t *cpuinfo;
82
83 cpuinfo = cpuid_info();
84 return cpuinfo->thread_count > cpuinfo->core_count;
85 }
86
87 static x86_cpu_cache_t *
x86_cache_alloc(void)88 x86_cache_alloc(void)
89 {
90 x86_cpu_cache_t *cache;
91 int i;
92
93 if (x86_caches == NULL) {
94 cache = zalloc_permanent(sizeof(x86_cpu_cache_t) +
95 (MAX_CPUS * sizeof(x86_lcpu_t *)), ZALIGN(x86_cpu_cache_t));
96 if (cache == NULL) {
97 return NULL;
98 }
99 } else {
100 cache = x86_caches;
101 x86_caches = cache->next;
102 cache->next = NULL;
103 }
104
105 cache->next = NULL;
106 cache->maxcpus = MAX_CPUS;
107 for (i = 0; i < cache->maxcpus; i += 1) {
108 cache->cpus[i] = NULL;
109 }
110
111 num_caches += 1;
112
113 return cache;
114 }
115
116 static void
x86_LLC_info(void)117 x86_LLC_info(void)
118 {
119 int cache_level = 0;
120 uint32_t nCPUsSharing = 1;
121 i386_cpu_info_t *cpuinfo;
122 struct cpu_cache *cachep;
123 int i;
124
125 cpuinfo = cpuid_info();
126
127 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
128 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) {
129 continue;
130 }
131
132 /*
133 * Only worry about it if it's a deeper level than
134 * what we've seen before.
135 */
136 if (cachep->level > cache_level) {
137 cache_level = cachep->level;
138
139 /*
140 * Save the number of CPUs sharing this cache.
141 */
142 nCPUsSharing = cpuinfo->cache_sharing[i];
143 }
144 }
145
146 /*
147 * Make the level of the LLC be 0 based.
148 */
149 topoParms.LLCDepth = cache_level - 1;
150
151 /*
152 * nCPUsSharing represents the *maximum* number of cores or
153 * logical CPUs sharing the cache.
154 */
155 topoParms.maxSharingLLC = nCPUsSharing;
156
157 topoParms.nCoresSharingLLC = nCPUsSharing / (cpuinfo->thread_count /
158 cpuinfo->core_count);
159 topoParms.nLCPUsSharingLLC = nCPUsSharing;
160
161 /*
162 * nCPUsSharing may not be the number of *active* cores or
163 * threads that are sharing the cache.
164 */
165 if (nCPUsSharing > cpuinfo->core_count) {
166 topoParms.nCoresSharingLLC = cpuinfo->core_count;
167 }
168 if (nCPUsSharing > cpuinfo->thread_count) {
169 topoParms.nLCPUsSharingLLC = cpuinfo->thread_count;
170 }
171 }
172
173 static void
initTopoParms(void)174 initTopoParms(void)
175 {
176 i386_cpu_info_t *cpuinfo;
177
178 topoParms.stable = FALSE;
179
180 cpuinfo = cpuid_info();
181
182 PE_parse_boot_argn("-topo", &topo_dbg, sizeof(topo_dbg));
183
184 /*
185 * We need to start with getting the LLC information correct.
186 */
187 x86_LLC_info();
188
189 /*
190 * Compute the number of threads (logical CPUs) per core.
191 */
192 DIVISOR_GUARD(cpuinfo->core_count);
193 topoParms.nLThreadsPerCore = cpuinfo->thread_count / cpuinfo->core_count;
194 DIVISOR_GUARD(cpuinfo->cpuid_cores_per_package);
195 topoParms.nPThreadsPerCore = cpuinfo->cpuid_logical_per_package / cpuinfo->cpuid_cores_per_package;
196
197 /*
198 * Compute the number of dies per package.
199 */
200 DIVISOR_GUARD(topoParms.nCoresSharingLLC);
201 topoParms.nLDiesPerPackage = cpuinfo->core_count / topoParms.nCoresSharingLLC;
202 DIVISOR_GUARD(topoParms.nPThreadsPerCore);
203 DIVISOR_GUARD(topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
204 topoParms.nPDiesPerPackage = cpuinfo->cpuid_cores_per_package / (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
205
206
207 /*
208 * Compute the number of cores per die.
209 */
210 topoParms.nLCoresPerDie = topoParms.nCoresSharingLLC;
211 topoParms.nPCoresPerDie = (topoParms.maxSharingLLC / topoParms.nPThreadsPerCore);
212
213 /*
214 * Compute the number of threads per die.
215 */
216 topoParms.nLThreadsPerDie = topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie;
217 topoParms.nPThreadsPerDie = topoParms.nPThreadsPerCore * topoParms.nPCoresPerDie;
218
219 /*
220 * Compute the number of cores per package.
221 */
222 topoParms.nLCoresPerPackage = topoParms.nLCoresPerDie * topoParms.nLDiesPerPackage;
223 topoParms.nPCoresPerPackage = topoParms.nPCoresPerDie * topoParms.nPDiesPerPackage;
224
225 /*
226 * Compute the number of threads per package.
227 */
228 topoParms.nLThreadsPerPackage = topoParms.nLThreadsPerCore * topoParms.nLCoresPerPackage;
229 topoParms.nPThreadsPerPackage = topoParms.nPThreadsPerCore * topoParms.nPCoresPerPackage;
230
231 TOPO_DBG("\nCache Topology Parameters:\n");
232 TOPO_DBG("\tLLC Depth: %d\n", topoParms.LLCDepth);
233 TOPO_DBG("\tCores Sharing LLC: %d\n", topoParms.nCoresSharingLLC);
234 TOPO_DBG("\tThreads Sharing LLC: %d\n", topoParms.nLCPUsSharingLLC);
235 TOPO_DBG("\tmax Sharing of LLC: %d\n", topoParms.maxSharingLLC);
236
237 TOPO_DBG("\nLogical Topology Parameters:\n");
238 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nLThreadsPerCore);
239 TOPO_DBG("\tCores per Die: %d\n", topoParms.nLCoresPerDie);
240 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nLThreadsPerDie);
241 TOPO_DBG("\tDies per Package: %d\n", topoParms.nLDiesPerPackage);
242 TOPO_DBG("\tCores per Package: %d\n", topoParms.nLCoresPerPackage);
243 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nLThreadsPerPackage);
244
245 TOPO_DBG("\nPhysical Topology Parameters:\n");
246 TOPO_DBG("\tThreads per Core: %d\n", topoParms.nPThreadsPerCore);
247 TOPO_DBG("\tCores per Die: %d\n", topoParms.nPCoresPerDie);
248 TOPO_DBG("\tThreads per Die: %d\n", topoParms.nPThreadsPerDie);
249 TOPO_DBG("\tDies per Package: %d\n", topoParms.nPDiesPerPackage);
250 TOPO_DBG("\tCores per Package: %d\n", topoParms.nPCoresPerPackage);
251 TOPO_DBG("\tThreads per Package: %d\n", topoParms.nPThreadsPerPackage);
252
253 topoParmsInited = TRUE;
254 }
255
256 static void
x86_cache_free(x86_cpu_cache_t * cache)257 x86_cache_free(x86_cpu_cache_t *cache)
258 {
259 num_caches -= 1;
260 if (cache->level > 0 && cache->level <= MAX_CACHE_DEPTH) {
261 num_Lx_caches[cache->level - 1] -= 1;
262 }
263 cache->next = x86_caches;
264 x86_caches = cache;
265 }
266
267 /*
268 * This returns a list of cache structures that represent the
269 * caches for a CPU. Some of the structures may have to be
270 * "freed" if they are actually shared between CPUs.
271 */
272 static x86_cpu_cache_t *
x86_cache_list(void)273 x86_cache_list(void)
274 {
275 x86_cpu_cache_t *root = NULL;
276 x86_cpu_cache_t *cur = NULL;
277 x86_cpu_cache_t *last = NULL;
278 struct cpu_cache *cachep;
279 int i;
280
281 /*
282 * Cons up a list driven not by CPUID leaf 4 (deterministic cache params)
283 * but by the table above plus parameters already cracked from cpuid...
284 */
285 for (i = 0, cachep = &cpu_caches[0]; i < LCACHE_MAX; i++, cachep++) {
286 if (cachep->type == 0 || cpuid_info()->cache_size[i] == 0) {
287 continue;
288 }
289
290 cur = x86_cache_alloc();
291 if (cur == NULL) {
292 break;
293 }
294
295 cur->type = cachep->type;
296 cur->level = cachep->level;
297 cur->nlcpus = 0;
298 cur->maxcpus = cpuid_info()->cache_sharing[i];
299 cur->partitions = cpuid_info()->cache_partitions[i];
300 cur->cache_size = cpuid_info()->cache_size[i];
301 cur->line_size = cpuid_info()->cache_linesize;
302
303 if (last == NULL) {
304 root = cur;
305 last = cur;
306 } else {
307 last->next = cur;
308 last = cur;
309 }
310 num_Lx_caches[cur->level - 1] += 1;
311 }
312 return root;
313 }
314
315
316 static x86_cpu_cache_t *
x86_match_cache(x86_cpu_cache_t * list,x86_cpu_cache_t * matcher)317 x86_match_cache(x86_cpu_cache_t *list, x86_cpu_cache_t *matcher)
318 {
319 x86_cpu_cache_t *cur_cache;
320
321 cur_cache = list;
322 while (cur_cache != NULL) {
323 if (cur_cache->maxcpus == matcher->maxcpus
324 && cur_cache->type == matcher->type
325 && cur_cache->level == matcher->level
326 && cur_cache->partitions == matcher->partitions
327 && cur_cache->line_size == matcher->line_size
328 && cur_cache->cache_size == matcher->cache_size) {
329 break;
330 }
331
332 cur_cache = cur_cache->next;
333 }
334
335 return cur_cache;
336 }
337
338 static void
x86_lcpu_init(int cpu)339 x86_lcpu_init(int cpu)
340 {
341 cpu_data_t *cpup;
342 x86_lcpu_t *lcpu;
343 int i;
344
345 cpup = cpu_datap(cpu);
346
347 lcpu = &cpup->lcpu;
348 lcpu->lcpu = lcpu;
349 lcpu->cpu = cpup;
350 lcpu->next_in_core = NULL;
351 lcpu->next_in_die = NULL;
352 lcpu->next_in_pkg = NULL;
353 lcpu->core = NULL;
354 lcpu->die = NULL;
355 lcpu->package = NULL;
356 lcpu->cpu_num = cpu;
357 lcpu->lnum = cpu;
358 lcpu->pnum = cpup->cpu_phys_number;
359 lcpu->state = LCPU_OFF;
360 for (i = 0; i < MAX_CACHE_DEPTH; i += 1) {
361 lcpu->caches[i] = NULL;
362 }
363 }
364
365 static x86_core_t *
x86_core_alloc(int cpu)366 x86_core_alloc(int cpu)
367 {
368 x86_core_t *core;
369 cpu_data_t *cpup;
370
371 cpup = cpu_datap(cpu);
372
373 mp_safe_spin_lock(&x86_topo_lock);
374 if (free_cores != NULL) {
375 core = free_cores;
376 free_cores = core->next_in_die;
377 core->next_in_die = NULL;
378 simple_unlock(&x86_topo_lock);
379 } else {
380 simple_unlock(&x86_topo_lock);
381 core = zalloc_permanent_type(x86_core_t);
382 if (core == NULL) {
383 panic("x86_core_alloc() alloc of x86_core_t failed!");
384 }
385 }
386
387 core->pcore_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
388 core->lcore_num = core->pcore_num % topoParms.nPCoresPerPackage;
389
390 core->flags = X86CORE_FL_PRESENT | X86CORE_FL_READY
391 | X86CORE_FL_HALTED | X86CORE_FL_IDLE;
392
393 return core;
394 }
395
396 static void
x86_core_free(x86_core_t * core)397 x86_core_free(x86_core_t *core)
398 {
399 mp_safe_spin_lock(&x86_topo_lock);
400 core->next_in_die = free_cores;
401 free_cores = core;
402 simple_unlock(&x86_topo_lock);
403 }
404
405 static x86_pkg_t *
x86_package_find(int cpu)406 x86_package_find(int cpu)
407 {
408 x86_pkg_t *pkg;
409 cpu_data_t *cpup;
410 uint32_t pkg_num;
411
412 cpup = cpu_datap(cpu);
413
414 pkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
415
416 pkg = x86_pkgs;
417 while (pkg != NULL) {
418 if (pkg->ppkg_num == pkg_num) {
419 break;
420 }
421 pkg = pkg->next;
422 }
423
424 return pkg;
425 }
426
427 static x86_die_t *
x86_die_find(int cpu)428 x86_die_find(int cpu)
429 {
430 x86_die_t *die;
431 x86_pkg_t *pkg;
432 cpu_data_t *cpup;
433 uint32_t die_num;
434
435 cpup = cpu_datap(cpu);
436
437 die_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
438
439 pkg = x86_package_find(cpu);
440 if (pkg == NULL) {
441 return NULL;
442 }
443
444 die = pkg->dies;
445 while (die != NULL) {
446 if (die->pdie_num == die_num) {
447 break;
448 }
449 die = die->next_in_pkg;
450 }
451
452 return die;
453 }
454
455 static x86_core_t *
x86_core_find(int cpu)456 x86_core_find(int cpu)
457 {
458 x86_core_t *core;
459 x86_die_t *die;
460 cpu_data_t *cpup;
461 uint32_t core_num;
462
463 cpup = cpu_datap(cpu);
464
465 core_num = cpup->cpu_phys_number / topoParms.nPThreadsPerCore;
466
467 die = x86_die_find(cpu);
468 if (die == NULL) {
469 return NULL;
470 }
471
472 core = die->cores;
473 while (core != NULL) {
474 if (core->pcore_num == core_num) {
475 break;
476 }
477 core = core->next_in_die;
478 }
479
480 return core;
481 }
482
483 void
x86_set_logical_topology(x86_lcpu_t * lcpu,int pnum,int lnum)484 x86_set_logical_topology(x86_lcpu_t *lcpu, int pnum, int lnum)
485 {
486 x86_core_t *core = lcpu->core;
487 x86_die_t *die = lcpu->die;
488 x86_pkg_t *pkg = lcpu->package;
489
490 assert(core != NULL);
491 assert(die != NULL);
492 assert(pkg != NULL);
493
494 lcpu->cpu_num = lnum;
495 lcpu->pnum = pnum;
496 lcpu->master = (lnum == master_cpu);
497 lcpu->primary = (lnum % topoParms.nLThreadsPerPackage) == 0;
498
499 lcpu->lnum = lnum % topoParms.nLThreadsPerCore;
500
501 core->pcore_num = lnum / topoParms.nLThreadsPerCore;
502 core->lcore_num = core->pcore_num % topoParms.nLCoresPerDie;
503
504 die->pdie_num = lnum / (topoParms.nLThreadsPerCore * topoParms.nLCoresPerDie);
505 die->ldie_num = die->pdie_num % topoParms.nLDiesPerPackage;
506
507 pkg->ppkg_num = lnum / topoParms.nLThreadsPerPackage;
508 pkg->lpkg_num = pkg->ppkg_num;
509 }
510
511 static x86_die_t *
x86_die_alloc(int cpu)512 x86_die_alloc(int cpu)
513 {
514 x86_die_t *die;
515 cpu_data_t *cpup;
516
517 cpup = cpu_datap(cpu);
518
519 mp_safe_spin_lock(&x86_topo_lock);
520 if (free_dies != NULL) {
521 die = free_dies;
522 free_dies = die->next_in_pkg;
523 die->next_in_pkg = NULL;
524 simple_unlock(&x86_topo_lock);
525 } else {
526 simple_unlock(&x86_topo_lock);
527 die = zalloc_permanent_type(x86_die_t);
528 if (die == NULL) {
529 panic("x86_die_alloc() alloc of x86_die_t failed!");
530 }
531 }
532
533 die->pdie_num = cpup->cpu_phys_number / topoParms.nPThreadsPerDie;
534
535 die->ldie_num = num_dies;
536 atomic_incl((long *) &num_dies, 1);
537
538 die->flags = X86DIE_FL_PRESENT;
539 return die;
540 }
541
542 static void
x86_die_free(x86_die_t * die)543 x86_die_free(x86_die_t *die)
544 {
545 mp_safe_spin_lock(&x86_topo_lock);
546 die->next_in_pkg = free_dies;
547 free_dies = die;
548 atomic_decl((long *) &num_dies, 1);
549 simple_unlock(&x86_topo_lock);
550 }
551
552 static x86_pkg_t *
x86_package_alloc(int cpu)553 x86_package_alloc(int cpu)
554 {
555 x86_pkg_t *pkg;
556 cpu_data_t *cpup;
557
558 cpup = cpu_datap(cpu);
559
560 mp_safe_spin_lock(&x86_topo_lock);
561 if (free_pkgs != NULL) {
562 pkg = free_pkgs;
563 free_pkgs = pkg->next;
564 pkg->next = NULL;
565 simple_unlock(&x86_topo_lock);
566 } else {
567 simple_unlock(&x86_topo_lock);
568 pkg = zalloc_permanent_type(x86_pkg_t);
569 if (pkg == NULL) {
570 panic("x86_package_alloc() alloc of x86_pkg_t failed!");
571 }
572 }
573
574 pkg->ppkg_num = cpup->cpu_phys_number / topoParms.nPThreadsPerPackage;
575
576 pkg->lpkg_num = topoParms.nPackages;
577 atomic_incl((long *) &topoParms.nPackages, 1);
578
579 pkg->flags = X86PKG_FL_PRESENT | X86PKG_FL_READY;
580 return pkg;
581 }
582
583 static void
x86_package_free(x86_pkg_t * pkg)584 x86_package_free(x86_pkg_t *pkg)
585 {
586 mp_safe_spin_lock(&x86_topo_lock);
587 pkg->next = free_pkgs;
588 free_pkgs = pkg;
589 atomic_decl((long *) &topoParms.nPackages, 1);
590 simple_unlock(&x86_topo_lock);
591 }
592
593 static void
x86_cache_add_lcpu(x86_cpu_cache_t * cache,x86_lcpu_t * lcpu)594 x86_cache_add_lcpu(x86_cpu_cache_t *cache, x86_lcpu_t *lcpu)
595 {
596 x86_cpu_cache_t *cur_cache;
597 int i;
598
599 /*
600 * Put the new CPU into the list of the cache.
601 */
602 cur_cache = lcpu->caches[cache->level - 1];
603 lcpu->caches[cache->level - 1] = cache;
604 cache->next = cur_cache;
605 cache->nlcpus += 1;
606 for (i = 0; i < cache->nlcpus; i += 1) {
607 if (cache->cpus[i] == NULL) {
608 cache->cpus[i] = lcpu;
609 break;
610 }
611 }
612 }
613
614 static void
x86_lcpu_add_caches(x86_lcpu_t * lcpu)615 x86_lcpu_add_caches(x86_lcpu_t *lcpu)
616 {
617 x86_cpu_cache_t *list;
618 x86_cpu_cache_t *cur;
619 x86_cpu_cache_t *match;
620 x86_die_t *die;
621 x86_core_t *core;
622 x86_lcpu_t *cur_lcpu;
623 uint32_t level;
624 boolean_t found = FALSE;
625
626 assert(lcpu != NULL);
627
628 /*
629 * Add the cache data to the topology.
630 */
631 list = x86_cache_list();
632
633 mp_safe_spin_lock(&x86_topo_lock);
634
635 while (list != NULL) {
636 /*
637 * Remove the cache from the front of the list.
638 */
639 cur = list;
640 list = cur->next;
641 cur->next = NULL;
642 level = cur->level - 1;
643
644 /*
645 * If the cache isn't shared then just put it where it
646 * belongs.
647 */
648 if (cur->maxcpus == 1) {
649 x86_cache_add_lcpu(cur, lcpu);
650 continue;
651 }
652
653 /*
654 * We'll assume that all of the caches at a particular level
655 * have the same sharing. So if we have a cache already at
656 * this level, we'll just skip looking for the match.
657 */
658 if (lcpu->caches[level] != NULL) {
659 x86_cache_free(cur);
660 continue;
661 }
662
663 /*
664 * This is a shared cache, so we have to figure out if
665 * this is the first time we've seen this cache. We do
666 * this by searching through the topology and seeing if
667 * this cache is already described.
668 *
669 * Assume that L{LLC-1} are all at the core level and that
670 * LLC is shared at the die level.
671 */
672 if (level < topoParms.LLCDepth) {
673 /*
674 * Shared at the core.
675 */
676 core = lcpu->core;
677 cur_lcpu = core->lcpus;
678 while (cur_lcpu != NULL) {
679 /*
680 * Skip ourselves.
681 */
682 if (cur_lcpu == lcpu) {
683 cur_lcpu = cur_lcpu->next_in_core;
684 continue;
685 }
686
687 /*
688 * If there's a cache on this logical CPU,
689 * then use that one.
690 */
691 match = x86_match_cache(cur_lcpu->caches[level], cur);
692 if (match != NULL) {
693 x86_cache_free(cur);
694 x86_cache_add_lcpu(match, lcpu);
695 found = TRUE;
696 break;
697 }
698
699 cur_lcpu = cur_lcpu->next_in_core;
700 }
701 } else {
702 /*
703 * Shared at the die.
704 */
705 die = lcpu->die;
706 cur_lcpu = die->lcpus;
707 while (cur_lcpu != NULL) {
708 /*
709 * Skip ourselves.
710 */
711 if (cur_lcpu == lcpu) {
712 cur_lcpu = cur_lcpu->next_in_die;
713 continue;
714 }
715
716 /*
717 * If there's a cache on this logical CPU,
718 * then use that one.
719 */
720 match = x86_match_cache(cur_lcpu->caches[level], cur);
721 if (match != NULL) {
722 x86_cache_free(cur);
723 x86_cache_add_lcpu(match, lcpu);
724 found = TRUE;
725 break;
726 }
727
728 cur_lcpu = cur_lcpu->next_in_die;
729 }
730 }
731
732 /*
733 * If a shared cache wasn't found, then this logical CPU must
734 * be the first one encountered.
735 */
736 if (!found) {
737 x86_cache_add_lcpu(cur, lcpu);
738 }
739 }
740
741 simple_unlock(&x86_topo_lock);
742 }
743
744 static void
x86_core_add_lcpu(x86_core_t * core,x86_lcpu_t * lcpu)745 x86_core_add_lcpu(x86_core_t *core, x86_lcpu_t *lcpu)
746 {
747 assert(core != NULL);
748 assert(lcpu != NULL);
749
750 mp_safe_spin_lock(&x86_topo_lock);
751
752 lcpu->next_in_core = core->lcpus;
753 lcpu->core = core;
754 core->lcpus = lcpu;
755 core->num_lcpus += 1;
756 simple_unlock(&x86_topo_lock);
757 }
758
759 static void
x86_die_add_lcpu(x86_die_t * die,x86_lcpu_t * lcpu)760 x86_die_add_lcpu(x86_die_t *die, x86_lcpu_t *lcpu)
761 {
762 assert(die != NULL);
763 assert(lcpu != NULL);
764
765 lcpu->next_in_die = die->lcpus;
766 lcpu->die = die;
767 die->lcpus = lcpu;
768 }
769
770 static void
x86_die_add_core(x86_die_t * die,x86_core_t * core)771 x86_die_add_core(x86_die_t *die, x86_core_t *core)
772 {
773 assert(die != NULL);
774 assert(core != NULL);
775
776 core->next_in_die = die->cores;
777 core->die = die;
778 die->cores = core;
779 die->num_cores += 1;
780 }
781
782 static void
x86_package_add_lcpu(x86_pkg_t * pkg,x86_lcpu_t * lcpu)783 x86_package_add_lcpu(x86_pkg_t *pkg, x86_lcpu_t *lcpu)
784 {
785 assert(pkg != NULL);
786 assert(lcpu != NULL);
787
788 lcpu->next_in_pkg = pkg->lcpus;
789 lcpu->package = pkg;
790 pkg->lcpus = lcpu;
791 }
792
793 static void
x86_package_add_core(x86_pkg_t * pkg,x86_core_t * core)794 x86_package_add_core(x86_pkg_t *pkg, x86_core_t *core)
795 {
796 assert(pkg != NULL);
797 assert(core != NULL);
798
799 core->next_in_pkg = pkg->cores;
800 core->package = pkg;
801 pkg->cores = core;
802 }
803
804 static void
x86_package_add_die(x86_pkg_t * pkg,x86_die_t * die)805 x86_package_add_die(x86_pkg_t *pkg, x86_die_t *die)
806 {
807 assert(pkg != NULL);
808 assert(die != NULL);
809
810 die->next_in_pkg = pkg->dies;
811 die->package = pkg;
812 pkg->dies = die;
813 pkg->num_dies += 1;
814 }
815
816 void *
cpu_thread_alloc(int cpu)817 cpu_thread_alloc(int cpu)
818 {
819 x86_core_t *core = NULL;
820 x86_die_t *die = NULL;
821 x86_pkg_t *pkg = NULL;
822 cpu_data_t *cpup;
823 uint32_t phys_cpu;
824
825 /*
826 * Only allow one to manipulate the topology at a time.
827 */
828 mp_safe_spin_lock(&x86_topo_lock);
829
830 /*
831 * Make sure all of the topology parameters have been initialized.
832 */
833 if (!topoParmsInited) {
834 initTopoParms();
835 }
836
837 cpup = cpu_datap(cpu);
838
839 phys_cpu = cpup->cpu_phys_number;
840
841 x86_lcpu_init(cpu);
842
843 /*
844 * Assume that all cpus have the same features.
845 */
846 if (cpu_is_hyperthreaded()) {
847 cpup->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
848 } else {
849 cpup->cpu_threadtype = CPU_THREADTYPE_NONE;
850 }
851
852 /*
853 * Get the package that the logical CPU is in.
854 */
855 do {
856 pkg = x86_package_find(cpu);
857 if (pkg == NULL) {
858 /*
859 * Package structure hasn't been created yet, do it now.
860 */
861 simple_unlock(&x86_topo_lock);
862 pkg = x86_package_alloc(cpu);
863 mp_safe_spin_lock(&x86_topo_lock);
864 if (x86_package_find(cpu) != NULL) {
865 x86_package_free(pkg);
866 continue;
867 }
868
869 /*
870 * Add the new package to the global list of packages.
871 */
872 pkg->next = x86_pkgs;
873 x86_pkgs = pkg;
874 }
875 } while (pkg == NULL);
876
877 /*
878 * Get the die that the logical CPU is in.
879 */
880 do {
881 die = x86_die_find(cpu);
882 if (die == NULL) {
883 /*
884 * Die structure hasn't been created yet, do it now.
885 */
886 simple_unlock(&x86_topo_lock);
887 die = x86_die_alloc(cpu);
888 mp_safe_spin_lock(&x86_topo_lock);
889 if (x86_die_find(cpu) != NULL) {
890 x86_die_free(die);
891 continue;
892 }
893
894 /*
895 * Add the die to the package.
896 */
897 x86_package_add_die(pkg, die);
898 }
899 } while (die == NULL);
900
901 /*
902 * Get the core for this logical CPU.
903 */
904 do {
905 core = x86_core_find(cpu);
906 if (core == NULL) {
907 /*
908 * Allocate the core structure now.
909 */
910 simple_unlock(&x86_topo_lock);
911 core = x86_core_alloc(cpu);
912 mp_safe_spin_lock(&x86_topo_lock);
913 if (x86_core_find(cpu) != NULL) {
914 x86_core_free(core);
915 continue;
916 }
917
918 /*
919 * Add the core to the die & package.
920 */
921 x86_die_add_core(die, core);
922 x86_package_add_core(pkg, core);
923 machine_info.physical_cpu_max += 1;
924 }
925 } while (core == NULL);
926
927
928 /*
929 * Done manipulating the topology, so others can get in.
930 */
931 machine_info.logical_cpu_max += 1;
932 simple_unlock(&x86_topo_lock);
933
934 /*
935 * Add the logical CPU to the other topology structures.
936 */
937 x86_core_add_lcpu(core, &cpup->lcpu);
938 x86_die_add_lcpu(core->die, &cpup->lcpu);
939 x86_package_add_lcpu(core->package, &cpup->lcpu);
940 x86_lcpu_add_caches(&cpup->lcpu);
941
942 return (void *) core;
943 }
944
945 void
cpu_thread_init(void)946 cpu_thread_init(void)
947 {
948 int my_cpu = get_cpu_number();
949 cpu_data_t *cpup = current_cpu_datap();
950 x86_core_t *core;
951 static int initialized = 0;
952
953 /*
954 * If we're the boot processor, we do all of the initialization of
955 * the CPU topology infrastructure.
956 */
957 if (my_cpu == master_cpu && !initialized) {
958 simple_lock_init(&x86_topo_lock, 0);
959
960 /*
961 * Put this logical CPU into the physical CPU topology.
962 */
963 cpup->lcpu.core = cpu_thread_alloc(my_cpu);
964
965 initialized = 1;
966 }
967
968 /*
969 * Do the CPU accounting.
970 */
971 core = cpup->lcpu.core;
972 mp_safe_spin_lock(&x86_topo_lock);
973 machine_info.logical_cpu += 1;
974 if (core->active_lcpus == 0) {
975 machine_info.physical_cpu += 1;
976 }
977 core->active_lcpus += 1;
978 simple_unlock(&x86_topo_lock);
979
980 pmCPUMarkRunning(cpup);
981 timer_resync_deadlines();
982 }
983
984 /*
985 * Called for a cpu to halt permanently
986 * (as opposed to halting and expecting an interrupt to awaken it).
987 */
988 __attribute__((noreturn))
989 void
cpu_thread_halt(void)990 cpu_thread_halt(void)
991 {
992 x86_core_t *core;
993 cpu_data_t *cpup = current_cpu_datap();
994
995 mp_safe_spin_lock(&x86_topo_lock);
996 machine_info.logical_cpu -= 1;
997 core = cpup->lcpu.core;
998 core->active_lcpus -= 1;
999 if (core->active_lcpus == 0) {
1000 machine_info.physical_cpu -= 1;
1001 }
1002 simple_unlock(&x86_topo_lock);
1003
1004 /*
1005 * Let the power management code determine the best way to "stop"
1006 * the processor.
1007 */
1008 ml_set_interrupts_enabled(FALSE);
1009 while (1) {
1010 pmCPUHalt(PM_HALT_NORMAL);
1011 }
1012 /* NOT REACHED */
1013 }
1014
1015 /*
1016 * Validates that the topology was built correctly. Must be called only
1017 * after the complete topology is built and no other changes are being made.
1018 */
1019 void
x86_validate_topology(void)1020 x86_validate_topology(void)
1021 {
1022 x86_pkg_t *pkg;
1023 x86_die_t *die;
1024 x86_core_t *core;
1025 x86_lcpu_t *lcpu;
1026 uint32_t nDies;
1027 uint32_t nCores;
1028 uint32_t nCPUs;
1029
1030 if (topo_dbg) {
1031 debug_topology_print();
1032 }
1033
1034 /*
1035 * Called after processors are registered but before non-boot processors
1036 * are started:
1037 * - real_ncpus: number of registered processors driven from MADT
1038 * - max_ncpus: max number of processors that will be started
1039 */
1040 nCPUs = topoParms.nPackages * topoParms.nLThreadsPerPackage;
1041 if (nCPUs != real_ncpus) {
1042 panic("x86_validate_topology() %d threads but %d registered from MADT",
1043 nCPUs, real_ncpus);
1044 }
1045
1046 pkg = x86_pkgs;
1047 while (pkg != NULL) {
1048 /*
1049 * Make sure that the package has the correct number of dies.
1050 */
1051 nDies = 0;
1052 die = pkg->dies;
1053 while (die != NULL) {
1054 if (die->package == NULL) {
1055 panic("Die(%d)->package is NULL",
1056 die->pdie_num);
1057 }
1058 if (die->package != pkg) {
1059 panic("Die %d points to package %d, should be %d",
1060 die->pdie_num, die->package->lpkg_num, pkg->lpkg_num);
1061 }
1062
1063 TOPO_DBG("Die(%d)->package %d\n",
1064 die->pdie_num, pkg->lpkg_num);
1065
1066 /*
1067 * Make sure that the die has the correct number of cores.
1068 */
1069 TOPO_DBG("Die(%d)->cores: ", die->pdie_num);
1070 nCores = 0;
1071 core = die->cores;
1072 while (core != NULL) {
1073 if (core->die == NULL) {
1074 panic("Core(%d)->die is NULL",
1075 core->pcore_num);
1076 }
1077 if (core->die != die) {
1078 panic("Core %d points to die %d, should be %d",
1079 core->pcore_num, core->die->pdie_num, die->pdie_num);
1080 }
1081 nCores += 1;
1082 TOPO_DBG("%d ", core->pcore_num);
1083 core = core->next_in_die;
1084 }
1085 TOPO_DBG("\n");
1086
1087 if (nCores != topoParms.nLCoresPerDie) {
1088 panic("Should have %d Cores, but only found %d for Die %d",
1089 topoParms.nLCoresPerDie, nCores, die->pdie_num);
1090 }
1091
1092 /*
1093 * Make sure that the die has the correct number of CPUs.
1094 */
1095 TOPO_DBG("Die(%d)->lcpus: ", die->pdie_num);
1096 nCPUs = 0;
1097 lcpu = die->lcpus;
1098 while (lcpu != NULL) {
1099 if (lcpu->die == NULL) {
1100 panic("CPU(%d)->die is NULL",
1101 lcpu->cpu_num);
1102 }
1103 if (lcpu->die != die) {
1104 panic("CPU %d points to die %d, should be %d",
1105 lcpu->cpu_num, lcpu->die->pdie_num, die->pdie_num);
1106 }
1107 nCPUs += 1;
1108 TOPO_DBG("%d ", lcpu->cpu_num);
1109 lcpu = lcpu->next_in_die;
1110 }
1111 TOPO_DBG("\n");
1112
1113 if (nCPUs != topoParms.nLThreadsPerDie) {
1114 panic("Should have %d Threads, but only found %d for Die %d",
1115 topoParms.nLThreadsPerDie, nCPUs, die->pdie_num);
1116 }
1117
1118 nDies += 1;
1119 die = die->next_in_pkg;
1120 }
1121
1122 if (nDies != topoParms.nLDiesPerPackage) {
1123 panic("Should have %d Dies, but only found %d for package %d",
1124 topoParms.nLDiesPerPackage, nDies, pkg->lpkg_num);
1125 }
1126
1127 /*
1128 * Make sure that the package has the correct number of cores.
1129 */
1130 nCores = 0;
1131 core = pkg->cores;
1132 while (core != NULL) {
1133 if (core->package == NULL) {
1134 panic("Core(%d)->package is NULL",
1135 core->pcore_num);
1136 }
1137 if (core->package != pkg) {
1138 panic("Core %d points to package %d, should be %d",
1139 core->pcore_num, core->package->lpkg_num, pkg->lpkg_num);
1140 }
1141 TOPO_DBG("Core(%d)->package %d\n",
1142 core->pcore_num, pkg->lpkg_num);
1143
1144 /*
1145 * Make sure that the core has the correct number of CPUs.
1146 */
1147 nCPUs = 0;
1148 lcpu = core->lcpus;
1149 TOPO_DBG("Core(%d)->lcpus: ", core->pcore_num);
1150 while (lcpu != NULL) {
1151 if (lcpu->core == NULL) {
1152 panic("CPU(%d)->core is NULL",
1153 lcpu->cpu_num);
1154 }
1155 if (lcpu->core != core) {
1156 panic("CPU %d points to core %d, should be %d",
1157 lcpu->cpu_num, lcpu->core->pcore_num, core->pcore_num);
1158 }
1159 TOPO_DBG("%d ", lcpu->cpu_num);
1160 nCPUs += 1;
1161 lcpu = lcpu->next_in_core;
1162 }
1163 TOPO_DBG("\n");
1164
1165 if (nCPUs != topoParms.nLThreadsPerCore) {
1166 panic("Should have %d Threads, but only found %d for Core %d",
1167 topoParms.nLThreadsPerCore, nCPUs, core->pcore_num);
1168 }
1169 nCores += 1;
1170 core = core->next_in_pkg;
1171 }
1172
1173 if (nCores != topoParms.nLCoresPerPackage) {
1174 panic("Should have %d Cores, but only found %d for package %d",
1175 topoParms.nLCoresPerPackage, nCores, pkg->lpkg_num);
1176 }
1177
1178 /*
1179 * Make sure that the package has the correct number of CPUs.
1180 */
1181 nCPUs = 0;
1182 lcpu = pkg->lcpus;
1183 while (lcpu != NULL) {
1184 if (lcpu->package == NULL) {
1185 panic("CPU(%d)->package is NULL",
1186 lcpu->cpu_num);
1187 }
1188 if (lcpu->package != pkg) {
1189 panic("CPU %d points to package %d, should be %d",
1190 lcpu->cpu_num, lcpu->package->lpkg_num, pkg->lpkg_num);
1191 }
1192 TOPO_DBG("CPU(%d)->package %d\n",
1193 lcpu->cpu_num, pkg->lpkg_num);
1194 nCPUs += 1;
1195 lcpu = lcpu->next_in_pkg;
1196 }
1197
1198 if (nCPUs != topoParms.nLThreadsPerPackage) {
1199 panic("Should have %d Threads, but only found %d for package %d",
1200 topoParms.nLThreadsPerPackage, nCPUs, pkg->lpkg_num);
1201 }
1202
1203 pkg = pkg->next;
1204 }
1205 }
1206
1207 /*
1208 * Prints out the topology
1209 */
1210 static void
debug_topology_print(void)1211 debug_topology_print(void)
1212 {
1213 x86_pkg_t *pkg;
1214 x86_die_t *die;
1215 x86_core_t *core;
1216 x86_lcpu_t *cpu;
1217
1218 pkg = x86_pkgs;
1219 while (pkg != NULL) {
1220 kprintf("Package:\n");
1221 kprintf(" Physical: %d\n", pkg->ppkg_num);
1222 kprintf(" Logical: %d\n", pkg->lpkg_num);
1223
1224 die = pkg->dies;
1225 while (die != NULL) {
1226 kprintf(" Die:\n");
1227 kprintf(" Physical: %d\n", die->pdie_num);
1228 kprintf(" Logical: %d\n", die->ldie_num);
1229
1230 core = die->cores;
1231 while (core != NULL) {
1232 kprintf(" Core:\n");
1233 kprintf(" Physical: %d\n", core->pcore_num);
1234 kprintf(" Logical: %d\n", core->lcore_num);
1235
1236 cpu = core->lcpus;
1237 while (cpu != NULL) {
1238 kprintf(" LCPU:\n");
1239 kprintf(" CPU #: %d\n", cpu->cpu_num);
1240 kprintf(" Physical: %d\n", cpu->pnum);
1241 kprintf(" Logical: %d\n", cpu->lnum);
1242 kprintf(" Flags: ");
1243 if (cpu->master) {
1244 kprintf("MASTER ");
1245 }
1246 if (cpu->primary) {
1247 kprintf("PRIMARY");
1248 }
1249 if (!cpu->master && !cpu->primary) {
1250 kprintf("(NONE)");
1251 }
1252 kprintf("\n");
1253
1254 cpu = cpu->next_in_core;
1255 }
1256
1257 core = core->next_in_die;
1258 }
1259
1260 die = die->next_in_pkg;
1261 }
1262
1263 pkg = pkg->next;
1264 }
1265 }
1266