1 /*
2 * Copyright (c) 2000-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 #include <vm/vm_page.h>
32 #include <pexpert/pexpert.h>
33
34 #include <i386/cpu_threads.h>
35 #include <i386/cpuid.h>
36 #include <i386/machine_routines.h>
37
38 int force_tecs_at_idle;
39 int tecs_mode_supported;
40
41 static boolean_t cpuid_dbg
42 #if DEBUG
43 = TRUE;
44 #else
45 = FALSE;
46 #endif
47 #define DBG(x...) \
48 do { \
49 if (cpuid_dbg) \
50 kprintf(x); \
51 } while (0) \
52
53 #define min(a, b) ((a) < (b) ? (a) : (b))
54 #define quad(hi, lo) (((uint64_t)(hi)) << 32 | (lo))
55
56 /*
57 * Leaf 2 cache descriptor encodings.
58 */
59 typedef enum {
60 _NULL_, /* NULL (empty) descriptor */
61 CACHE, /* Cache */
62 TLB, /* TLB */
63 STLB, /* Shared second-level unified TLB */
64 PREFETCH /* Prefetch size */
65 } cpuid_leaf2_desc_type_t;
66
67 typedef enum {
68 NA, /* Not Applicable */
69 FULLY, /* Fully-associative */
70 TRACE, /* Trace Cache (P4 only) */
71 INST, /* Instruction TLB */
72 DATA, /* Data TLB */
73 DATA0, /* Data TLB, 1st level */
74 DATA1, /* Data TLB, 2nd level */
75 L1, /* L1 (unified) cache */
76 L1_INST, /* L1 Instruction cache */
77 L1_DATA, /* L1 Data cache */
78 L2, /* L2 (unified) cache */
79 L3, /* L3 (unified) cache */
80 L2_2LINESECTOR, /* L2 (unified) cache with 2 lines per sector */
81 L3_2LINESECTOR, /* L3(unified) cache with 2 lines per sector */
82 SMALL, /* Small page TLB */
83 LARGE, /* Large page TLB */
84 BOTH /* Small and Large page TLB */
85 } cpuid_leaf2_qualifier_t;
86
87 typedef struct cpuid_cache_descriptor {
88 uint8_t value; /* descriptor code */
89 uint8_t type; /* cpuid_leaf2_desc_type_t */
90 uint8_t level; /* level of cache/TLB hierachy */
91 uint8_t ways; /* wayness of cache */
92 uint16_t size; /* cachesize or TLB pagesize */
93 uint16_t entries; /* number of TLB entries or linesize */
94 } cpuid_cache_descriptor_t;
95
96 /*
97 * These multipliers are used to encode 1*K .. 64*M in a 16 bit size field
98 */
99 #define K (1)
100 #define M (1024)
101
102 /*
103 * Intel cache descriptor table:
104 */
105 static const cpuid_cache_descriptor_t intel_cpuid_leaf2_descriptor_table[] = {
106 // -------------------------------------------------------
107 // value type level ways size entries
108 // -------------------------------------------------------
109 { 0x00, _NULL_, NA, NA, NA, NA },
110 { 0x01, TLB, INST, 4, SMALL, 32 },
111 { 0x02, TLB, INST, FULLY, LARGE, 2 },
112 { 0x03, TLB, DATA, 4, SMALL, 64 },
113 { 0x04, TLB, DATA, 4, LARGE, 8 },
114 { 0x05, TLB, DATA1, 4, LARGE, 32 },
115 { 0x06, CACHE, L1_INST, 4, 8 * K, 32 },
116 { 0x08, CACHE, L1_INST, 4, 16 * K, 32 },
117 { 0x09, CACHE, L1_INST, 4, 32 * K, 64 },
118 { 0x0A, CACHE, L1_DATA, 2, 8 * K, 32 },
119 { 0x0B, TLB, INST, 4, LARGE, 4 },
120 { 0x0C, CACHE, L1_DATA, 4, 16 * K, 32 },
121 { 0x0D, CACHE, L1_DATA, 4, 16 * K, 64 },
122 { 0x0E, CACHE, L1_DATA, 6, 24 * K, 64 },
123 { 0x21, CACHE, L2, 8, 256 * K, 64 },
124 { 0x22, CACHE, L3_2LINESECTOR, 4, 512 * K, 64 },
125 { 0x23, CACHE, L3_2LINESECTOR, 8, 1 * M, 64 },
126 { 0x25, CACHE, L3_2LINESECTOR, 8, 2 * M, 64 },
127 { 0x29, CACHE, L3_2LINESECTOR, 8, 4 * M, 64 },
128 { 0x2C, CACHE, L1_DATA, 8, 32 * K, 64 },
129 { 0x30, CACHE, L1_INST, 8, 32 * K, 64 },
130 { 0x40, CACHE, L2, NA, 0, NA },
131 { 0x41, CACHE, L2, 4, 128 * K, 32 },
132 { 0x42, CACHE, L2, 4, 256 * K, 32 },
133 { 0x43, CACHE, L2, 4, 512 * K, 32 },
134 { 0x44, CACHE, L2, 4, 1 * M, 32 },
135 { 0x45, CACHE, L2, 4, 2 * M, 32 },
136 { 0x46, CACHE, L3, 4, 4 * M, 64 },
137 { 0x47, CACHE, L3, 8, 8 * M, 64 },
138 { 0x48, CACHE, L2, 12, 3 * M, 64 },
139 { 0x49, CACHE, L2, 16, 4 * M, 64 },
140 { 0x4A, CACHE, L3, 12, 6 * M, 64 },
141 { 0x4B, CACHE, L3, 16, 8 * M, 64 },
142 { 0x4C, CACHE, L3, 12, 12 * M, 64 },
143 { 0x4D, CACHE, L3, 16, 16 * M, 64 },
144 { 0x4E, CACHE, L2, 24, 6 * M, 64 },
145 { 0x4F, TLB, INST, NA, SMALL, 32 },
146 { 0x50, TLB, INST, NA, BOTH, 64 },
147 { 0x51, TLB, INST, NA, BOTH, 128 },
148 { 0x52, TLB, INST, NA, BOTH, 256 },
149 { 0x55, TLB, INST, FULLY, BOTH, 7 },
150 { 0x56, TLB, DATA0, 4, LARGE, 16 },
151 { 0x57, TLB, DATA0, 4, SMALL, 16 },
152 { 0x59, TLB, DATA0, FULLY, SMALL, 16 },
153 { 0x5A, TLB, DATA0, 4, LARGE, 32 },
154 { 0x5B, TLB, DATA, NA, BOTH, 64 },
155 { 0x5C, TLB, DATA, NA, BOTH, 128 },
156 { 0x5D, TLB, DATA, NA, BOTH, 256 },
157 { 0x60, CACHE, L1, 16 * K, 8, 64 },
158 { 0x61, CACHE, L1, 4, 8 * K, 64 },
159 { 0x62, CACHE, L1, 4, 16 * K, 64 },
160 { 0x63, CACHE, L1, 4, 32 * K, 64 },
161 { 0x70, CACHE, TRACE, 8, 12 * K, NA },
162 { 0x71, CACHE, TRACE, 8, 16 * K, NA },
163 { 0x72, CACHE, TRACE, 8, 32 * K, NA },
164 { 0x76, TLB, INST, NA, BOTH, 8 },
165 { 0x78, CACHE, L2, 4, 1 * M, 64 },
166 { 0x79, CACHE, L2_2LINESECTOR, 8, 128 * K, 64 },
167 { 0x7A, CACHE, L2_2LINESECTOR, 8, 256 * K, 64 },
168 { 0x7B, CACHE, L2_2LINESECTOR, 8, 512 * K, 64 },
169 { 0x7C, CACHE, L2_2LINESECTOR, 8, 1 * M, 64 },
170 { 0x7D, CACHE, L2, 8, 2 * M, 64 },
171 { 0x7F, CACHE, L2, 2, 512 * K, 64 },
172 { 0x80, CACHE, L2, 8, 512 * K, 64 },
173 { 0x82, CACHE, L2, 8, 256 * K, 32 },
174 { 0x83, CACHE, L2, 8, 512 * K, 32 },
175 { 0x84, CACHE, L2, 8, 1 * M, 32 },
176 { 0x85, CACHE, L2, 8, 2 * M, 32 },
177 { 0x86, CACHE, L2, 4, 512 * K, 64 },
178 { 0x87, CACHE, L2, 8, 1 * M, 64 },
179 { 0xB0, TLB, INST, 4, SMALL, 128 },
180 { 0xB1, TLB, INST, 4, LARGE, 8 },
181 { 0xB2, TLB, INST, 4, SMALL, 64 },
182 { 0xB3, TLB, DATA, 4, SMALL, 128 },
183 { 0xB4, TLB, DATA1, 4, SMALL, 256 },
184 { 0xB5, TLB, DATA1, 8, SMALL, 64 },
185 { 0xB6, TLB, DATA1, 8, SMALL, 128 },
186 { 0xBA, TLB, DATA1, 4, BOTH, 64 },
187 { 0xC1, STLB, DATA1, 8, SMALL, 1024},
188 { 0xCA, STLB, DATA1, 4, SMALL, 512 },
189 { 0xD0, CACHE, L3, 4, 512 * K, 64 },
190 { 0xD1, CACHE, L3, 4, 1 * M, 64 },
191 { 0xD2, CACHE, L3, 4, 2 * M, 64 },
192 { 0xD3, CACHE, L3, 4, 4 * M, 64 },
193 { 0xD4, CACHE, L3, 4, 8 * M, 64 },
194 { 0xD6, CACHE, L3, 8, 1 * M, 64 },
195 { 0xD7, CACHE, L3, 8, 2 * M, 64 },
196 { 0xD8, CACHE, L3, 8, 4 * M, 64 },
197 { 0xD9, CACHE, L3, 8, 8 * M, 64 },
198 { 0xDA, CACHE, L3, 8, 12 * M, 64 },
199 { 0xDC, CACHE, L3, 12, 1536 * K, 64 },
200 { 0xDD, CACHE, L3, 12, 3 * M, 64 },
201 { 0xDE, CACHE, L3, 12, 6 * M, 64 },
202 { 0xDF, CACHE, L3, 12, 12 * M, 64 },
203 { 0xE0, CACHE, L3, 12, 18 * M, 64 },
204 { 0xE2, CACHE, L3, 16, 2 * M, 64 },
205 { 0xE3, CACHE, L3, 16, 4 * M, 64 },
206 { 0xE4, CACHE, L3, 16, 8 * M, 64 },
207 { 0xE5, CACHE, L3, 16, 16 * M, 64 },
208 { 0xE6, CACHE, L3, 16, 24 * M, 64 },
209 { 0xF0, PREFETCH, NA, NA, 64, NA },
210 { 0xF1, PREFETCH, NA, NA, 128, NA },
211 { 0xFF, CACHE, NA, NA, 0, NA }
212 };
213 #define INTEL_LEAF2_DESC_NUM (sizeof(intel_cpuid_leaf2_descriptor_table) / \
214 sizeof(cpuid_cache_descriptor_t))
215
216 boolean_t cpuid_tsx_disabled = false; /* true if XNU disabled TSX */
217 boolean_t cpuid_tsx_supported = false;
218
219 static void do_cwas(i386_cpu_info_t *cpuinfo, boolean_t on_slave);
220 static void cpuid_do_precpuid_was(void);
221
222 static void cpuid_vmm_detect_pv_interface(i386_vmm_info_t *info_p, const char *signature,
223 bool (*)(i386_vmm_info_t*, const uint32_t, const uint32_t));
224 static bool cpuid_vmm_detect_applepv_features(i386_vmm_info_t *info_p, const uint32_t base, const uint32_t max_leaf);
225
226 static inline const cpuid_cache_descriptor_t *
cpuid_leaf2_find(uint8_t value)227 cpuid_leaf2_find(uint8_t value)
228 {
229 unsigned int i;
230
231 for (i = 0; i < INTEL_LEAF2_DESC_NUM; i++) {
232 if (intel_cpuid_leaf2_descriptor_table[i].value == value) {
233 return &intel_cpuid_leaf2_descriptor_table[i];
234 }
235 }
236 return NULL;
237 }
238
239 /*
240 * CPU identification routines.
241 */
242
243 static i386_cpu_info_t cpuid_cpu_info;
244 static i386_cpu_info_t *cpuid_cpu_infop = NULL;
245
246 static void
cpuid_fn(uint32_t selector,uint32_t * result)247 cpuid_fn(uint32_t selector, uint32_t *result)
248 {
249 do_cpuid(selector, result);
250 DBG("cpuid_fn(0x%08x) eax:0x%08x ebx:0x%08x ecx:0x%08x edx:0x%08x\n",
251 selector, result[0], result[1], result[2], result[3]);
252 }
253
254 static const char *cache_type_str[LCACHE_MAX] = {
255 "Lnone", "L1I", "L1D", "L2U", "L3U"
256 };
257
258 static void
do_cwas(i386_cpu_info_t * cpuinfo,boolean_t on_slave)259 do_cwas(i386_cpu_info_t *cpuinfo, boolean_t on_slave)
260 {
261 extern int force_thread_policy_tecs;
262 cwa_classifier_e wa_reqd;
263
264 /*
265 * Workaround for reclaiming perf counter 3 due to TSX memory ordering erratum.
266 * This workaround does not support being forcibly set (since an MSR must be
267 * enumerated, lest we #GP when forced to access it.)
268 *
269 * Note that if disabling TSX is supported, disablement is prefered over forcing
270 * TSX transactions to abort.
271 */
272 if (cpuid_wa_required(CPU_INTEL_TSXDA) == CWA_ON) {
273 /* This must be executed on all logical processors */
274 wrmsr64(MSR_IA32_TSX_CTRL, MSR_IA32_TSXCTRL_TSX_CPU_CLEAR | MSR_IA32_TSXCTRL_RTM_DISABLE);
275 } else if (cpuid_wa_required(CPU_INTEL_TSXFA) == CWA_ON) {
276 /* This must be executed on all logical processors */
277 wrmsr64(MSR_IA32_TSX_FORCE_ABORT,
278 rdmsr64(MSR_IA32_TSX_FORCE_ABORT) | MSR_IA32_TSXFA_RTM_FORCE_ABORT);
279 }
280
281 if (((wa_reqd = cpuid_wa_required(CPU_INTEL_SRBDS)) & CWA_ON) != 0 &&
282 ((wa_reqd & CWA_FORCE_ON) == CWA_ON ||
283 (cpuinfo->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_SRBDS_CTRL) != 0)) {
284 /* This must be executed on all logical processors */
285 uint64_t mcuoptctrl = rdmsr64(MSR_IA32_MCU_OPT_CTRL);
286 mcuoptctrl |= MSR_IA32_MCUOPTCTRL_RNGDS_MITG_DIS;
287 wrmsr64(MSR_IA32_MCU_OPT_CTRL, mcuoptctrl);
288 }
289
290 if (on_slave) {
291 return;
292 }
293
294 switch (cpuid_wa_required(CPU_INTEL_SEGCHK)) {
295 case CWA_FORCE_ON:
296 force_thread_policy_tecs = 1;
297
298 /* If hyperthreaded, enable idle workaround */
299 if (cpuinfo->thread_count > cpuinfo->core_count) {
300 force_tecs_at_idle = 1;
301 }
302
303 OS_FALLTHROUGH;
304 case CWA_ON:
305 tecs_mode_supported = 1;
306 break;
307
308 case CWA_FORCE_OFF:
309 case CWA_OFF:
310 tecs_mode_supported = 0;
311 force_tecs_at_idle = 0;
312 force_thread_policy_tecs = 0;
313 break;
314
315 default:
316 break;
317 }
318 }
319
320 void
cpuid_do_was(void)321 cpuid_do_was(void)
322 {
323 do_cwas(cpuid_info(), TRUE);
324 }
325
326
327 /* this function is Intel-specific */
328 static void
cpuid_set_cache_info(i386_cpu_info_t * info_p)329 cpuid_set_cache_info( i386_cpu_info_t * info_p )
330 {
331 uint32_t cpuid_result[4];
332 uint32_t reg[4];
333 uint32_t index;
334 uint32_t linesizes[LCACHE_MAX];
335 unsigned int i;
336 unsigned int j;
337 boolean_t cpuid_deterministic_supported = FALSE;
338 unsigned int dcnt = 0;
339
340 DBG("cpuid_set_cache_info(%p)\n", info_p);
341
342 bzero( linesizes, sizeof(linesizes));
343
344 /* Get processor cache descriptor info using leaf 2. We don't use
345 * this internally, but must publish it for KEXTs.
346 */
347 for (i = 0; i < sizeof(info_p->cache_info) / 16; i++) {
348 /* byte 0 gives number of cpuid calls to get all descriptors */
349 if (i > 0 && i >= info_p->cache_info[0]) {
350 break;
351 }
352
353 cpuid_fn(2, cpuid_result);
354 for (j = 0; j < 4; j++) {
355 if ((cpuid_result[j] >> 31) == 1) {
356 continue;
357 }
358 memcpy(&info_p->cache_info[dcnt], &cpuid_result[j], 4);
359 dcnt += 4;
360 }
361 }
362
363 /*
364 * Get cache info using leaf 4, the "deterministic cache parameters."
365 * Most processors Mac OS X supports implement this flavor of CPUID.
366 * Loop over each cache on the processor.
367 */
368 cpuid_fn(0, cpuid_result);
369 if (cpuid_result[eax] >= 4) {
370 cpuid_deterministic_supported = TRUE;
371 }
372
373 for (index = 0; cpuid_deterministic_supported; index++) {
374 cache_type_t type = Lnone;
375 uint32_t cache_type;
376 uint32_t cache_level;
377 uint32_t cache_sharing;
378 uint32_t cache_linesize;
379 uint32_t cache_sets;
380 uint32_t cache_associativity;
381 uint32_t cache_size;
382 uint32_t cache_partitions;
383 uint32_t colors;
384
385 reg[eax] = 4; /* cpuid request 4 */
386 reg[ecx] = index; /* index starting at 0 */
387 cpuid(reg);
388 DBG("cpuid(4) index=%d eax=0x%x\n", index, reg[eax]);
389 cache_type = bitfield32(reg[eax], 4, 0);
390 if (cache_type == 0) {
391 break; /* no more caches */
392 }
393 cache_level = bitfield32(reg[eax], 7, 5);
394 cache_sharing = bitfield32(reg[eax], 25, 14) + 1;
395 info_p->cpuid_cores_per_package
396 = bitfield32(reg[eax], 31, 26) + 1;
397 cache_linesize = bitfield32(reg[ebx], 11, 0) + 1;
398 cache_partitions = bitfield32(reg[ebx], 21, 12) + 1;
399 cache_associativity = bitfield32(reg[ebx], 31, 22) + 1;
400 cache_sets = bitfield32(reg[ecx], 31, 0) + 1;
401
402 /* Map type/levels returned by CPUID into cache_type_t */
403 switch (cache_level) {
404 case 1:
405 type = cache_type == 1 ? L1D :
406 cache_type == 2 ? L1I :
407 Lnone;
408 break;
409 case 2:
410 type = cache_type == 3 ? L2U :
411 Lnone;
412 break;
413 case 3:
414 type = cache_type == 3 ? L3U :
415 Lnone;
416 break;
417 default:
418 type = Lnone;
419 }
420
421 /* The total size of a cache is:
422 * ( linesize * sets * associativity * partitions )
423 */
424 if (type != Lnone) {
425 cache_size = cache_linesize * cache_sets *
426 cache_associativity * cache_partitions;
427 info_p->cache_size[type] = cache_size;
428 info_p->cache_sharing[type] = cache_sharing;
429 info_p->cache_partitions[type] = cache_partitions;
430 linesizes[type] = cache_linesize;
431
432 DBG(" cache_size[%s] : %d\n",
433 cache_type_str[type], cache_size);
434 DBG(" cache_sharing[%s] : %d\n",
435 cache_type_str[type], cache_sharing);
436 DBG(" cache_partitions[%s]: %d\n",
437 cache_type_str[type], cache_partitions);
438
439 /*
440 * Overwrite associativity determined via
441 * CPUID.0x80000006 -- this leaf is more
442 * accurate
443 */
444 if (type == L2U) {
445 info_p->cpuid_cache_L2_associativity = cache_associativity;
446 }
447 /*
448 * Adjust #sets to account for the N CBos
449 * This is because addresses are hashed across CBos
450 */
451 if (type == L3U && info_p->core_count) {
452 cache_sets = cache_sets / info_p->core_count;
453 }
454
455 /* Compute the number of page colors for this cache,
456 * which is:
457 * ( linesize * sets ) / page_size
458 *
459 * To help visualize this, consider two views of a
460 * physical address. To the cache, it is composed
461 * of a line offset, a set selector, and a tag.
462 * To VM, it is composed of a page offset, a page
463 * color, and other bits in the pageframe number:
464 *
465 * +-----------------+---------+--------+
466 * cache: | tag | set | offset |
467 * +-----------------+---------+--------+
468 *
469 * +-----------------+-------+----------+
470 * VM: | don't care | color | pg offset|
471 * +-----------------+-------+----------+
472 *
473 * The color is those bits in (set+offset) not covered
474 * by the page offset.
475 */
476 colors = (cache_linesize * cache_sets) >> 12;
477
478 if (colors > vm_cache_geometry_colors) {
479 vm_cache_geometry_colors = colors;
480 }
481 }
482 }
483 DBG(" vm_cache_geometry_colors: %d\n", vm_cache_geometry_colors);
484
485 /*
486 * If deterministic cache parameters are not available, use
487 * something else
488 */
489 if (info_p->cpuid_cores_per_package == 0) {
490 info_p->cpuid_cores_per_package = 1;
491
492 /* cpuid define in 1024 quantities */
493 info_p->cache_size[L2U] = info_p->cpuid_cache_size * 1024;
494 info_p->cache_sharing[L2U] = 1;
495 info_p->cache_partitions[L2U] = 1;
496
497 linesizes[L2U] = info_p->cpuid_cache_linesize;
498
499 DBG(" cache_size[L2U] : %d\n",
500 info_p->cache_size[L2U]);
501 DBG(" cache_sharing[L2U] : 1\n");
502 DBG(" cache_partitions[L2U]: 1\n");
503 DBG(" linesizes[L2U] : %d\n",
504 info_p->cpuid_cache_linesize);
505 }
506
507 /*
508 * What linesize to publish? We use the L2 linesize if any,
509 * else the L1D.
510 */
511 if (linesizes[L2U]) {
512 info_p->cache_linesize = linesizes[L2U];
513 } else if (linesizes[L1D]) {
514 info_p->cache_linesize = linesizes[L1D];
515 } else {
516 panic("no linesize");
517 }
518 DBG(" cache_linesize : %d\n", info_p->cache_linesize);
519
520 /*
521 * Extract and publish TLB information from Leaf 2 descriptors.
522 */
523 DBG(" %ld leaf2 descriptors:\n", sizeof(info_p->cache_info));
524 for (i = 1; i < sizeof(info_p->cache_info); i++) {
525 const cpuid_cache_descriptor_t *descp;
526 int id;
527 int level;
528 int page;
529
530 DBG(" 0x%02x", info_p->cache_info[i]);
531 descp = cpuid_leaf2_find(info_p->cache_info[i]);
532 if (descp == NULL) {
533 continue;
534 }
535
536 switch (descp->type) {
537 case TLB:
538 page = (descp->size == SMALL) ? TLB_SMALL : TLB_LARGE;
539 /* determine I or D: */
540 switch (descp->level) {
541 case INST:
542 id = TLB_INST;
543 break;
544 case DATA:
545 case DATA0:
546 case DATA1:
547 id = TLB_DATA;
548 break;
549 default:
550 continue;
551 }
552 /* determine level: */
553 switch (descp->level) {
554 case DATA1:
555 level = 1;
556 break;
557 default:
558 level = 0;
559 }
560 info_p->cpuid_tlb[id][page][level] = descp->entries;
561 break;
562 case STLB:
563 info_p->cpuid_stlb = descp->entries;
564 }
565 }
566 DBG("\n");
567 }
568
569 static void
cpuid_set_generic_info(i386_cpu_info_t * info_p)570 cpuid_set_generic_info(i386_cpu_info_t *info_p)
571 {
572 uint32_t reg[4];
573 char str[128], *p;
574
575 DBG("cpuid_set_generic_info(%p)\n", info_p);
576
577 /* do cpuid 0 to get vendor */
578 cpuid_fn(0, reg);
579 info_p->cpuid_max_basic = reg[eax];
580 bcopy((char *)®[ebx], &info_p->cpuid_vendor[0], 4); /* ug */
581 bcopy((char *)®[ecx], &info_p->cpuid_vendor[8], 4);
582 bcopy((char *)®[edx], &info_p->cpuid_vendor[4], 4);
583 info_p->cpuid_vendor[12] = 0;
584
585 /* get extended cpuid results */
586 cpuid_fn(0x80000000, reg);
587 info_p->cpuid_max_ext = reg[eax];
588
589 /* check to see if we can get brand string */
590 if (info_p->cpuid_max_ext >= 0x80000004) {
591 /*
592 * The brand string 48 bytes (max), guaranteed to
593 * be NUL terminated.
594 */
595 cpuid_fn(0x80000002, reg);
596 bcopy((char *)reg, &str[0], 16);
597 cpuid_fn(0x80000003, reg);
598 bcopy((char *)reg, &str[16], 16);
599 cpuid_fn(0x80000004, reg);
600 bcopy((char *)reg, &str[32], 16);
601 for (p = str; *p != '\0'; p++) {
602 if (*p != ' ') {
603 break;
604 }
605 }
606 strlcpy(info_p->cpuid_brand_string,
607 p, sizeof(info_p->cpuid_brand_string));
608
609 if (!strncmp(info_p->cpuid_brand_string, CPUID_STRING_UNKNOWN,
610 min(sizeof(info_p->cpuid_brand_string),
611 strlen(CPUID_STRING_UNKNOWN) + 1))) {
612 /*
613 * This string means we have a firmware-programmable brand string,
614 * and the firmware couldn't figure out what sort of CPU we have.
615 */
616 info_p->cpuid_brand_string[0] = '\0';
617 }
618 }
619
620 /* Get cache and addressing info. */
621 if (info_p->cpuid_max_ext >= 0x80000006) {
622 cpuid_fn(0x80000006, reg);
623 info_p->cpuid_cache_linesize = bitfield32(reg[ecx], 7, 0);
624 DBG(" cpuid_cache_linesize: %d\n",
625 info_p->cpuid_cache_linesize);
626
627 uint32_t assoc = bitfield32(reg[ecx], 15, 12);
628 /*
629 * L2 associativity is encoded, though in an insufficiently
630 * descriptive fashion, e.g. 24-way is mapped to 16-way.
631 * Represent a fully associative cache as 0xFFFF.
632 * Overwritten by associativity as determined via CPUID.4
633 * if available.
634 */
635 if (assoc == 6) {
636 assoc = 8;
637 } else if (assoc == 8) {
638 assoc = 16;
639 } else if (assoc == 0xF) {
640 assoc = 0xFFFF;
641 }
642 info_p->cpuid_cache_L2_associativity = assoc;
643 info_p->cpuid_cache_size = bitfield32(reg[ecx], 31, 16);
644 DBG(" cpuid_cache_size : %dKiB\n",
645 info_p->cpuid_cache_size);
646
647 cpuid_fn(0x80000008, reg);
648 info_p->cpuid_address_bits_physical =
649 bitfield32(reg[eax], 7, 0);
650 info_p->cpuid_address_bits_virtual =
651 bitfield32(reg[eax], 15, 8);
652 }
653
654 /*
655 * Get processor signature and decode
656 * and bracket this with the approved procedure for reading the
657 * the microcode version number a.k.a. signature a.k.a. BIOS ID
658 */
659 wrmsr64(MSR_IA32_BIOS_SIGN_ID, 0);
660 cpuid_fn(1, reg);
661 info_p->cpuid_microcode_version =
662 (uint32_t) (rdmsr64(MSR_IA32_BIOS_SIGN_ID) >> 32);
663 info_p->cpuid_signature = reg[eax];
664 info_p->cpuid_stepping = bitfield32(reg[eax], 3, 0);
665 info_p->cpuid_model = bitfield32(reg[eax], 7, 4);
666 info_p->cpuid_family = bitfield32(reg[eax], 11, 8);
667 info_p->cpuid_type = bitfield32(reg[eax], 13, 12);
668 info_p->cpuid_extmodel = bitfield32(reg[eax], 19, 16);
669 info_p->cpuid_extfamily = bitfield32(reg[eax], 27, 20);
670 info_p->cpuid_brand = bitfield32(reg[ebx], 7, 0);
671 info_p->cpuid_features = quad(reg[ecx], reg[edx]);
672
673 /* Get "processor flag"; necessary for microcode update matching */
674 info_p->cpuid_processor_flag = (rdmsr64(MSR_IA32_PLATFORM_ID) >> 50) & 0x7;
675
676 /* Fold extensions into family/model */
677 if (info_p->cpuid_family == 0x0f || info_p->cpuid_family == 0x06) {
678 info_p->cpuid_model += (info_p->cpuid_extmodel << 4);
679 if (info_p->cpuid_family == 0x0f) {
680 info_p->cpuid_family += info_p->cpuid_extfamily;
681 }
682 }
683
684 if (info_p->cpuid_features & CPUID_FEATURE_HTT) {
685 info_p->cpuid_logical_per_package =
686 bitfield32(reg[ebx], 23, 16);
687 } else {
688 info_p->cpuid_logical_per_package = 1;
689 }
690
691 if (info_p->cpuid_max_ext >= 0x80000001) {
692 cpuid_fn(0x80000001, reg);
693 info_p->cpuid_extfeatures =
694 quad(reg[ecx], reg[edx]);
695 }
696
697 DBG(" max_basic : %d\n", info_p->cpuid_max_basic);
698 DBG(" max_ext : 0x%08x\n", info_p->cpuid_max_ext);
699 DBG(" vendor : %s\n", info_p->cpuid_vendor);
700 DBG(" brand_string : %s\n", info_p->cpuid_brand_string);
701 DBG(" signature : 0x%08x\n", info_p->cpuid_signature);
702 DBG(" stepping : %d\n", info_p->cpuid_stepping);
703 DBG(" model : %d\n", info_p->cpuid_model);
704 DBG(" family : %d\n", info_p->cpuid_family);
705 DBG(" type : %d\n", info_p->cpuid_type);
706 DBG(" extmodel : %d\n", info_p->cpuid_extmodel);
707 DBG(" extfamily : %d\n", info_p->cpuid_extfamily);
708 DBG(" brand : %d\n", info_p->cpuid_brand);
709 DBG(" features : 0x%016llx\n", info_p->cpuid_features);
710 DBG(" extfeatures : 0x%016llx\n", info_p->cpuid_extfeatures);
711 DBG(" logical_per_package : %d\n", info_p->cpuid_logical_per_package);
712 DBG(" microcode_version : 0x%08x\n", info_p->cpuid_microcode_version);
713
714 /* Fold in the Invariant TSC feature bit, if present */
715 if (info_p->cpuid_max_ext >= 0x80000007) {
716 cpuid_fn(0x80000007, reg);
717 info_p->cpuid_extfeatures |=
718 reg[edx] & (uint32_t)CPUID_EXTFEATURE_TSCI;
719 DBG(" extfeatures : 0x%016llx\n",
720 info_p->cpuid_extfeatures);
721 }
722
723 if (info_p->cpuid_max_basic >= 0x5) {
724 cpuid_mwait_leaf_t *cmp = &info_p->cpuid_mwait_leaf;
725
726 /*
727 * Extract the Monitor/Mwait Leaf info:
728 */
729 cpuid_fn(5, reg);
730 cmp->linesize_min = reg[eax];
731 cmp->linesize_max = reg[ebx];
732 cmp->extensions = reg[ecx];
733 cmp->sub_Cstates = reg[edx];
734 info_p->cpuid_mwait_leafp = cmp;
735
736 DBG(" Monitor/Mwait Leaf:\n");
737 DBG(" linesize_min : %d\n", cmp->linesize_min);
738 DBG(" linesize_max : %d\n", cmp->linesize_max);
739 DBG(" extensions : %d\n", cmp->extensions);
740 DBG(" sub_Cstates : 0x%08x\n", cmp->sub_Cstates);
741 }
742
743 if (info_p->cpuid_max_basic >= 0x6) {
744 cpuid_thermal_leaf_t *ctp = &info_p->cpuid_thermal_leaf;
745
746 /*
747 * The thermal and Power Leaf:
748 */
749 cpuid_fn(6, reg);
750 ctp->sensor = bitfield32(reg[eax], 0, 0);
751 ctp->dynamic_acceleration = bitfield32(reg[eax], 1, 1);
752 ctp->invariant_APIC_timer = bitfield32(reg[eax], 2, 2);
753 ctp->core_power_limits = bitfield32(reg[eax], 4, 4);
754 ctp->fine_grain_clock_mod = bitfield32(reg[eax], 5, 5);
755 ctp->package_thermal_intr = bitfield32(reg[eax], 6, 6);
756 ctp->thresholds = bitfield32(reg[ebx], 3, 0);
757 ctp->ACNT_MCNT = bitfield32(reg[ecx], 0, 0);
758 ctp->hardware_feedback = bitfield32(reg[ecx], 1, 1);
759 ctp->energy_policy = bitfield32(reg[ecx], 3, 3);
760 info_p->cpuid_thermal_leafp = ctp;
761
762 DBG(" Thermal/Power Leaf:\n");
763 DBG(" sensor : %d\n", ctp->sensor);
764 DBG(" dynamic_acceleration : %d\n", ctp->dynamic_acceleration);
765 DBG(" invariant_APIC_timer : %d\n", ctp->invariant_APIC_timer);
766 DBG(" core_power_limits : %d\n", ctp->core_power_limits);
767 DBG(" fine_grain_clock_mod : %d\n", ctp->fine_grain_clock_mod);
768 DBG(" package_thermal_intr : %d\n", ctp->package_thermal_intr);
769 DBG(" thresholds : %d\n", ctp->thresholds);
770 DBG(" ACNT_MCNT : %d\n", ctp->ACNT_MCNT);
771 DBG(" ACNT2 : %d\n", ctp->hardware_feedback);
772 DBG(" energy_policy : %d\n", ctp->energy_policy);
773 }
774
775 if (info_p->cpuid_max_basic >= 0xa) {
776 cpuid_arch_perf_leaf_t *capp = &info_p->cpuid_arch_perf_leaf;
777
778 /*
779 * Architectural Performance Monitoring Leaf:
780 */
781 cpuid_fn(0xa, reg);
782 capp->version = bitfield32(reg[eax], 7, 0);
783 capp->number = bitfield32(reg[eax], 15, 8);
784 capp->width = bitfield32(reg[eax], 23, 16);
785 capp->events_number = bitfield32(reg[eax], 31, 24);
786 capp->events = reg[ebx];
787 capp->fixed_number = bitfield32(reg[edx], 4, 0);
788 capp->fixed_width = bitfield32(reg[edx], 12, 5);
789 info_p->cpuid_arch_perf_leafp = capp;
790
791 DBG(" Architectural Performance Monitoring Leaf:\n");
792 DBG(" version : %d\n", capp->version);
793 DBG(" number : %d\n", capp->number);
794 DBG(" width : %d\n", capp->width);
795 DBG(" events_number : %d\n", capp->events_number);
796 DBG(" events : %d\n", capp->events);
797 DBG(" fixed_number : %d\n", capp->fixed_number);
798 DBG(" fixed_width : %d\n", capp->fixed_width);
799 }
800
801 if (info_p->cpuid_max_basic >= 0xd) {
802 cpuid_xsave_leaf_t *xsp;
803 /*
804 * XSAVE Features:
805 */
806 xsp = &info_p->cpuid_xsave_leaf[0];
807 info_p->cpuid_xsave_leafp = xsp;
808 xsp->extended_state[eax] = 0xd;
809 xsp->extended_state[ecx] = 0;
810 cpuid(xsp->extended_state);
811 DBG(" XSAVE Main leaf:\n");
812 DBG(" EAX : 0x%x\n", xsp->extended_state[eax]);
813 DBG(" EBX : 0x%x\n", xsp->extended_state[ebx]);
814 DBG(" ECX : 0x%x\n", xsp->extended_state[ecx]);
815 DBG(" EDX : 0x%x\n", xsp->extended_state[edx]);
816
817 xsp = &info_p->cpuid_xsave_leaf[1];
818 xsp->extended_state[eax] = 0xd;
819 xsp->extended_state[ecx] = 1;
820 cpuid(xsp->extended_state);
821 DBG(" XSAVE Sub-leaf1:\n");
822 DBG(" EAX : 0x%x\n", xsp->extended_state[eax]);
823 DBG(" EBX : 0x%x\n", xsp->extended_state[ebx]);
824 DBG(" ECX : 0x%x\n", xsp->extended_state[ecx]);
825 DBG(" EDX : 0x%x\n", xsp->extended_state[edx]);
826
827 const uint32_t valid =
828 info_p->cpuid_xsave_leafp->extended_state[eax];
829 for (unsigned n = 2; n < 8; n++) {
830 if ((valid & (1u << n)) == 0) {
831 continue;
832 }
833 xsp = &info_p->cpuid_xsave_leaf[n];
834 xsp->extended_state[eax] = 0xd;
835 xsp->extended_state[ecx] = n;
836 cpuid(xsp->extended_state);
837 DBG(" XSAVE Sub-leaf%d:\n", n);
838 DBG(" EAX : 0x%x\n",
839 xsp->extended_state[eax]);
840 DBG(" EBX : 0x%x\n",
841 xsp->extended_state[ebx]);
842 DBG(" ECX : 0x%x\n",
843 xsp->extended_state[ecx]);
844 DBG(" EDX : 0x%x\n",
845 xsp->extended_state[edx]);
846 }
847 }
848
849 if (info_p->cpuid_max_basic >= 7) {
850 /*
851 * Leaf7 Features:
852 */
853 cpuid_fn(0x7, reg);
854 info_p->cpuid_leaf7_features = quad(reg[ecx], reg[ebx]);
855 info_p->cpuid_leaf7_extfeatures = reg[edx];
856
857 cpuid_tsx_supported = (reg[ebx] & (CPUID_LEAF7_FEATURE_HLE | CPUID_LEAF7_FEATURE_RTM)) != 0;
858
859 DBG(" Feature Leaf7:\n");
860 DBG(" EBX : 0x%x\n", reg[ebx]);
861 DBG(" ECX : 0x%x\n", reg[ecx]);
862 DBG(" EDX : 0x%x\n", reg[edx]);
863 }
864
865 if (info_p->cpuid_max_basic >= 0x15) {
866 /*
867 * TCS/CCC frequency leaf:
868 */
869 cpuid_fn(0x15, reg);
870 info_p->cpuid_tsc_leaf.denominator = reg[eax];
871 info_p->cpuid_tsc_leaf.numerator = reg[ebx];
872
873 DBG(" TSC/CCC Information Leaf:\n");
874 DBG(" numerator : 0x%x\n", reg[ebx]);
875 DBG(" denominator : 0x%x\n", reg[eax]);
876 }
877 }
878
879 static uint32_t
cpuid_set_cpufamily(i386_cpu_info_t * info_p)880 cpuid_set_cpufamily(i386_cpu_info_t *info_p)
881 {
882 uint32_t cpufamily = CPUFAMILY_UNKNOWN;
883
884 switch (info_p->cpuid_family) {
885 case 6:
886 switch (info_p->cpuid_model) {
887 case 23:
888 cpufamily = CPUFAMILY_INTEL_PENRYN;
889 break;
890 case CPUID_MODEL_NEHALEM:
891 case CPUID_MODEL_FIELDS:
892 case CPUID_MODEL_DALES:
893 case CPUID_MODEL_NEHALEM_EX:
894 cpufamily = CPUFAMILY_INTEL_NEHALEM;
895 break;
896 case CPUID_MODEL_DALES_32NM:
897 case CPUID_MODEL_WESTMERE:
898 case CPUID_MODEL_WESTMERE_EX:
899 cpufamily = CPUFAMILY_INTEL_WESTMERE;
900 break;
901 case CPUID_MODEL_SANDYBRIDGE:
902 case CPUID_MODEL_JAKETOWN:
903 cpufamily = CPUFAMILY_INTEL_SANDYBRIDGE;
904 break;
905 case CPUID_MODEL_IVYBRIDGE:
906 case CPUID_MODEL_IVYBRIDGE_EP:
907 cpufamily = CPUFAMILY_INTEL_IVYBRIDGE;
908 break;
909 case CPUID_MODEL_HASWELL:
910 case CPUID_MODEL_HASWELL_EP:
911 case CPUID_MODEL_HASWELL_ULT:
912 case CPUID_MODEL_CRYSTALWELL:
913 cpufamily = CPUFAMILY_INTEL_HASWELL;
914 break;
915 case CPUID_MODEL_BROADWELL:
916 case CPUID_MODEL_BRYSTALWELL:
917 cpufamily = CPUFAMILY_INTEL_BROADWELL;
918 break;
919 case CPUID_MODEL_SKYLAKE:
920 case CPUID_MODEL_SKYLAKE_DT:
921 case CPUID_MODEL_SKYLAKE_W:
922 cpufamily = CPUFAMILY_INTEL_SKYLAKE;
923 break;
924 case CPUID_MODEL_KABYLAKE:
925 case CPUID_MODEL_KABYLAKE_DT:
926 cpufamily = CPUFAMILY_INTEL_KABYLAKE;
927 break;
928 case CPUID_MODEL_ICELAKE:
929 case CPUID_MODEL_ICELAKE_H:
930 case CPUID_MODEL_ICELAKE_DT:
931 cpufamily = CPUFAMILY_INTEL_ICELAKE;
932 break;
933 case CPUID_MODEL_COMETLAKE_DT:
934 cpufamily = CPUFAMILY_INTEL_COMETLAKE;
935 break;
936 }
937 break;
938 }
939
940 info_p->cpuid_cpufamily = cpufamily;
941 DBG("cpuid_set_cpufamily(%p) returning 0x%x\n", info_p, cpufamily);
942 return cpufamily;
943 }
944 /*
945 * Must be invoked either when executing single threaded, or with
946 * independent synchronization.
947 */
948 void
cpuid_set_info(void)949 cpuid_set_info(void)
950 {
951 i386_cpu_info_t *info_p = &cpuid_cpu_info;
952 boolean_t enable_x86_64h = TRUE;
953
954 /* Perform pre-cpuid workarounds (since their effects impact values returned via cpuid) */
955 cpuid_do_precpuid_was();
956
957 cpuid_set_generic_info(info_p);
958
959 /* verify we are running on a supported CPU */
960 if ((strncmp(CPUID_VID_INTEL, info_p->cpuid_vendor,
961 min(strlen(CPUID_STRING_UNKNOWN) + 1,
962 sizeof(info_p->cpuid_vendor)))) ||
963 (cpuid_set_cpufamily(info_p) == CPUFAMILY_UNKNOWN)) {
964 panic("Unsupported CPU");
965 }
966
967 info_p->cpuid_cpu_type = CPU_TYPE_X86;
968
969 if (!PE_parse_boot_argn("-enable_x86_64h", &enable_x86_64h, sizeof(enable_x86_64h))) {
970 boolean_t disable_x86_64h = FALSE;
971
972 if (PE_parse_boot_argn("-disable_x86_64h", &disable_x86_64h, sizeof(disable_x86_64h))) {
973 enable_x86_64h = FALSE;
974 }
975 }
976
977 if (enable_x86_64h &&
978 ((info_p->cpuid_features & CPUID_X86_64_H_FEATURE_SUBSET) == CPUID_X86_64_H_FEATURE_SUBSET) &&
979 ((info_p->cpuid_extfeatures & CPUID_X86_64_H_EXTFEATURE_SUBSET) == CPUID_X86_64_H_EXTFEATURE_SUBSET) &&
980 ((info_p->cpuid_leaf7_features & CPUID_X86_64_H_LEAF7_FEATURE_SUBSET) == CPUID_X86_64_H_LEAF7_FEATURE_SUBSET)) {
981 info_p->cpuid_cpu_subtype = CPU_SUBTYPE_X86_64_H;
982 } else {
983 info_p->cpuid_cpu_subtype = CPU_SUBTYPE_X86_ARCH1;
984 }
985 /* cpuid_set_cache_info must be invoked after set_generic_info */
986
987 /*
988 * Find the number of enabled cores and threads
989 * (which determines whether SMT/Hyperthreading is active).
990 */
991
992 /*
993 * Not all VMMs emulate MSR_CORE_THREAD_COUNT (0x35).
994 */
995 if (0 != (info_p->cpuid_features & CPUID_FEATURE_VMM) &&
996 PE_parse_boot_argn("-nomsr35h", NULL, 0)) {
997 info_p->core_count = 1;
998 info_p->thread_count = 1;
999 cpuid_set_cache_info(info_p);
1000 } else {
1001 switch (info_p->cpuid_cpufamily) {
1002 case CPUFAMILY_INTEL_PENRYN:
1003 cpuid_set_cache_info(info_p);
1004 info_p->core_count = info_p->cpuid_cores_per_package;
1005 info_p->thread_count = info_p->cpuid_logical_per_package;
1006 break;
1007 case CPUFAMILY_INTEL_WESTMERE: {
1008 /*
1009 * This should be the same as Nehalem but an A0 silicon bug returns
1010 * invalid data in the top 12 bits. Hence, we use only bits [19..16]
1011 * rather than [31..16] for core count - which actually can't exceed 8.
1012 */
1013 uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT);
1014 if (0 == msr) {
1015 /* Provide a non-zero default for some VMMs */
1016 msr = (1 << 16) | 1;
1017 }
1018 info_p->core_count = bitfield32((uint32_t)msr, 19, 16);
1019 info_p->thread_count = bitfield32((uint32_t)msr, 15, 0);
1020 cpuid_set_cache_info(info_p);
1021 break;
1022 }
1023 default: {
1024 uint64_t msr = rdmsr64(MSR_CORE_THREAD_COUNT);
1025 if (0 == msr) {
1026 /* Provide a non-zero default for some VMMs */
1027 msr = (1 << 16) | 1;
1028 }
1029 info_p->core_count = bitfield32((uint32_t)msr, 31, 16);
1030 info_p->thread_count = bitfield32((uint32_t)msr, 15, 0);
1031 cpuid_set_cache_info(info_p);
1032 break;
1033 }
1034 }
1035 }
1036
1037 DBG("cpuid_set_info():\n");
1038 DBG(" core_count : %d\n", info_p->core_count);
1039 DBG(" thread_count : %d\n", info_p->thread_count);
1040 DBG(" cpu_type: 0x%08x\n", info_p->cpuid_cpu_type);
1041 DBG(" cpu_subtype: 0x%08x\n", info_p->cpuid_cpu_subtype);
1042
1043 info_p->cpuid_model_string = ""; /* deprecated */
1044
1045 /* Init CPU LBRs */
1046 i386_lbr_init(info_p, true);
1047
1048 do_cwas(info_p, FALSE);
1049 }
1050
1051 static struct table {
1052 uint64_t mask;
1053 const char *name;
1054 } feature_map[] = {
1055 {CPUID_FEATURE_FPU, "FPU"},
1056 {CPUID_FEATURE_VME, "VME"},
1057 {CPUID_FEATURE_DE, "DE"},
1058 {CPUID_FEATURE_PSE, "PSE"},
1059 {CPUID_FEATURE_TSC, "TSC"},
1060 {CPUID_FEATURE_MSR, "MSR"},
1061 {CPUID_FEATURE_PAE, "PAE"},
1062 {CPUID_FEATURE_MCE, "MCE"},
1063 {CPUID_FEATURE_CX8, "CX8"},
1064 {CPUID_FEATURE_APIC, "APIC"},
1065 {CPUID_FEATURE_SEP, "SEP"},
1066 {CPUID_FEATURE_MTRR, "MTRR"},
1067 {CPUID_FEATURE_PGE, "PGE"},
1068 {CPUID_FEATURE_MCA, "MCA"},
1069 {CPUID_FEATURE_CMOV, "CMOV"},
1070 {CPUID_FEATURE_PAT, "PAT"},
1071 {CPUID_FEATURE_PSE36, "PSE36"},
1072 {CPUID_FEATURE_PSN, "PSN"},
1073 {CPUID_FEATURE_CLFSH, "CLFSH"},
1074 {CPUID_FEATURE_DS, "DS"},
1075 {CPUID_FEATURE_ACPI, "ACPI"},
1076 {CPUID_FEATURE_MMX, "MMX"},
1077 {CPUID_FEATURE_FXSR, "FXSR"},
1078 {CPUID_FEATURE_SSE, "SSE"},
1079 {CPUID_FEATURE_SSE2, "SSE2"},
1080 {CPUID_FEATURE_SS, "SS"},
1081 {CPUID_FEATURE_HTT, "HTT"},
1082 {CPUID_FEATURE_TM, "TM"},
1083 {CPUID_FEATURE_PBE, "PBE"},
1084 {CPUID_FEATURE_SSE3, "SSE3"},
1085 {CPUID_FEATURE_PCLMULQDQ, "PCLMULQDQ"},
1086 {CPUID_FEATURE_DTES64, "DTES64"},
1087 {CPUID_FEATURE_MONITOR, "MON"},
1088 {CPUID_FEATURE_DSCPL, "DSCPL"},
1089 {CPUID_FEATURE_VMX, "VMX"},
1090 {CPUID_FEATURE_SMX, "SMX"},
1091 {CPUID_FEATURE_EST, "EST"},
1092 {CPUID_FEATURE_TM2, "TM2"},
1093 {CPUID_FEATURE_SSSE3, "SSSE3"},
1094 {CPUID_FEATURE_CID, "CID"},
1095 {CPUID_FEATURE_FMA, "FMA"},
1096 {CPUID_FEATURE_CX16, "CX16"},
1097 {CPUID_FEATURE_xTPR, "TPR"},
1098 {CPUID_FEATURE_PDCM, "PDCM"},
1099 {CPUID_FEATURE_SSE4_1, "SSE4.1"},
1100 {CPUID_FEATURE_SSE4_2, "SSE4.2"},
1101 {CPUID_FEATURE_x2APIC, "x2APIC"},
1102 {CPUID_FEATURE_MOVBE, "MOVBE"},
1103 {CPUID_FEATURE_POPCNT, "POPCNT"},
1104 {CPUID_FEATURE_AES, "AES"},
1105 {CPUID_FEATURE_VMM, "VMM"},
1106 {CPUID_FEATURE_PCID, "PCID"},
1107 {CPUID_FEATURE_XSAVE, "XSAVE"},
1108 {CPUID_FEATURE_OSXSAVE, "OSXSAVE"},
1109 {CPUID_FEATURE_SEGLIM64, "SEGLIM64"},
1110 {CPUID_FEATURE_TSCTMR, "TSCTMR"},
1111 {CPUID_FEATURE_AVX1_0, "AVX1.0"},
1112 {CPUID_FEATURE_RDRAND, "RDRAND"},
1113 {CPUID_FEATURE_F16C, "F16C"},
1114 {0, 0}
1115 },
1116 extfeature_map[] = {
1117 {CPUID_EXTFEATURE_SYSCALL, "SYSCALL"},
1118 {CPUID_EXTFEATURE_XD, "XD"},
1119 {CPUID_EXTFEATURE_1GBPAGE, "1GBPAGE"},
1120 {CPUID_EXTFEATURE_EM64T, "EM64T"},
1121 {CPUID_EXTFEATURE_LAHF, "LAHF"},
1122 {CPUID_EXTFEATURE_LZCNT, "LZCNT"},
1123 {CPUID_EXTFEATURE_PREFETCHW, "PREFETCHW"},
1124 {CPUID_EXTFEATURE_RDTSCP, "RDTSCP"},
1125 {CPUID_EXTFEATURE_TSCI, "TSCI"},
1126 {0, 0}
1127 },
1128 leaf7_feature_map[] = {
1129 {CPUID_LEAF7_FEATURE_RDWRFSGS, "RDWRFSGS"},
1130 {CPUID_LEAF7_FEATURE_TSCOFF, "TSC_THREAD_OFFSET"},
1131 {CPUID_LEAF7_FEATURE_SGX, "SGX"},
1132 {CPUID_LEAF7_FEATURE_BMI1, "BMI1"},
1133 {CPUID_LEAF7_FEATURE_HLE, "HLE"},
1134 {CPUID_LEAF7_FEATURE_AVX2, "AVX2"},
1135 {CPUID_LEAF7_FEATURE_FDPEO, "FDPEO"},
1136 {CPUID_LEAF7_FEATURE_SMEP, "SMEP"},
1137 {CPUID_LEAF7_FEATURE_BMI2, "BMI2"},
1138 {CPUID_LEAF7_FEATURE_ERMS, "ERMS"},
1139 {CPUID_LEAF7_FEATURE_INVPCID, "INVPCID"},
1140 {CPUID_LEAF7_FEATURE_RTM, "RTM"},
1141 {CPUID_LEAF7_FEATURE_PQM, "PQM"},
1142 {CPUID_LEAF7_FEATURE_FPU_CSDS, "FPU_CSDS"},
1143 {CPUID_LEAF7_FEATURE_MPX, "MPX"},
1144 {CPUID_LEAF7_FEATURE_PQE, "PQE"},
1145 {CPUID_LEAF7_FEATURE_AVX512F, "AVX512F"},
1146 {CPUID_LEAF7_FEATURE_AVX512DQ, "AVX512DQ"},
1147 {CPUID_LEAF7_FEATURE_RDSEED, "RDSEED"},
1148 {CPUID_LEAF7_FEATURE_ADX, "ADX"},
1149 {CPUID_LEAF7_FEATURE_SMAP, "SMAP"},
1150 {CPUID_LEAF7_FEATURE_AVX512IFMA, "AVX512IFMA"},
1151 {CPUID_LEAF7_FEATURE_CLFSOPT, "CLFSOPT"},
1152 {CPUID_LEAF7_FEATURE_CLWB, "CLWB"},
1153 {CPUID_LEAF7_FEATURE_IPT, "IPT"},
1154 {CPUID_LEAF7_FEATURE_AVX512CD, "AVX512CD"},
1155 {CPUID_LEAF7_FEATURE_SHA, "SHA"},
1156 {CPUID_LEAF7_FEATURE_AVX512BW, "AVX512BW"},
1157 {CPUID_LEAF7_FEATURE_AVX512VL, "AVX512VL"},
1158 {CPUID_LEAF7_FEATURE_PREFETCHWT1, "PREFETCHWT1"},
1159 {CPUID_LEAF7_FEATURE_AVX512VBMI, "AVX512VBMI"},
1160 {CPUID_LEAF7_FEATURE_UMIP, "UMIP"},
1161 {CPUID_LEAF7_FEATURE_PKU, "PKU"},
1162 {CPUID_LEAF7_FEATURE_OSPKE, "OSPKE"},
1163 {CPUID_LEAF7_FEATURE_WAITPKG, "WAITPKG"},
1164 {CPUID_LEAF7_FEATURE_GFNI, "GFNI"},
1165 {CPUID_LEAF7_FEATURE_VAES, "VAES"},
1166 {CPUID_LEAF7_FEATURE_VPCLMULQDQ, "VPCLMULQDQ"},
1167 {CPUID_LEAF7_FEATURE_AVX512VNNI, "AVX512VNNI"},
1168 {CPUID_LEAF7_FEATURE_AVX512BITALG, "AVX512BITALG"},
1169 {CPUID_LEAF7_FEATURE_AVX512VPCDQ, "AVX512VPOPCNTDQ"},
1170 {CPUID_LEAF7_FEATURE_RDPID, "RDPID"},
1171 {CPUID_LEAF7_FEATURE_CLDEMOTE, "CLDEMOTE"},
1172 {CPUID_LEAF7_FEATURE_MOVDIRI, "MOVDIRI"},
1173 {CPUID_LEAF7_FEATURE_MOVDIRI64B, "MOVDIRI64B"},
1174 {CPUID_LEAF7_FEATURE_SGXLC, "SGXLC"},
1175 {0, 0}
1176 },
1177 leaf7_extfeature_map[] = {
1178 { CPUID_LEAF7_EXTFEATURE_AVX5124VNNIW, "AVX5124VNNIW" },
1179 { CPUID_LEAF7_EXTFEATURE_AVX5124FMAPS, "AVX5124FMAPS" },
1180 { CPUID_LEAF7_EXTFEATURE_FSREPMOV, "FSREPMOV" },
1181 { CPUID_LEAF7_EXTFEATURE_MDCLEAR, "MDCLEAR" },
1182 { CPUID_LEAF7_EXTFEATURE_TSXFA, "TSXFA" },
1183 { CPUID_LEAF7_EXTFEATURE_IBRS, "IBRS" },
1184 { CPUID_LEAF7_EXTFEATURE_STIBP, "STIBP" },
1185 { CPUID_LEAF7_EXTFEATURE_L1DF, "L1DF" },
1186 { CPUID_LEAF7_EXTFEATURE_ACAPMSR, "ACAPMSR" },
1187 { CPUID_LEAF7_EXTFEATURE_CCAPMSR, "CCAPMSR" },
1188 { CPUID_LEAF7_EXTFEATURE_SSBD, "SSBD" },
1189 {0, 0}
1190 };
1191
1192 static char *
cpuid_get_names(struct table * map,uint64_t bits,char * buf,unsigned buf_len)1193 cpuid_get_names(struct table *map, uint64_t bits, char *buf, unsigned buf_len)
1194 {
1195 size_t len = 0;
1196 char *p = buf;
1197 int i;
1198
1199 for (i = 0; map[i].mask != 0; i++) {
1200 if ((bits & map[i].mask) == 0) {
1201 continue;
1202 }
1203 if (len && ((size_t) (p - buf) < (buf_len - 1))) {
1204 *p++ = ' ';
1205 }
1206 len = min(strlen(map[i].name), (size_t)((buf_len - 1) - (p - buf)));
1207 if (len == 0) {
1208 break;
1209 }
1210 bcopy(map[i].name, p, len);
1211 p += len;
1212 }
1213 *p = '\0';
1214 return buf;
1215 }
1216
1217 i386_cpu_info_t *
cpuid_info(void)1218 cpuid_info(void)
1219 {
1220 /* Set-up the cpuid_info stucture lazily */
1221 if (cpuid_cpu_infop == NULL) {
1222 PE_parse_boot_argn("-cpuid", &cpuid_dbg, sizeof(cpuid_dbg));
1223 cpuid_set_info();
1224 cpuid_cpu_infop = &cpuid_cpu_info;
1225 }
1226 return cpuid_cpu_infop;
1227 }
1228
1229 char *
cpuid_get_feature_names(uint64_t features,char * buf,unsigned buf_len)1230 cpuid_get_feature_names(uint64_t features, char *buf, unsigned buf_len)
1231 {
1232 return cpuid_get_names(feature_map, features, buf, buf_len);
1233 }
1234
1235 char *
cpuid_get_extfeature_names(uint64_t extfeatures,char * buf,unsigned buf_len)1236 cpuid_get_extfeature_names(uint64_t extfeatures, char *buf, unsigned buf_len)
1237 {
1238 return cpuid_get_names(extfeature_map, extfeatures, buf, buf_len);
1239 }
1240
1241 char *
cpuid_get_leaf7_feature_names(uint64_t features,char * buf,unsigned buf_len)1242 cpuid_get_leaf7_feature_names(uint64_t features, char *buf, unsigned buf_len)
1243 {
1244 return cpuid_get_names(leaf7_feature_map, features, buf, buf_len);
1245 }
1246
1247 char *
cpuid_get_leaf7_extfeature_names(uint64_t features,char * buf,unsigned buf_len)1248 cpuid_get_leaf7_extfeature_names(uint64_t features, char *buf, unsigned buf_len)
1249 {
1250 return cpuid_get_names(leaf7_extfeature_map, features, buf, buf_len);
1251 }
1252
1253 void
cpuid_feature_display(const char * header)1254 cpuid_feature_display(
1255 const char *header)
1256 {
1257 char buf[320];
1258
1259 kprintf("%s: %s", header,
1260 cpuid_get_feature_names(cpuid_features(), buf, sizeof(buf)));
1261 if (cpuid_leaf7_features()) {
1262 kprintf(" %s", cpuid_get_leaf7_feature_names(
1263 cpuid_leaf7_features(), buf, sizeof(buf)));
1264 }
1265 if (cpuid_leaf7_extfeatures()) {
1266 kprintf(" %s", cpuid_get_leaf7_extfeature_names(
1267 cpuid_leaf7_extfeatures(), buf, sizeof(buf)));
1268 }
1269 kprintf("\n");
1270 if (cpuid_features() & CPUID_FEATURE_HTT) {
1271 #define s_if_plural(n) ((n > 1) ? "s" : "")
1272 kprintf(" HTT: %d core%s per package;"
1273 " %d logical cpu%s per package\n",
1274 cpuid_cpu_infop->cpuid_cores_per_package,
1275 s_if_plural(cpuid_cpu_infop->cpuid_cores_per_package),
1276 cpuid_cpu_infop->cpuid_logical_per_package,
1277 s_if_plural(cpuid_cpu_infop->cpuid_logical_per_package));
1278 }
1279 }
1280
1281 void
cpuid_extfeature_display(const char * header)1282 cpuid_extfeature_display(
1283 const char *header)
1284 {
1285 char buf[256];
1286
1287 kprintf("%s: %s\n", header,
1288 cpuid_get_extfeature_names(cpuid_extfeatures(),
1289 buf, sizeof(buf)));
1290 }
1291
1292 void
cpuid_cpu_display(const char * header)1293 cpuid_cpu_display(
1294 const char *header)
1295 {
1296 if (cpuid_cpu_infop->cpuid_brand_string[0] != '\0') {
1297 kprintf("%s: %s\n", header, cpuid_cpu_infop->cpuid_brand_string);
1298 }
1299 }
1300
1301 unsigned int
cpuid_family(void)1302 cpuid_family(void)
1303 {
1304 return cpuid_info()->cpuid_family;
1305 }
1306
1307 uint32_t
cpuid_cpufamily(void)1308 cpuid_cpufamily(void)
1309 {
1310 return cpuid_info()->cpuid_cpufamily;
1311 }
1312
1313 cpu_type_t
cpuid_cputype(void)1314 cpuid_cputype(void)
1315 {
1316 return cpuid_info()->cpuid_cpu_type;
1317 }
1318
1319 cpu_subtype_t
cpuid_cpusubtype(void)1320 cpuid_cpusubtype(void)
1321 {
1322 return cpuid_info()->cpuid_cpu_subtype;
1323 }
1324
1325 uint64_t
cpuid_features(void)1326 cpuid_features(void)
1327 {
1328 static int checked = 0;
1329 char fpu_arg[20] = { 0 };
1330
1331 (void) cpuid_info();
1332 if (!checked) {
1333 /* check for boot-time fpu limitations */
1334 if (PE_parse_boot_argn("_fpu", &fpu_arg[0], sizeof(fpu_arg))) {
1335 printf("limiting fpu features to: %s\n", fpu_arg);
1336 if (!strncmp("387", fpu_arg, sizeof("387")) || !strncmp("mmx", fpu_arg, sizeof("mmx"))) {
1337 printf("no sse or sse2\n");
1338 cpuid_cpu_infop->cpuid_features &= ~(CPUID_FEATURE_SSE | CPUID_FEATURE_SSE2 | CPUID_FEATURE_FXSR);
1339 } else if (!strncmp("sse", fpu_arg, sizeof("sse"))) {
1340 printf("no sse2\n");
1341 cpuid_cpu_infop->cpuid_features &= ~(CPUID_FEATURE_SSE2);
1342 }
1343 }
1344 checked = 1;
1345 }
1346 return cpuid_cpu_infop->cpuid_features;
1347 }
1348
1349 uint64_t
cpuid_extfeatures(void)1350 cpuid_extfeatures(void)
1351 {
1352 return cpuid_info()->cpuid_extfeatures;
1353 }
1354
1355 uint64_t
cpuid_leaf7_features(void)1356 cpuid_leaf7_features(void)
1357 {
1358 return cpuid_info()->cpuid_leaf7_features;
1359 }
1360
1361 uint64_t
cpuid_leaf7_extfeatures(void)1362 cpuid_leaf7_extfeatures(void)
1363 {
1364 return cpuid_info()->cpuid_leaf7_extfeatures;
1365 }
1366
1367 const char *
cpuid_vmm_family_string(void)1368 cpuid_vmm_family_string(void)
1369 {
1370 switch (cpuid_vmm_info()->cpuid_vmm_family) {
1371 case CPUID_VMM_FAMILY_NONE:
1372 return "None";
1373
1374 case CPUID_VMM_FAMILY_VMWARE:
1375 return "VMWare";
1376
1377 case CPUID_VMM_FAMILY_PARALLELS:
1378 return "Parallels";
1379
1380 case CPUID_VMM_FAMILY_HYVE:
1381 return "xHyve";
1382
1383 case CPUID_VMM_FAMILY_HVF:
1384 return "HVF";
1385
1386 case CPUID_VMM_FAMILY_KVM:
1387 return "KVM";
1388
1389 case CPUID_VMM_FAMILY_UNKNOWN:
1390 /*FALLTHROUGH*/
1391 default:
1392 return "Unknown VMM";
1393 }
1394 }
1395
1396 static i386_vmm_info_t *_cpuid_vmm_infop = NULL;
1397 static i386_vmm_info_t _cpuid_vmm_info;
1398
1399 static void
cpuid_init_vmm_info(i386_vmm_info_t * info_p)1400 cpuid_init_vmm_info(i386_vmm_info_t *info_p)
1401 {
1402 uint32_t reg[4], maxbasic_regs[4];
1403 uint32_t max_vmm_leaf;
1404
1405 bzero(info_p, sizeof(*info_p));
1406
1407 if (!cpuid_vmm_present()) {
1408 return;
1409 }
1410
1411 DBG("cpuid_init_vmm_info(%p)\n", info_p);
1412
1413 /*
1414 * Get the highest basic leaf value, then save the cpuid details for that leaf
1415 * for comparison with the [ostensible] VMM leaf.
1416 */
1417 cpuid_fn(0, reg);
1418 cpuid_fn(reg[eax], maxbasic_regs);
1419
1420 /* do cpuid 0x40000000 to get VMM vendor */
1421 cpuid_fn(0x40000000, reg);
1422
1423 /*
1424 * If leaf 0x40000000 is non-existent, cpuid will return the values as
1425 * if the highest basic leaf was requested, so compare to those values
1426 * we just retrieved to see if no vmm is present.
1427 */
1428 if (bcmp(reg, maxbasic_regs, sizeof(reg)) == 0) {
1429 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_NONE;
1430 DBG(" vmm_vendor : NONE\n");
1431 return;
1432 }
1433
1434 max_vmm_leaf = reg[eax];
1435 bcopy((char *)®[ebx], &info_p->cpuid_vmm_vendor[0], 4);
1436 bcopy((char *)®[ecx], &info_p->cpuid_vmm_vendor[4], 4);
1437 bcopy((char *)®[edx], &info_p->cpuid_vmm_vendor[8], 4);
1438 info_p->cpuid_vmm_vendor[12] = '\0';
1439
1440 if (0 == strcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_VMWARE)) {
1441 /* VMware identification string: kb.vmware.com/kb/1009458 */
1442 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_VMWARE;
1443 } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_PARALLELS, 12)) {
1444 /* Parallels identification string */
1445 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_PARALLELS;
1446 } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_HYVE, 12)) {
1447 /* bhyve/xhyve identification string */
1448 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_HYVE;
1449 } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_HVF, 12)) {
1450 /* HVF identification string */
1451 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_HVF;
1452 } else if (0 == bcmp(info_p->cpuid_vmm_vendor, CPUID_VMM_ID_KVM, 12)) {
1453 /* KVM identification string */
1454 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_KVM;
1455 if (max_vmm_leaf >= 0x40000001) {
1456 cpuid_fn(0x40000001, reg);
1457 info_p->cpuid_vmm_kvm_features =
1458 quad(reg[edx], reg[eax]);
1459 }
1460 } else {
1461 info_p->cpuid_vmm_family = CPUID_VMM_FAMILY_UNKNOWN;
1462 }
1463
1464 /* VMM generic leaves: https://lkml.org/lkml/2008/10/1/246 */
1465 if (max_vmm_leaf >= 0x40000010) {
1466 cpuid_fn(0x40000010, reg);
1467
1468 info_p->cpuid_vmm_tsc_frequency = reg[eax];
1469 info_p->cpuid_vmm_bus_frequency = reg[ebx];
1470 }
1471
1472 cpuid_vmm_detect_pv_interface(info_p, APPLEPV_SIGNATURE, &cpuid_vmm_detect_applepv_features);
1473
1474 DBG(" vmm_vendor : %s\n", info_p->cpuid_vmm_vendor);
1475 DBG(" vmm_family : %u\n", info_p->cpuid_vmm_family);
1476 DBG(" vmm_bus_frequency : %u\n", info_p->cpuid_vmm_bus_frequency);
1477 DBG(" vmm_tsc_frequency : %u\n", info_p->cpuid_vmm_tsc_frequency);
1478 }
1479
1480 boolean_t
cpuid_vmm_present(void)1481 cpuid_vmm_present(void)
1482 {
1483 return (cpuid_features() & CPUID_FEATURE_VMM) ? TRUE : FALSE;
1484 }
1485
1486 i386_vmm_info_t *
cpuid_vmm_info(void)1487 cpuid_vmm_info(void)
1488 {
1489 if (_cpuid_vmm_infop == NULL) {
1490 cpuid_init_vmm_info(&_cpuid_vmm_info);
1491 _cpuid_vmm_infop = &_cpuid_vmm_info;
1492 }
1493 return _cpuid_vmm_infop;
1494 }
1495
1496 uint32_t
cpuid_vmm_family(void)1497 cpuid_vmm_family(void)
1498 {
1499 return cpuid_vmm_info()->cpuid_vmm_family;
1500 }
1501
1502 uint64_t
cpuid_vmm_get_kvm_features(void)1503 cpuid_vmm_get_kvm_features(void)
1504 {
1505 return cpuid_vmm_info()->cpuid_vmm_kvm_features;
1506 }
1507
1508 uint64_t
cpuid_vmm_get_applepv_features(void)1509 cpuid_vmm_get_applepv_features(void)
1510 {
1511 return cpuid_vmm_info()->cpuid_vmm_applepv_features;
1512 }
1513
1514 cwa_classifier_e
cpuid_wa_required(cpu_wa_e wa)1515 cpuid_wa_required(cpu_wa_e wa)
1516 {
1517 i386_cpu_info_t *info_p = &cpuid_cpu_info;
1518 static uint64_t bootarg_cpu_wa_enables = 0;
1519 static uint64_t bootarg_cpu_wa_disables = 0;
1520 static int bootargs_overrides_processed = 0;
1521 uint32_t reg[4];
1522
1523 if (!bootargs_overrides_processed) {
1524 if (!PE_parse_boot_argn("cwae", &bootarg_cpu_wa_enables, sizeof(bootarg_cpu_wa_enables))) {
1525 bootarg_cpu_wa_enables = 0;
1526 }
1527
1528 if (!PE_parse_boot_argn("cwad", &bootarg_cpu_wa_disables, sizeof(bootarg_cpu_wa_disables))) {
1529 bootarg_cpu_wa_disables = 0;
1530 }
1531 bootargs_overrides_processed = 1;
1532 }
1533
1534 if (bootarg_cpu_wa_enables & (1 << wa)) {
1535 return CWA_FORCE_ON;
1536 }
1537
1538 if (bootarg_cpu_wa_disables & (1 << wa)) {
1539 return CWA_FORCE_OFF;
1540 }
1541
1542 switch (wa) {
1543 case CPU_INTEL_SEGCHK:
1544 /* First, check to see if this CPU requires the workaround */
1545 if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_ACAPMSR) != 0) {
1546 /* We have ARCHCAP, so check it for either RDCL_NO or MDS_NO */
1547 uint64_t archcap_msr = rdmsr64(MSR_IA32_ARCH_CAPABILITIES);
1548 if ((archcap_msr & (MSR_IA32_ARCH_CAPABILITIES_RDCL_NO | MSR_IA32_ARCH_CAPABILITIES_MDS_NO)) != 0) {
1549 /* Workaround not needed */
1550 return CWA_OFF;
1551 }
1552 }
1553
1554 if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_MDCLEAR) != 0) {
1555 return CWA_ON;
1556 }
1557
1558 /*
1559 * If the CPU supports the ARCHCAP MSR and neither the RDCL_NO bit nor the MDS_NO
1560 * bit are set, OR the CPU does not support the ARCHCAP MSR and the CPU does
1561 * not enumerate the presence of the enhanced VERW instruction, report
1562 * that the workaround should not be enabled.
1563 */
1564 break;
1565
1566 case CPU_INTEL_TSXFA:
1567 /*
1568 * Note that if TSX was disabled in cpuid_do_precpuid_was(), the cached cpuid
1569 * info will indicate that RTM is *not* supported and this workaround will not
1570 * be enabled.
1571 */
1572 /*
1573 * Otherwise, if the CPU supports both TSX(HLE) and FORCE_ABORT, return that
1574 * the workaround should be enabled.
1575 */
1576 if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_TSXFA) != 0 &&
1577 (info_p->cpuid_leaf7_features & CPUID_LEAF7_FEATURE_RTM) != 0) {
1578 return CWA_ON;
1579 }
1580 break;
1581
1582 case CPU_INTEL_TSXDA:
1583 /*
1584 * Since this workaround might be requested before cpuid_set_info() is complete,
1585 * we need to invoke cpuid directly when looking for the required bits.
1586 */
1587 cpuid_fn(0x7, reg);
1588 if (reg[edx] & CPUID_LEAF7_EXTFEATURE_ACAPMSR) {
1589 uint64_t archcap_msr = rdmsr64(MSR_IA32_ARCH_CAPABILITIES);
1590 /*
1591 * If this CPU supports TSX (HLE being the proxy for TSX detection) AND it does
1592 * not include a hardware fix for TAA and it supports the TSX_CTRL MSR, disable TSX entirely.
1593 * (Note this can be overridden (above) if the cwad boot-arg's value has bit 2 set.)
1594 */
1595 if ((reg[ebx] & CPUID_LEAF7_FEATURE_HLE) != 0 &&
1596 (archcap_msr & (MSR_IA32_ARCH_CAPABILITIES_TAA_NO | MSR_IA32_ARCH_CAPABILITIES_TSX_CTRL))
1597 == MSR_IA32_ARCH_CAPABILITIES_TSX_CTRL) {
1598 return CWA_ON;
1599 }
1600 }
1601 break;
1602
1603 case CPU_INTEL_SRBDS:
1604 /*
1605 * SRBDS mitigations are enabled by default. CWA_ON returned here indicates
1606 * the caller should disable the mitigation. Mitigations should be disabled
1607 * at least for CPUs that advertise MDS_NO *and* (either TAA_NO is set OR TSX
1608 * has been disabled).
1609 */
1610 if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_SRBDS_CTRL) != 0) {
1611 if ((info_p->cpuid_leaf7_extfeatures & CPUID_LEAF7_EXTFEATURE_ACAPMSR) != 0) {
1612 uint64_t archcap_msr = rdmsr64(MSR_IA32_ARCH_CAPABILITIES);
1613 if ((archcap_msr & MSR_IA32_ARCH_CAPABILITIES_MDS_NO) != 0 &&
1614 ((archcap_msr & MSR_IA32_ARCH_CAPABILITIES_TAA_NO) != 0 ||
1615 cpuid_tsx_disabled)) {
1616 return CWA_ON;
1617 }
1618 }
1619 }
1620 break;
1621
1622 case CPU_INTEL_RSBST:
1623 /*
1624 * RSB-stuffing in the kernel exit trampolines (when returning to user)
1625 * RSB depth is 32. This workaround must be explicitly enabled via the
1626 * cwae boot-arg.
1627 */
1628 break;
1629
1630 default:
1631 break;
1632 }
1633
1634 return CWA_OFF;
1635 }
1636
1637 static void
cpuid_do_precpuid_was(void)1638 cpuid_do_precpuid_was(void)
1639 {
1640 /*
1641 * Note that care must be taken not to use any data from the cached cpuid data since it is
1642 * likely uninitialized at this point. That includes calling functions that make use of
1643 * that data as well.
1644 */
1645
1646 /* Note the TSX disablement, we do not support force-on since it depends on MSRs being present */
1647 if (cpuid_wa_required(CPU_INTEL_TSXDA) == CWA_ON) {
1648 /* This must be executed on all logical processors */
1649 wrmsr64(MSR_IA32_TSX_CTRL, MSR_IA32_TSXCTRL_TSX_CPU_CLEAR | MSR_IA32_TSXCTRL_RTM_DISABLE);
1650 cpuid_tsx_disabled = true;
1651 }
1652 }
1653
1654
1655 /*
1656 * Hunt for Apple Paravirtualization support in the hypervisor class leaves [0x4000_0000-0x4001_0000].
1657 * Hypervisor interfaces are expected to be found at 0x100 boundaries for compatibility.
1658 */
1659
1660 static bool
cpuid_vmm_detect_applepv_features(i386_vmm_info_t * info_p,const uint32_t base,const uint32_t max_leaf)1661 cpuid_vmm_detect_applepv_features(i386_vmm_info_t *info_p, const uint32_t base, const uint32_t max_leaf)
1662 {
1663 if ((max_leaf - base) < APPLEPV_LEAF_INDEX_MAX) {
1664 return false;
1665 }
1666
1667 /*
1668 * Issue cpuid to make sure the interface supports "AH#1" features.
1669 * This avoids a possible collision with "Hv#1" used by Hyper-V.
1670 */
1671 uint32_t reg[4];
1672 char interface[5];
1673 cpuid_fn(base + APPLEPV_INTERFACE_LEAF_INDEX, reg);
1674 memcpy(&interface[0], ®[eax], 4);
1675 interface[4] = '\0';
1676 if (0 == strcmp(interface, APPLEPV_INTERFACE)) {
1677 cpuid_fn(base + APPLEPV_FEATURES_LEAF_INDEX, reg);
1678 info_p->cpuid_vmm_applepv_features = quad(reg[ecx], reg[edx]);
1679 return true;
1680 }
1681 return false;
1682 }
1683
1684 static void
cpuid_vmm_detect_pv_interface(i386_vmm_info_t * info_p,const char * signature,bool (* searcher)(i386_vmm_info_t *,const uint32_t,const uint32_t))1685 cpuid_vmm_detect_pv_interface(i386_vmm_info_t *info_p, const char *signature,
1686 bool (*searcher)(i386_vmm_info_t*, const uint32_t, const uint32_t))
1687 {
1688 int hcalls;
1689 if (PE_parse_boot_argn("hcalls", &hcalls, sizeof(hcalls)) &&
1690 hcalls == 0) {
1691 return;
1692 }
1693
1694 assert(info_p);
1695
1696 /*
1697 * Look for PV interface matching signature
1698 */
1699 for (uint32_t base = 0x40000100; base < 0x40010000; base += 0x100) {
1700 uint32_t reg[4];
1701 char vendor[13];
1702
1703 cpuid_fn(base, reg);
1704 memcpy(&vendor[0], ®[ebx], 4);
1705 memcpy(&vendor[4], ®[ecx], 4);
1706 memcpy(&vendor[8], ®[edx], 4);
1707 vendor[12] = '\0';
1708 if ((0 == strcmp(vendor, signature)) &&
1709 (reg[eax] - base) < 0x100 &&
1710 (*searcher)(info_p, base, reg[eax])) {
1711 break;
1712 }
1713 }
1714 }
1715