1 /*
2 * Copyright (c) 2005-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 /*
33 * File: i386/tsc.c
34 * Purpose: Initializes the TSC and the various conversion
35 * factors needed by other parts of the system.
36 */
37
38
39 #include <mach/mach_types.h>
40
41 #include <kern/cpu_data.h>
42 #include <kern/cpu_number.h>
43 #include <kern/clock.h>
44 #include <kern/host_notify.h>
45 #include <kern/macro_help.h>
46 #include <kern/misc_protos.h>
47 #include <kern/spl.h>
48 #include <kern/assert.h>
49 #include <mach/vm_prot.h>
50 #include <vm/pmap.h>
51 #include <vm/vm_kern.h> /* for kernel_map */
52 #include <architecture/i386/pio.h>
53 #include <i386/machine_cpu.h>
54 #include <i386/cpuid.h>
55 #include <i386/mp.h>
56 #include <i386/machine_routines.h>
57 #include <i386/proc_reg.h>
58 #include <i386/tsc.h>
59 #include <i386/misc_protos.h>
60 #include <pexpert/pexpert.h>
61 #include <machine/limits.h>
62 #include <machine/commpage.h>
63 #include <sys/kdebug.h>
64 #include <pexpert/device_tree.h>
65
66 uint64_t busFCvtt2n = 0;
67 uint64_t busFCvtn2t = 0;
68 uint64_t tscFreq = 0;
69 uint64_t tscFCvtt2n = 0;
70 uint64_t tscFCvtn2t = 0;
71 uint64_t tscGranularity = 0;
72 uint64_t bus2tsc = 0;
73 uint64_t busFreq = 0;
74 uint32_t flex_ratio = 0;
75 uint32_t flex_ratio_min = 0;
76 uint32_t flex_ratio_max = 0;
77
78 uint64_t tsc_at_boot = 0;
79
80 #define bit(n) (1ULL << (n))
81 #define bitmask(h, l) ((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
82 #define bitfield(x, h, l) (((x) & bitmask(h,l)) >> l)
83
84 /* Decimal powers: */
85 #define kilo (1000ULL)
86 #define Mega (kilo * kilo)
87 #define Giga (kilo * Mega)
88 #define Tera (kilo * Giga)
89 #define Peta (kilo * Tera)
90
91 #define CPU_FAMILY_PENTIUM_M (0x6)
92
93 /*
94 * This routine extracts a frequency property in Hz from the device tree.
95 * Also reads any initial TSC value at boot from the device tree.
96 */
97 static uint64_t
EFI_get_frequency(const char * prop)98 EFI_get_frequency(const char *prop)
99 {
100 uint64_t frequency = 0;
101 DTEntry entry;
102 void const *value;
103 unsigned int size;
104
105 if (SecureDTLookupEntry(0, "/efi/platform", &entry) != kSuccess) {
106 kprintf("EFI_get_frequency: didn't find /efi/platform\n");
107 return 0;
108 }
109
110 /*
111 * While we're here, see if EFI published an initial TSC value.
112 */
113 if (SecureDTGetProperty(entry, "InitialTSC", &value, &size) == kSuccess) {
114 if (size == sizeof(uint64_t)) {
115 tsc_at_boot = *(uint64_t const *) value;
116 kprintf("EFI_get_frequency: read InitialTSC: %llu\n",
117 tsc_at_boot);
118 }
119 }
120
121 if (SecureDTGetProperty(entry, prop, &value, &size) != kSuccess) {
122 kprintf("EFI_get_frequency: property %s not found\n", prop);
123 return 0;
124 }
125 if (size == sizeof(uint64_t)) {
126 frequency = *(uint64_t const *) value;
127 kprintf("EFI_get_frequency: read %s value: %llu\n",
128 prop, frequency);
129 }
130
131 return frequency;
132 }
133
134 /*
135 * Initialize the various conversion factors needed by code referencing
136 * the TSC.
137 */
138 void
tsc_init(void)139 tsc_init(void)
140 {
141 boolean_t N_by_2_bus_ratio = FALSE;
142
143 if (cpuid_vmm_present()) {
144 kprintf("VMM vendor %s TSC frequency %u KHz bus frequency %u KHz\n",
145 cpuid_vmm_family_string(),
146 cpuid_vmm_info()->cpuid_vmm_tsc_frequency,
147 cpuid_vmm_info()->cpuid_vmm_bus_frequency);
148
149 if (cpuid_vmm_info()->cpuid_vmm_tsc_frequency &&
150 cpuid_vmm_info()->cpuid_vmm_bus_frequency) {
151 busFreq = (uint64_t)cpuid_vmm_info()->cpuid_vmm_bus_frequency * kilo;
152 busFCvtt2n = ((1 * Giga) << 32) / busFreq;
153 busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
154
155 tscFreq = (uint64_t)cpuid_vmm_info()->cpuid_vmm_tsc_frequency * kilo;
156 tscFCvtt2n = ((1 * Giga) << 32) / tscFreq;
157 tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
158
159 tscGranularity = tscFreq / busFreq;
160
161 bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
162
163 return;
164 }
165 }
166
167 switch (cpuid_cpufamily()) {
168 case CPUFAMILY_INTEL_KABYLAKE:
169 case CPUFAMILY_INTEL_ICELAKE:
170 case CPUFAMILY_INTEL_COMETLAKE:
171 case CPUFAMILY_INTEL_SKYLAKE: {
172 /*
173 * SkyLake and later has an Always Running Timer (ART) providing
174 * the reference frequency. CPUID leaf 0x15 determines the
175 * rationship between this and the TSC frequency expressed as
176 * - multiplier (numerator, N), and
177 * - divisor (denominator, M).
178 * So that TSC = ART * N / M.
179 */
180 i386_cpu_info_t *infop = cpuid_info();
181 cpuid_tsc_leaf_t *tsc_leafp = &infop->cpuid_tsc_leaf;
182 uint64_t N = (uint64_t) tsc_leafp->numerator;
183 uint64_t M = (uint64_t) tsc_leafp->denominator;
184 uint64_t refFreq;
185
186 refFreq = EFI_get_frequency("ARTFrequency");
187 if (refFreq == 0) {
188 /*
189 * Intel Scalable Processor (Xeon-SP) CPUs use a different
190 * ART frequency. Use that default here if EFI didn't
191 * specify the frequency. Since Xeon-SP uses the same
192 * DisplayModel / DisplayFamily as Xeon-W, we need to
193 * use the platform ID (or, as XNU calls it, the "processor
194 * flag") to differentiate the two.
195 */
196 if (cpuid_family() == 0x06 &&
197 infop->cpuid_model == CPUID_MODEL_SKYLAKE_W &&
198 is_xeon_sp(infop->cpuid_processor_flag)) {
199 refFreq = BASE_ART_CLOCK_SOURCE_SP;
200 } else {
201 refFreq = BASE_ART_CLOCK_SOURCE;
202 }
203 }
204
205 assert(N != 0);
206 assert(M != 1);
207 tscFreq = refFreq * N / M;
208 busFreq = tscFreq; /* bus is APIC frequency */
209
210 kprintf(" ART: Frequency = %6d.%06dMHz, N/M = %lld/%llu\n",
211 (uint32_t)(refFreq / Mega),
212 (uint32_t)(refFreq % Mega),
213 N, M);
214
215 break;
216 }
217 default: {
218 uint64_t msr_flex_ratio;
219 uint64_t msr_platform_info;
220
221 /* See if FLEX_RATIO is being used */
222 msr_flex_ratio = rdmsr64(MSR_FLEX_RATIO);
223 msr_platform_info = rdmsr64(MSR_PLATFORM_INFO);
224 flex_ratio_min = (uint32_t)bitfield(msr_platform_info, 47, 40);
225 flex_ratio_max = (uint32_t)bitfield(msr_platform_info, 15, 8);
226 /* No BIOS-programed flex ratio. Use hardware max as default */
227 tscGranularity = flex_ratio_max;
228 if (msr_flex_ratio & bit(16)) {
229 /* Flex Enabled: Use this MSR if less than max */
230 flex_ratio = (uint32_t)bitfield(msr_flex_ratio, 15, 8);
231 if (flex_ratio < flex_ratio_max) {
232 tscGranularity = flex_ratio;
233 }
234 }
235
236 busFreq = EFI_get_frequency("FSBFrequency");
237 /* If EFI isn't configured correctly, use a constant
238 * value. See 6036811.
239 */
240 if (busFreq == 0) {
241 busFreq = BASE_NHM_CLOCK_SOURCE;
242 }
243
244 break;
245 }
246 case CPUFAMILY_INTEL_PENRYN: {
247 uint64_t prfsts;
248
249 prfsts = rdmsr64(IA32_PERF_STS);
250 tscGranularity = (uint32_t)bitfield(prfsts, 44, 40);
251 N_by_2_bus_ratio = (prfsts & bit(46)) != 0;
252
253 busFreq = EFI_get_frequency("FSBFrequency");
254 }
255 }
256
257 if (busFreq != 0) {
258 busFCvtt2n = ((1 * Giga) << 32) / busFreq;
259 busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
260 } else {
261 panic("tsc_init: EFI not supported!");
262 }
263
264 kprintf(" BUS: Frequency = %6d.%06dMHz, "
265 "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X\n",
266 (uint32_t)(busFreq / Mega),
267 (uint32_t)(busFreq % Mega),
268 (uint32_t)(busFCvtt2n >> 32), (uint32_t)busFCvtt2n,
269 (uint32_t)(busFCvtn2t >> 32), (uint32_t)busFCvtn2t);
270
271 if (tscFreq == busFreq) {
272 bus2tsc = 1;
273 tscGranularity = 1;
274 tscFCvtn2t = busFCvtn2t;
275 tscFCvtt2n = busFCvtt2n;
276 } else {
277 /*
278 * Get the TSC increment. The TSC is incremented by this
279 * on every bus tick. Calculate the TSC conversion factors
280 * to and from nano-seconds.
281 * The tsc granularity is also called the "bus ratio".
282 * If the N/2 bit is set this indicates the bus ration is
283 * 0.5 more than this - i.e. that the true bus ratio
284 * is (2*tscGranularity + 1)/2.
285 */
286 if (N_by_2_bus_ratio) {
287 tscFCvtt2n = busFCvtt2n * 2 / (1 + 2 * tscGranularity);
288 } else {
289 tscFCvtt2n = busFCvtt2n / tscGranularity;
290 }
291
292 tscFreq = ((1 * Giga) << 32) / tscFCvtt2n;
293 tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
294
295 /*
296 * Calculate conversion from BUS to TSC
297 */
298 bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
299 }
300
301 kprintf(" TSC: Frequency = %6d.%06dMHz, "
302 "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, gran = %lld%s\n",
303 (uint32_t)(tscFreq / Mega),
304 (uint32_t)(tscFreq % Mega),
305 (uint32_t)(tscFCvtt2n >> 32), (uint32_t)tscFCvtt2n,
306 (uint32_t)(tscFCvtn2t >> 32), (uint32_t)tscFCvtn2t,
307 tscGranularity, N_by_2_bus_ratio ? " (N/2)" : "");
308 }
309
310 void
tsc_get_info(tscInfo_t * info)311 tsc_get_info(tscInfo_t *info)
312 {
313 info->busFCvtt2n = busFCvtt2n;
314 info->busFCvtn2t = busFCvtn2t;
315 info->tscFreq = tscFreq;
316 info->tscFCvtt2n = tscFCvtt2n;
317 info->tscFCvtn2t = tscFCvtn2t;
318 info->tscGranularity = tscGranularity;
319 info->bus2tsc = bus2tsc;
320 info->busFreq = busFreq;
321 info->flex_ratio = flex_ratio;
322 info->flex_ratio_min = flex_ratio_min;
323 info->flex_ratio_max = flex_ratio_max;
324 }
325
326 #if DEVELOPMENT || DEBUG
327 void
cpu_data_tsc_sync_deltas_string(char * buf,uint32_t buflen,uint32_t start_cpu,uint32_t end_cpu)328 cpu_data_tsc_sync_deltas_string(char *buf, uint32_t buflen,
329 uint32_t start_cpu, uint32_t end_cpu)
330 {
331 int cnt;
332 uint32_t offset = 0;
333
334 if (start_cpu >= real_ncpus || end_cpu >= real_ncpus) {
335 if (buflen >= 1) {
336 buf[0] = 0;
337 }
338 return;
339 }
340
341 for (uint32_t curcpu = start_cpu; curcpu <= end_cpu; curcpu++) {
342 cnt = snprintf(buf + offset, buflen - offset, "0x%llx ", cpu_datap(curcpu)->tsc_sync_delta);
343 if (cnt < 0 || (offset + (unsigned) cnt >= buflen)) {
344 break;
345 }
346 offset += cnt;
347 }
348 if (offset >= 1) {
349 buf[offset - 1] = 0; /* Clip the final, trailing space */
350 }
351 }
352 #endif /* DEVELOPMENT || DEBUG */
353