1 /*
2 * Copyright (c) 2005-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 /*
33 * File: i386/tsc.c
34 * Purpose: Initializes the TSC and the various conversion
35 * factors needed by other parts of the system.
36 */
37
38
39 #include <mach/mach_types.h>
40
41 #include <kern/cpu_data.h>
42 #include <kern/cpu_number.h>
43 #include <kern/clock.h>
44 #include <kern/host_notify.h>
45 #include <kern/macro_help.h>
46 #include <kern/misc_protos.h>
47 #include <kern/spl.h>
48 #include <kern/assert.h>
49 #include <mach/vm_prot.h>
50 #include <vm/pmap.h>
51 #include <vm/vm_kern.h> /* for kernel_map */
52 #include <architecture/i386/pio.h>
53 #include <i386/machine_cpu.h>
54 #include <i386/cpuid.h>
55 #include <i386/mp.h>
56 #include <i386/machine_routines.h>
57 #include <i386/proc_reg.h>
58 #include <i386/tsc.h>
59 #include <i386/misc_protos.h>
60 #include <pexpert/pexpert.h>
61 #include <machine/limits.h>
62 #include <machine/commpage.h>
63 #include <sys/kdebug.h>
64 #include <pexpert/device_tree.h>
65
66 uint64_t busFCvtt2n = 0;
67 uint64_t busFCvtn2t = 0;
68 uint64_t tscFreq = 0;
69 uint64_t tscFCvtt2n = 0;
70 uint64_t tscFCvtn2t = 0;
71 uint64_t tscGranularity = 0;
72 uint64_t bus2tsc = 0;
73 uint64_t busFreq = 0;
74 uint32_t flex_ratio = 0;
75 uint32_t flex_ratio_min = 0;
76 uint32_t flex_ratio_max = 0;
77
78 uint64_t tsc_at_boot = 0;
79
80 #define bit(n) (1ULL << (n))
81 #define bitmask(h, l) ((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
82 #define bitfield(x, h, l) (((x) & bitmask(h,l)) >> l)
83
84 /* Decimal powers: */
85 #define kilo (1000ULL)
86 #define Mega (kilo * kilo)
87 #define Giga (kilo * Mega)
88 #define Tera (kilo * Giga)
89 #define Peta (kilo * Tera)
90
91 #define CPU_FAMILY_PENTIUM_M (0x6)
92
93 /*
94 * This routine extracts a frequency property in Hz from the device tree.
95 * Also reads any initial TSC value at boot from the device tree.
96 */
97 static uint64_t
EFI_get_frequency(const char * prop)98 EFI_get_frequency(const char *prop)
99 {
100 uint64_t frequency = 0;
101 DTEntry entry;
102 void const *value;
103 unsigned int size;
104
105 if (SecureDTLookupEntry(0, "/efi/platform", &entry) != kSuccess) {
106 kprintf("EFI_get_frequency: didn't find /efi/platform\n");
107 return 0;
108 }
109
110 /*
111 * While we're here, see if EFI published an initial TSC value.
112 */
113 if (SecureDTGetProperty(entry, "InitialTSC", &value, &size) == kSuccess) {
114 if (size == sizeof(uint64_t)) {
115 tsc_at_boot = *(uint64_t const *) value;
116 kprintf("EFI_get_frequency: read InitialTSC: %llu\n",
117 tsc_at_boot);
118 }
119 }
120
121 if (SecureDTGetProperty(entry, prop, &value, &size) != kSuccess) {
122 kprintf("EFI_get_frequency: property %s not found\n", prop);
123 return 0;
124 }
125 if (size == sizeof(uint64_t)) {
126 frequency = *(uint64_t const *) value;
127 kprintf("EFI_get_frequency: read %s value: %llu\n",
128 prop, frequency);
129 }
130
131 return frequency;
132 }
133
134 /*
135 * Initialize the various conversion factors needed by code referencing
136 * the TSC.
137 */
138 void
tsc_init(void)139 tsc_init(void)
140 {
141 boolean_t N_by_2_bus_ratio = FALSE;
142
143 if (cpuid_vmm_present()) {
144 kprintf("VMM vendor %s TSC frequency %u KHz bus frequency %u KHz\n",
145 cpuid_vmm_family_string(),
146 cpuid_vmm_info()->cpuid_vmm_tsc_frequency,
147 cpuid_vmm_info()->cpuid_vmm_bus_frequency);
148
149 if (cpuid_vmm_info()->cpuid_vmm_tsc_frequency &&
150 cpuid_vmm_info()->cpuid_vmm_bus_frequency) {
151 busFreq = (uint64_t)cpuid_vmm_info()->cpuid_vmm_bus_frequency * kilo;
152 busFCvtt2n = ((1 * Giga) << 32) / busFreq;
153 busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
154
155 tscFreq = (uint64_t)cpuid_vmm_info()->cpuid_vmm_tsc_frequency * kilo;
156 tscFCvtt2n = ((1 * Giga) << 32) / tscFreq;
157 tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
158
159 tscGranularity = tscFreq / busFreq;
160
161 bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
162
163 return;
164 }
165 }
166
167 switch (cpuid_cpufamily()) {
168 case CPUFAMILY_INTEL_KABYLAKE:
169 case CPUFAMILY_INTEL_ICELAKE:
170 case CPUFAMILY_INTEL_SKYLAKE: {
171 /*
172 * SkyLake and later has an Always Running Timer (ART) providing
173 * the reference frequency. CPUID leaf 0x15 determines the
174 * rationship between this and the TSC frequency expressed as
175 * - multiplier (numerator, N), and
176 * - divisor (denominator, M).
177 * So that TSC = ART * N / M.
178 */
179 i386_cpu_info_t *infop = cpuid_info();
180 cpuid_tsc_leaf_t *tsc_leafp = &infop->cpuid_tsc_leaf;
181 uint64_t N = (uint64_t) tsc_leafp->numerator;
182 uint64_t M = (uint64_t) tsc_leafp->denominator;
183 uint64_t refFreq;
184
185 refFreq = EFI_get_frequency("ARTFrequency");
186 if (refFreq == 0) {
187 /*
188 * Intel Scalable Processor (Xeon-SP) CPUs use a different
189 * ART frequency. Use that default here if EFI didn't
190 * specify the frequency. Since Xeon-SP uses the same
191 * DisplayModel / DisplayFamily as Xeon-W, we need to
192 * use the platform ID (or, as XNU calls it, the "processor
193 * flag") to differentiate the two.
194 */
195 if (cpuid_family() == 0x06 &&
196 infop->cpuid_model == CPUID_MODEL_SKYLAKE_W &&
197 is_xeon_sp(infop->cpuid_processor_flag)) {
198 refFreq = BASE_ART_CLOCK_SOURCE_SP;
199 } else {
200 refFreq = BASE_ART_CLOCK_SOURCE;
201 }
202 }
203
204 assert(N != 0);
205 assert(M != 1);
206 tscFreq = refFreq * N / M;
207 busFreq = tscFreq; /* bus is APIC frequency */
208
209 kprintf(" ART: Frequency = %6d.%06dMHz, N/M = %lld/%llu\n",
210 (uint32_t)(refFreq / Mega),
211 (uint32_t)(refFreq % Mega),
212 N, M);
213
214 break;
215 }
216 default: {
217 uint64_t msr_flex_ratio;
218 uint64_t msr_platform_info;
219
220 /* See if FLEX_RATIO is being used */
221 msr_flex_ratio = rdmsr64(MSR_FLEX_RATIO);
222 msr_platform_info = rdmsr64(MSR_PLATFORM_INFO);
223 flex_ratio_min = (uint32_t)bitfield(msr_platform_info, 47, 40);
224 flex_ratio_max = (uint32_t)bitfield(msr_platform_info, 15, 8);
225 /* No BIOS-programed flex ratio. Use hardware max as default */
226 tscGranularity = flex_ratio_max;
227 if (msr_flex_ratio & bit(16)) {
228 /* Flex Enabled: Use this MSR if less than max */
229 flex_ratio = (uint32_t)bitfield(msr_flex_ratio, 15, 8);
230 if (flex_ratio < flex_ratio_max) {
231 tscGranularity = flex_ratio;
232 }
233 }
234
235 busFreq = EFI_get_frequency("FSBFrequency");
236 /* If EFI isn't configured correctly, use a constant
237 * value. See 6036811.
238 */
239 if (busFreq == 0) {
240 busFreq = BASE_NHM_CLOCK_SOURCE;
241 }
242
243 break;
244 }
245 case CPUFAMILY_INTEL_PENRYN: {
246 uint64_t prfsts;
247
248 prfsts = rdmsr64(IA32_PERF_STS);
249 tscGranularity = (uint32_t)bitfield(prfsts, 44, 40);
250 N_by_2_bus_ratio = (prfsts & bit(46)) != 0;
251
252 busFreq = EFI_get_frequency("FSBFrequency");
253 }
254 }
255
256 if (busFreq != 0) {
257 busFCvtt2n = ((1 * Giga) << 32) / busFreq;
258 busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
259 } else {
260 panic("tsc_init: EFI not supported!");
261 }
262
263 kprintf(" BUS: Frequency = %6d.%06dMHz, "
264 "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X\n",
265 (uint32_t)(busFreq / Mega),
266 (uint32_t)(busFreq % Mega),
267 (uint32_t)(busFCvtt2n >> 32), (uint32_t)busFCvtt2n,
268 (uint32_t)(busFCvtn2t >> 32), (uint32_t)busFCvtn2t);
269
270 if (tscFreq == busFreq) {
271 bus2tsc = 1;
272 tscGranularity = 1;
273 tscFCvtn2t = busFCvtn2t;
274 tscFCvtt2n = busFCvtt2n;
275 } else {
276 /*
277 * Get the TSC increment. The TSC is incremented by this
278 * on every bus tick. Calculate the TSC conversion factors
279 * to and from nano-seconds.
280 * The tsc granularity is also called the "bus ratio".
281 * If the N/2 bit is set this indicates the bus ration is
282 * 0.5 more than this - i.e. that the true bus ratio
283 * is (2*tscGranularity + 1)/2.
284 */
285 if (N_by_2_bus_ratio) {
286 tscFCvtt2n = busFCvtt2n * 2 / (1 + 2 * tscGranularity);
287 } else {
288 tscFCvtt2n = busFCvtt2n / tscGranularity;
289 }
290
291 tscFreq = ((1 * Giga) << 32) / tscFCvtt2n;
292 tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
293
294 /*
295 * Calculate conversion from BUS to TSC
296 */
297 bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
298 }
299
300 kprintf(" TSC: Frequency = %6d.%06dMHz, "
301 "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, gran = %lld%s\n",
302 (uint32_t)(tscFreq / Mega),
303 (uint32_t)(tscFreq % Mega),
304 (uint32_t)(tscFCvtt2n >> 32), (uint32_t)tscFCvtt2n,
305 (uint32_t)(tscFCvtn2t >> 32), (uint32_t)tscFCvtn2t,
306 tscGranularity, N_by_2_bus_ratio ? " (N/2)" : "");
307 }
308
309 void
tsc_get_info(tscInfo_t * info)310 tsc_get_info(tscInfo_t *info)
311 {
312 info->busFCvtt2n = busFCvtt2n;
313 info->busFCvtn2t = busFCvtn2t;
314 info->tscFreq = tscFreq;
315 info->tscFCvtt2n = tscFCvtt2n;
316 info->tscFCvtn2t = tscFCvtn2t;
317 info->tscGranularity = tscGranularity;
318 info->bus2tsc = bus2tsc;
319 info->busFreq = busFreq;
320 info->flex_ratio = flex_ratio;
321 info->flex_ratio_min = flex_ratio_min;
322 info->flex_ratio_max = flex_ratio_max;
323 }
324
325 #if DEVELOPMENT || DEBUG
326 void
cpu_data_tsc_sync_deltas_string(char * buf,uint32_t buflen,uint32_t start_cpu,uint32_t end_cpu)327 cpu_data_tsc_sync_deltas_string(char *buf, uint32_t buflen,
328 uint32_t start_cpu, uint32_t end_cpu)
329 {
330 int cnt;
331 uint32_t offset = 0;
332
333 if (start_cpu >= real_ncpus || end_cpu >= real_ncpus) {
334 if (buflen >= 1) {
335 buf[0] = 0;
336 }
337 return;
338 }
339
340 for (uint32_t curcpu = start_cpu; curcpu <= end_cpu; curcpu++) {
341 cnt = snprintf(buf + offset, buflen - offset, "0x%llx ", cpu_datap(curcpu)->tsc_sync_delta);
342 if (cnt < 0 || (offset + (unsigned) cnt >= buflen)) {
343 break;
344 }
345 offset += cnt;
346 }
347 if (offset >= 1) {
348 buf[offset - 1] = 0; /* Clip the final, trailing space */
349 }
350 }
351 #endif /* DEVELOPMENT || DEBUG */
352