1 /*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * @APPLE_FREE_COPYRIGHT@
33 */
34 /*
35 * File: arm/rtclock.c
36 * Purpose: Routines for handling the machine dependent
37 * real-time clock.
38 */
39
40 #include <mach/mach_types.h>
41
42 #include <kern/clock.h>
43 #include <kern/thread.h>
44 #include <kern/macro_help.h>
45 #include <kern/spl.h>
46 #include <kern/timer_queue.h>
47
48 #include <kern/host_notify.h>
49
50 #include <machine/commpage.h>
51 #include <machine/machine_routines.h>
52 #include <machine/config.h>
53 #include <arm/exception.h>
54 #include <arm/cpu_data_internal.h>
55 #if __arm64__
56 #include <arm64/proc_reg.h>
57 #elif __arm__
58 #include <arm/proc_reg.h>
59 #else
60 #error Unsupported arch
61 #endif
62 #include <arm/rtclock.h>
63
64 #include <IOKit/IOPlatformExpert.h>
65 #include <libkern/OSAtomic.h>
66
67 #include <sys/kdebug.h>
68
69 #define MAX_TIMEBASE_TRIES 10
70
71 int rtclock_init(void);
72
73 static int
74 deadline_to_decrementer(uint64_t deadline,
75 uint64_t now);
76 static void
77 timebase_callback(struct timebase_freq_t * freq);
78
79 #if DEVELOPMENT || DEBUG
80 uint32_t timebase_validation = 0;
81 #endif
82
83 /*
84 * Configure the real-time clock device at boot
85 */
86 void
rtclock_early_init(void)87 rtclock_early_init(void)
88 {
89 PE_register_timebase_callback(timebase_callback);
90 #if DEVELOPMENT || DEBUG
91 uint32_t tmp_mv = 1;
92
93 #if defined(APPLE_ARM64_ARCH_FAMILY)
94 /* Enable MAT validation on A0 hardware by default. */
95 timebase_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0;
96 #endif
97
98 if (kern_feature_override(KF_MATV_OVRD)) {
99 timebase_validation = 0;
100 }
101 if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
102 timebase_validation = tmp_mv;
103 }
104 #endif
105 }
106
107 static void
timebase_callback(struct timebase_freq_t * freq)108 timebase_callback(struct timebase_freq_t * freq)
109 {
110 unsigned long numer, denom;
111 uint64_t t64_1, t64_2;
112 uint32_t divisor;
113
114 if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
115 freq->timebase_num < freq->timebase_den) {
116 panic("rtclock timebase_callback: invalid constant %ld / %ld",
117 freq->timebase_num, freq->timebase_den);
118 }
119
120 denom = freq->timebase_num;
121 numer = freq->timebase_den * NSEC_PER_SEC;
122 // reduce by the greatest common denominator to minimize overflow
123 if (numer > denom) {
124 t64_1 = numer;
125 t64_2 = denom;
126 } else {
127 t64_1 = denom;
128 t64_2 = numer;
129 }
130 while (t64_2 != 0) {
131 uint64_t temp = t64_2;
132 t64_2 = t64_1 % t64_2;
133 t64_1 = temp;
134 }
135 numer /= t64_1;
136 denom /= t64_1;
137
138 rtclock_timebase_const.numer = (uint32_t)numer;
139 rtclock_timebase_const.denom = (uint32_t)denom;
140 divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
141
142 rtclock_sec_divisor = divisor;
143 rtclock_usec_divisor = divisor / USEC_PER_SEC;
144 }
145
146 /*
147 * Initialize the system clock device for the current cpu
148 */
149 int
rtclock_init(void)150 rtclock_init(void)
151 {
152 uint64_t abstime;
153 cpu_data_t * cdp;
154
155 clock_timebase_init();
156 ml_init_lock_timeout();
157
158 cdp = getCpuDatap();
159
160 abstime = mach_absolute_time();
161 cdp->rtcPop = EndOfAllTime; /* Init Pop time */
162 timer_resync_deadlines(); /* Start the timers going */
163
164 return 1;
165 }
166
167
168 uint64_t
mach_absolute_time(void)169 mach_absolute_time(void)
170 {
171 #if DEVELOPMENT || DEBUG
172 if (__improbable(timebase_validation)) {
173 static volatile uint64_t s_last_absolute_time = 0;
174 uint64_t new_absolute_time, old_absolute_time;
175 int attempts = 0;
176
177 /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
178 * completes before the timebase read. Were the load to complete after the
179 * timebase read, there would be a window for another CPU to update
180 * s_last_absolute_time and leave us in an inconsistent state. Consider the
181 * following interleaving:
182 *
183 * Let s_last_absolute_time = t0
184 * CPU0: Read timebase at t1
185 * CPU1: Read timebase at t2
186 * CPU1: Update s_last_absolute_time to t2
187 * CPU0: Load completes
188 * CPU0: Update s_last_absolute_time to t1
189 *
190 * This would cause the assertion to fail even though time did not go
191 * backwards. Thus, we use a dsb to guarantee completion of the load before
192 * the timebase read.
193 */
194 do {
195 attempts++;
196 old_absolute_time = s_last_absolute_time;
197
198 __builtin_arm_dsb(DSB_ISHLD);
199
200 new_absolute_time = ml_get_timebase();
201 } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
202
203 if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
204 timebase_validation = 0; // we know it's bad, now prevent nested panics
205 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)",
206 new_absolute_time, old_absolute_time);
207 }
208 return new_absolute_time;
209 }
210 #endif /* DEVELOPMENT || DEBUG */
211
212 return ml_get_timebase();
213 }
214
215 uint64_t
mach_approximate_time(void)216 mach_approximate_time(void)
217 {
218 #if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
219 /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
220 return ml_get_timebase();
221 #else
222 processor_t processor;
223 uint64_t approx_time;
224
225 disable_preemption();
226 processor = current_processor();
227 approx_time = processor->last_dispatch;
228 enable_preemption();
229
230 return approx_time;
231 #endif
232 }
233
234 void
clock_get_system_microtime(clock_sec_t * secs,clock_usec_t * microsecs)235 clock_get_system_microtime(clock_sec_t * secs,
236 clock_usec_t * microsecs)
237 {
238 absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
239 }
240
241 void
clock_get_system_nanotime(clock_sec_t * secs,clock_nsec_t * nanosecs)242 clock_get_system_nanotime(clock_sec_t * secs,
243 clock_nsec_t * nanosecs)
244 {
245 uint64_t abstime;
246 uint64_t t64;
247
248 abstime = mach_absolute_time();
249 *secs = (t64 = abstime / rtclock_sec_divisor);
250 abstime -= (t64 * rtclock_sec_divisor);
251
252 *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
253 }
254
255 void
clock_gettimeofday_set_commpage(uint64_t abstime,uint64_t sec,uint64_t frac,uint64_t scale,uint64_t tick_per_sec)256 clock_gettimeofday_set_commpage(uint64_t abstime,
257 uint64_t sec,
258 uint64_t frac,
259 uint64_t scale,
260 uint64_t tick_per_sec)
261 {
262 commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
263 }
264
265 void
clock_timebase_info(mach_timebase_info_t info)266 clock_timebase_info(mach_timebase_info_t info)
267 {
268 *info = rtclock_timebase_const;
269 }
270
271 /*
272 * Real-time clock device interrupt.
273 */
274 void
rtclock_intr(__unused unsigned int is_user_context)275 rtclock_intr(__unused unsigned int is_user_context)
276 {
277 uint64_t abstime;
278 cpu_data_t * cdp;
279 struct arm_saved_state * regs;
280 unsigned int user_mode;
281 uintptr_t pc;
282
283 cdp = getCpuDatap();
284
285 cdp->cpu_stat.timer_cnt++;
286 SCHED_STATS_INC(timer_pop_count);
287
288 assert(!ml_get_interrupts_enabled());
289
290 abstime = mach_absolute_time();
291
292 if (cdp->cpu_idle_pop != 0x0ULL) {
293 if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) {
294 cdp->cpu_idle_pop = 0x0ULL;
295 while (abstime < cdp->rtcPop) {
296 abstime = mach_absolute_time();
297 }
298 } else {
299 ClearIdlePop(FALSE);
300 }
301 }
302
303 if ((regs = cdp->cpu_int_state)) {
304 pc = get_saved_state_pc(regs);
305
306 #if __arm64__
307 user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
308 #else
309 user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE;
310 #endif
311 } else {
312 pc = 0;
313 user_mode = 0;
314 }
315 if (abstime >= cdp->rtcPop) {
316 /* Log the interrupt service latency (-ve value expected by tool) */
317 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
318 MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
319 -(abstime - cdp->rtcPop),
320 user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0);
321 }
322
323 /* call the generic etimer */
324 timer_intr(user_mode, pc);
325 }
326
327 static int
deadline_to_decrementer(uint64_t deadline,uint64_t now)328 deadline_to_decrementer(uint64_t deadline,
329 uint64_t now)
330 {
331 uint64_t delt;
332
333 if (deadline <= now) {
334 return DECREMENTER_MIN;
335 } else {
336 delt = deadline - now;
337
338 return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
339 }
340 }
341
342 /*
343 * Request a decrementer pop
344 */
345 int
setPop(uint64_t time)346 setPop(uint64_t time)
347 {
348 int delay_time;
349 uint64_t current_time;
350 cpu_data_t * cdp;
351
352 cdp = getCpuDatap();
353 current_time = mach_absolute_time();
354
355 delay_time = deadline_to_decrementer(time, current_time);
356 cdp->rtcPop = delay_time + current_time;
357
358 ml_set_decrementer((uint32_t) delay_time);
359
360 return delay_time;
361 }
362
363 /*
364 * Request decrementer Idle Pop. Return true if set
365 */
366 boolean_t
SetIdlePop(void)367 SetIdlePop(void)
368 {
369 int delay_time;
370 uint64_t time;
371 uint64_t current_time;
372 cpu_data_t * cdp;
373
374 cdp = getCpuDatap();
375 current_time = mach_absolute_time();
376
377 if (((cdp->rtcPop < current_time) ||
378 (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) {
379 return FALSE;
380 }
381
382 time = cdp->rtcPop - cdp->cpu_idle_latency;
383
384 delay_time = deadline_to_decrementer(time, current_time);
385 cdp->cpu_idle_pop = delay_time + current_time;
386 ml_set_decrementer((uint32_t) delay_time);
387
388 return TRUE;
389 }
390
391 /*
392 * Clear decrementer Idle Pop
393 */
394 void
ClearIdlePop(boolean_t wfi)395 ClearIdlePop(
396 boolean_t wfi)
397 {
398 #if !__arm64__
399 #pragma unused(wfi)
400 #endif
401 cpu_data_t * cdp;
402
403 cdp = getCpuDatap();
404 cdp->cpu_idle_pop = 0x0ULL;
405
406 #if __arm64__
407 /*
408 * Don't update the HW timer if there's a pending
409 * interrupt (we can lose interrupt assertion);
410 * we want to take the interrupt right now and update
411 * the deadline from the handler).
412 *
413 * ARM64_TODO: consider this more carefully.
414 */
415 if (!(wfi && ml_get_timer_pending()))
416 #endif
417 {
418 setPop(cdp->rtcPop);
419 }
420 }
421
422 void
absolutetime_to_microtime(uint64_t abstime,clock_sec_t * secs,clock_usec_t * microsecs)423 absolutetime_to_microtime(uint64_t abstime,
424 clock_sec_t * secs,
425 clock_usec_t * microsecs)
426 {
427 uint64_t t64;
428
429 *secs = t64 = abstime / rtclock_sec_divisor;
430 abstime -= (t64 * rtclock_sec_divisor);
431
432 *microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
433 }
434
435 void
absolutetime_to_nanoseconds(uint64_t abstime,uint64_t * result)436 absolutetime_to_nanoseconds(uint64_t abstime,
437 uint64_t * result)
438 {
439 uint64_t t64;
440
441 *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
442 abstime -= (t64 * rtclock_sec_divisor);
443 *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
444 }
445
446 void
nanoseconds_to_absolutetime(uint64_t nanosecs,uint64_t * result)447 nanoseconds_to_absolutetime(uint64_t nanosecs,
448 uint64_t * result)
449 {
450 uint64_t t64;
451
452 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
453 nanosecs -= (t64 * NSEC_PER_SEC);
454 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
455 }
456
457 void
nanotime_to_absolutetime(clock_sec_t secs,clock_nsec_t nanosecs,uint64_t * result)458 nanotime_to_absolutetime(clock_sec_t secs,
459 clock_nsec_t nanosecs,
460 uint64_t * result)
461 {
462 *result = ((uint64_t) secs * rtclock_sec_divisor) +
463 ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
464 }
465
466 void
clock_interval_to_absolutetime_interval(uint32_t interval,uint32_t scale_factor,uint64_t * result)467 clock_interval_to_absolutetime_interval(uint32_t interval,
468 uint32_t scale_factor,
469 uint64_t * result)
470 {
471 uint64_t nanosecs = (uint64_t) interval * scale_factor;
472 uint64_t t64;
473
474 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
475 nanosecs -= (t64 * NSEC_PER_SEC);
476 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
477 }
478
479 void
machine_delay_until(uint64_t interval,uint64_t deadline)480 machine_delay_until(uint64_t interval,
481 uint64_t deadline)
482 {
483 #pragma unused(interval)
484 uint64_t now;
485
486 do {
487 __builtin_arm_wfe();
488 now = mach_absolute_time();
489 } while (now < deadline);
490 }
491