/* * Copyright (c) 2000-2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* */ /*- * Copyright (c) 1982, 1986, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)time.h 8.5 (Berkeley) 5/4/95 * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if HIBERNATION && HAS_CONTINUOUS_HWCLOCK // On ARM64, the hwclock keeps ticking across a normal S2R so we use it to reset the // system clock after a normal wake. However, on hibernation we cut power to the hwclock, // so we have to add an offset to the hwclock to compute continuous_time after hibernate resume. uint64_t hwclock_conttime_offset = 0; #endif /* HIBERNATION && HAS_CONTINUOUS_HWCLOCK */ #if HIBERNATION_USES_LEGACY_CLOCK || !HAS_CONTINUOUS_HWCLOCK #define ENABLE_LEGACY_CLOCK_CODE 1 #endif /* HIBERNATION_USES_LEGACY_CLOCK || !HAS_CONTINUOUS_HWCLOCK */ #if HIBERNATION_USES_LEGACY_CLOCK #include #endif /* HIBERNATION_USES_LEGACY_CLOCK */ uint32_t hz_tick_interval = 1; #if ENABLE_LEGACY_CLOCK_CODE static uint64_t has_monotonic_clock = 0; #endif /* ENABLE_LEGACY_CLOCK_CODE */ lck_ticket_t clock_lock; LCK_GRP_DECLARE(clock_lock_grp, "clock"); static LCK_GRP_DECLARE(settime_lock_grp, "settime"); static LCK_MTX_DECLARE(settime_lock, &settime_lock_grp); #define clock_lock() \ lck_ticket_lock(&clock_lock, &clock_lock_grp) #define clock_unlock() \ lck_ticket_unlock(&clock_lock) boolean_t kdp_clock_is_locked() { return kdp_lck_ticket_is_acquired(&clock_lock); } struct bintime { time_t sec; uint64_t frac; }; static __inline void bintime_addx(struct bintime *_bt, uint64_t _x) { uint64_t _u; _u = _bt->frac; _bt->frac += _x; if (_u > _bt->frac) { _bt->sec++; } } static __inline void bintime_subx(struct bintime *_bt, uint64_t _x) { uint64_t _u; _u = _bt->frac; _bt->frac -= _x; if (_u < _bt->frac) { _bt->sec--; } } static __inline void bintime_addns(struct bintime *bt, uint64_t ns) { bt->sec += ns / (uint64_t)NSEC_PER_SEC; ns = ns % (uint64_t)NSEC_PER_SEC; if (ns) { /* 18446744073 = int(2^64 / NSEC_PER_SEC) */ ns = ns * (uint64_t)18446744073LL; bintime_addx(bt, ns); } } static __inline void bintime_subns(struct bintime *bt, uint64_t ns) { bt->sec -= ns / (uint64_t)NSEC_PER_SEC; ns = ns % (uint64_t)NSEC_PER_SEC; if (ns) { /* 18446744073 = int(2^64 / NSEC_PER_SEC) */ ns = ns * (uint64_t)18446744073LL; bintime_subx(bt, ns); } } static __inline void bintime_addxns(struct bintime *bt, uint64_t a, int64_t xns) { uint64_t uxns = (xns > 0)?(uint64_t)xns:(uint64_t)-xns; uint64_t ns = multi_overflow(a, uxns); if (xns > 0) { if (ns) { bintime_addns(bt, ns); } ns = (a * uxns) / (uint64_t)NSEC_PER_SEC; bintime_addx(bt, ns); } else { if (ns) { bintime_subns(bt, ns); } ns = (a * uxns) / (uint64_t)NSEC_PER_SEC; bintime_subx(bt, ns); } } static __inline void bintime_add(struct bintime *_bt, const struct bintime *_bt2) { uint64_t _u; _u = _bt->frac; _bt->frac += _bt2->frac; if (_u > _bt->frac) { _bt->sec++; } _bt->sec += _bt2->sec; } static __inline void bintime_sub(struct bintime *_bt, const struct bintime *_bt2) { uint64_t _u; _u = _bt->frac; _bt->frac -= _bt2->frac; if (_u < _bt->frac) { _bt->sec--; } _bt->sec -= _bt2->sec; } static __inline void clock2bintime(const clock_sec_t *secs, const clock_usec_t *microsecs, struct bintime *_bt) { _bt->sec = *secs; /* 18446744073709 = int(2^64 / 1000000) */ _bt->frac = *microsecs * (uint64_t)18446744073709LL; } static __inline void bintime2usclock(const struct bintime *_bt, clock_sec_t *secs, clock_usec_t *microsecs) { *secs = _bt->sec; *microsecs = ((uint64_t)USEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32; } static __inline void bintime2nsclock(const struct bintime *_bt, clock_sec_t *secs, clock_usec_t *nanosecs) { *secs = _bt->sec; *nanosecs = ((uint64_t)NSEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32; } #if ENABLE_LEGACY_CLOCK_CODE static __inline void bintime2absolutetime(const struct bintime *_bt, uint64_t *abs) { uint64_t nsec; nsec = (uint64_t) _bt->sec * (uint64_t)NSEC_PER_SEC + (((uint64_t)NSEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32); nanoseconds_to_absolutetime(nsec, abs); } struct latched_time { uint64_t monotonic_time_usec; uint64_t mach_time; }; extern int kernel_sysctlbyname(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); #endif /* ENABLE_LEGACY_CLOCK_CODE */ /* * Time of day (calendar) variables. * * Algorithm: * * TOD <- bintime + delta*scale * * where : * bintime is a cumulative offset that includes bootime and scaled time elapsed betweed bootime and last scale update. * delta is ticks elapsed since last scale update. * scale is computed according to an adjustment provided by ntp_kern. */ static struct clock_calend { uint64_t s_scale_ns; /* scale to apply for each second elapsed, it converts in ns */ int64_t s_adj_nsx; /* additional adj to apply for each second elapsed, it is expressed in 64 bit frac of ns */ uint64_t tick_scale_x; /* scale to apply for each tick elapsed, it converts in 64 bit frac of s */ uint64_t offset_count; /* abs time from which apply current scales */ struct bintime offset; /* cumulative offset expressed in (sec, 64 bits frac of a second) */ struct bintime bintime; /* cumulative offset (it includes bootime) expressed in (sec, 64 bits frac of a second) */ struct bintime boottime; /* boot time expressed in (sec, 64 bits frac of a second) */ #if ENABLE_LEGACY_CLOCK_CODE struct bintime basesleep; #endif /* ENABLE_LEGACY_CLOCK_CODE */ } clock_calend; static uint64_t ticks_per_sec; /* ticks in a second (expressed in abs time) */ #if DEVELOPMENT || DEBUG extern int g_should_log_clock_adjustments; static void print_all_clock_variables(const char*, clock_sec_t* pmu_secs, clock_usec_t* pmu_usec, clock_sec_t* sys_secs, clock_usec_t* sys_usec, struct clock_calend* calend_cp); static void print_all_clock_variables_internal(const char *, struct clock_calend* calend_cp); #else #define print_all_clock_variables(...) do { } while (0) #define print_all_clock_variables_internal(...) do { } while (0) #endif #if CONFIG_DTRACE /* * Unlocked calendar flipflop; this is used to track a clock_calend such * that we can safely access a snapshot of a valid clock_calend structure * without needing to take any locks to do it. * * The trick is to use a generation count and set the low bit when it is * being updated/read; by doing this, we guarantee, through use of the * os_atomic functions, that the generation is incremented when the bit * is cleared atomically (by using a 1 bit add). */ static struct unlocked_clock_calend { struct clock_calend calend; /* copy of calendar */ uint32_t gen; /* generation count */ } flipflop[2]; static void clock_track_calend_nowait(void); #endif void _clock_delay_until_deadline(uint64_t interval, uint64_t deadline); void _clock_delay_until_deadline_with_leeway(uint64_t interval, uint64_t deadline, uint64_t leeway); /* Boottime variables*/ static uint64_t clock_boottime; static uint32_t clock_boottime_usec; #define TIME_ADD(rsecs, secs, rfrac, frac, unit) \ MACRO_BEGIN \ if (((rfrac) += (frac)) >= (unit)) { \ (rfrac) -= (unit); \ (rsecs) += 1; \ } \ (rsecs) += (secs); \ MACRO_END #define TIME_SUB(rsecs, secs, rfrac, frac, unit) \ MACRO_BEGIN \ if ((int)((rfrac) -= (frac)) < 0) { \ (rfrac) += (unit); \ (rsecs) -= 1; \ } \ (rsecs) -= (secs); \ MACRO_END /* * clock_config: * * Called once at boot to configure the clock subsystem. */ void clock_config(void) { lck_ticket_init(&clock_lock, &clock_lock_grp); clock_oldconfig(); ntp_init(); nanoseconds_to_absolutetime((uint64_t)NSEC_PER_SEC, &ticks_per_sec); } /* * clock_init: * * Called on a processor each time started. */ void clock_init(void) { clock_oldinit(); } /* * clock_timebase_init: * * Called by machine dependent code * to initialize areas dependent on the * timebase value. May be called multiple * times during start up. */ void clock_timebase_init(void) { uint64_t abstime; /* * BSD expects a tick to represent 10ms. */ nanoseconds_to_absolutetime(NSEC_PER_SEC / 100, &abstime); hz_tick_interval = (uint32_t)abstime; sched_timebase_init(); } /* * mach_timebase_info_trap: * * User trap returns timebase constant. */ kern_return_t mach_timebase_info_trap( struct mach_timebase_info_trap_args *args) { mach_vm_address_t out_info_addr = args->info; mach_timebase_info_data_t info = {}; clock_timebase_info(&info); copyout((void *)&info, out_info_addr, sizeof(info)); return KERN_SUCCESS; } /* * Calendar routines. */ /* * clock_get_calendar_microtime: * * Returns the current calendar value, * microseconds as the fraction. */ void clock_get_calendar_microtime( clock_sec_t *secs, clock_usec_t *microsecs) { clock_get_calendar_absolute_and_microtime(secs, microsecs, NULL); } /* * get_scale_factors_from_adj: * * computes scale factors from the value given in adjustment. * * Part of the code has been taken from tc_windup of FreeBSD * written by Poul-Henning Kamp , Julien Ridoux and * Konstantin Belousov. * https://github.com/freebsd/freebsd/blob/master/sys/kern/kern_tc.c */ static void get_scale_factors_from_adj(int64_t adjustment, uint64_t* tick_scale_x, uint64_t* s_scale_ns, int64_t* s_adj_nsx) { uint64_t scale; int64_t nano, frac; /*- * Calculating the scaling factor. We want the number of 1/2^64 * fractions of a second per period of the hardware counter, taking * into account the th_adjustment factor which the NTP PLL/adjtime(2) * processing provides us with. * * The th_adjustment is nanoseconds per second with 32 bit binary * fraction and we want 64 bit binary fraction of second: * * x = a * 2^32 / 10^9 = a * 4.294967296 * * The range of th_adjustment is +/- 5000PPM so inside a 64bit int * we can only multiply by about 850 without overflowing, that * leaves no suitably precise fractions for multiply before divide. * * Divide before multiply with a fraction of 2199/512 results in a * systematic undercompensation of 10PPM of th_adjustment. On a * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. * * We happily sacrifice the lowest of the 64 bits of our result * to the goddess of code clarity. * */ scale = (uint64_t)1 << 63; scale += (adjustment / 1024) * 2199; scale /= ticks_per_sec; *tick_scale_x = scale * 2; /* * hi part of adj * it contains ns (without fraction) to add to the next sec. * Get ns scale factor for the next sec. */ nano = (adjustment > 0)? adjustment >> 32 : -((-adjustment) >> 32); scale = (uint64_t) NSEC_PER_SEC; scale += nano; *s_scale_ns = scale; /* * lo part of adj * it contains 32 bit frac of ns to add to the next sec. * Keep it as additional adjustment for the next sec. */ frac = (adjustment > 0)? ((uint32_t) adjustment) : -((uint32_t) (-adjustment)); *s_adj_nsx = (frac > 0)? ((uint64_t) frac) << 32 : -(((uint64_t) (-frac)) << 32); return; } /* * scale_delta: * * returns a bintime struct representing delta scaled accordingly to the * scale factors provided to this function. */ static struct bintime scale_delta(uint64_t delta, uint64_t tick_scale_x, uint64_t s_scale_ns, int64_t s_adj_nsx) { uint64_t sec, new_ns, over; struct bintime bt; bt.sec = 0; bt.frac = 0; /* * If more than one second is elapsed, * scale fully elapsed seconds using scale factors for seconds. * s_scale_ns -> scales sec to ns. * s_adj_nsx -> additional adj expressed in 64 bit frac of ns to apply to each sec. */ if (delta > ticks_per_sec) { sec = (delta / ticks_per_sec); new_ns = sec * s_scale_ns; bintime_addns(&bt, new_ns); if (s_adj_nsx) { if (sec == 1) { /* shortcut, no overflow can occur */ if (s_adj_nsx > 0) { bintime_addx(&bt, (uint64_t)s_adj_nsx / (uint64_t)NSEC_PER_SEC); } else { bintime_subx(&bt, (uint64_t)-s_adj_nsx / (uint64_t)NSEC_PER_SEC); } } else { /* * s_adj_nsx is 64 bit frac of ns. * sec*s_adj_nsx might overflow in int64_t. * use bintime_addxns to not lose overflowed ns. */ bintime_addxns(&bt, sec, s_adj_nsx); } } delta = (delta % ticks_per_sec); } over = multi_overflow(tick_scale_x, delta); if (over) { bt.sec += over; } /* * scale elapsed ticks using the scale factor for ticks. */ bintime_addx(&bt, delta * tick_scale_x); return bt; } /* * get_scaled_time: * * returns the scaled time of the time elapsed from the last time * scale factors were updated to now. */ static struct bintime get_scaled_time(uint64_t now) { uint64_t delta; /* * Compute ticks elapsed since last scale update. * This time will be scaled according to the value given by ntp kern. */ delta = now - clock_calend.offset_count; return scale_delta(delta, clock_calend.tick_scale_x, clock_calend.s_scale_ns, clock_calend.s_adj_nsx); } static void clock_get_calendar_absolute_and_microtime_locked( clock_sec_t *secs, clock_usec_t *microsecs, uint64_t *abstime) { uint64_t now; struct bintime bt; now = mach_absolute_time(); if (abstime) { *abstime = now; } bt = get_scaled_time(now); bintime_add(&bt, &clock_calend.bintime); bintime2usclock(&bt, secs, microsecs); } static void clock_get_calendar_absolute_and_nanotime_locked( clock_sec_t *secs, clock_usec_t *nanosecs, uint64_t *abstime) { uint64_t now; struct bintime bt; now = mach_absolute_time(); if (abstime) { *abstime = now; } bt = get_scaled_time(now); bintime_add(&bt, &clock_calend.bintime); bintime2nsclock(&bt, secs, nanosecs); } /* * clock_get_calendar_absolute_and_microtime: * * Returns the current calendar value, * microseconds as the fraction. Also * returns mach_absolute_time if abstime * is not NULL. */ void clock_get_calendar_absolute_and_microtime( clock_sec_t *secs, clock_usec_t *microsecs, uint64_t *abstime) { spl_t s; s = splclock(); clock_lock(); clock_get_calendar_absolute_and_microtime_locked(secs, microsecs, abstime); clock_unlock(); splx(s); } /* * clock_get_calendar_nanotime: * * Returns the current calendar value, * nanoseconds as the fraction. * * Since we do not have an interface to * set the calendar with resolution greater * than a microsecond, we honor that here. */ void clock_get_calendar_nanotime( clock_sec_t *secs, clock_nsec_t *nanosecs) { spl_t s; s = splclock(); clock_lock(); clock_get_calendar_absolute_and_nanotime_locked(secs, nanosecs, NULL); clock_unlock(); splx(s); } /* * clock_gettimeofday: * * Kernel interface for commpage implementation of * gettimeofday() syscall. * * Returns the current calendar value, and updates the * commpage info as appropriate. Because most calls to * gettimeofday() are handled in user mode by the commpage, * this routine should be used infrequently. */ void clock_gettimeofday( clock_sec_t *secs, clock_usec_t *microsecs) { clock_gettimeofday_and_absolute_time(secs, microsecs, NULL); } void clock_gettimeofday_and_absolute_time( clock_sec_t *secs, clock_usec_t *microsecs, uint64_t *mach_time) { uint64_t now; spl_t s; struct bintime bt; s = splclock(); clock_lock(); now = mach_absolute_time(); bt = get_scaled_time(now); bintime_add(&bt, &clock_calend.bintime); bintime2usclock(&bt, secs, microsecs); clock_gettimeofday_set_commpage(now, bt.sec, bt.frac, clock_calend.tick_scale_x, ticks_per_sec); clock_unlock(); splx(s); if (mach_time) { *mach_time = now; } } /* * clock_set_calendar_microtime: * * Sets the current calendar value by * recalculating the epoch and offset * from the system clock. * * Also adjusts the boottime to keep the * value consistent, writes the new * calendar value to the platform clock, * and sends calendar change notifications. */ void clock_set_calendar_microtime( clock_sec_t secs, clock_usec_t microsecs) { uint64_t absolutesys; clock_sec_t newsecs; clock_sec_t oldsecs; clock_usec_t newmicrosecs; clock_usec_t oldmicrosecs; uint64_t commpage_value; spl_t s; struct bintime bt; clock_sec_t deltasecs; clock_usec_t deltamicrosecs; newsecs = secs; newmicrosecs = microsecs; /* * settime_lock mtx is used to avoid that racing settimeofdays update the wall clock and * the platform clock concurrently. * * clock_lock cannot be used for this race because it is acquired from interrupt context * and it needs interrupts disabled while instead updating the platform clock needs to be * called with interrupts enabled. */ lck_mtx_lock(&settime_lock); s = splclock(); clock_lock(); #if DEVELOPMENT || DEBUG struct clock_calend clock_calend_cp = clock_calend; #endif commpage_disable_timestamp(); /* * Adjust the boottime based on the delta. */ clock_get_calendar_absolute_and_microtime_locked(&oldsecs, &oldmicrosecs, &absolutesys); #if DEVELOPMENT || DEBUG if (g_should_log_clock_adjustments) { os_log(OS_LOG_DEFAULT, "%s wall %lu s %d u computed with %llu abs\n", __func__, (unsigned long)oldsecs, oldmicrosecs, absolutesys); os_log(OS_LOG_DEFAULT, "%s requested %lu s %d u\n", __func__, (unsigned long)secs, microsecs ); } #endif if (oldsecs < secs || (oldsecs == secs && oldmicrosecs < microsecs)) { // moving forwards deltasecs = secs; deltamicrosecs = microsecs; TIME_SUB(deltasecs, oldsecs, deltamicrosecs, oldmicrosecs, USEC_PER_SEC); TIME_ADD(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC); clock2bintime(&deltasecs, &deltamicrosecs, &bt); bintime_add(&clock_calend.boottime, &bt); } else { // moving backwards deltasecs = oldsecs; deltamicrosecs = oldmicrosecs; TIME_SUB(deltasecs, secs, deltamicrosecs, microsecs, USEC_PER_SEC); TIME_SUB(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC); clock2bintime(&deltasecs, &deltamicrosecs, &bt); bintime_sub(&clock_calend.boottime, &bt); } clock_calend.bintime = clock_calend.boottime; bintime_add(&clock_calend.bintime, &clock_calend.offset); clock2bintime((clock_sec_t *) &secs, (clock_usec_t *) µsecs, &bt); clock_gettimeofday_set_commpage(absolutesys, bt.sec, bt.frac, clock_calend.tick_scale_x, ticks_per_sec); #if DEVELOPMENT || DEBUG struct clock_calend clock_calend_cp1 = clock_calend; #endif commpage_value = clock_boottime * USEC_PER_SEC + clock_boottime_usec; clock_unlock(); splx(s); /* * Set the new value for the platform clock. * This call might block, so interrupts must be enabled. */ #if DEVELOPMENT || DEBUG uint64_t now_b = mach_absolute_time(); #endif PESetUTCTimeOfDay(newsecs, newmicrosecs); #if DEVELOPMENT || DEBUG uint64_t now_a = mach_absolute_time(); if (g_should_log_clock_adjustments) { os_log(OS_LOG_DEFAULT, "%s mach bef PESet %llu mach aft %llu \n", __func__, now_b, now_a); } #endif print_all_clock_variables_internal(__func__, &clock_calend_cp); print_all_clock_variables_internal(__func__, &clock_calend_cp1); commpage_update_boottime(commpage_value); /* * Send host notifications. */ host_notify_calendar_change(); host_notify_calendar_set(); #if CONFIG_DTRACE clock_track_calend_nowait(); #endif lck_mtx_unlock(&settime_lock); } uint64_t mach_absolutetime_asleep = 0; uint64_t mach_absolutetime_last_sleep = 0; void clock_get_calendar_uptime(clock_sec_t *secs) { uint64_t now; spl_t s; struct bintime bt; s = splclock(); clock_lock(); now = mach_absolute_time(); bt = get_scaled_time(now); bintime_add(&bt, &clock_calend.offset); *secs = bt.sec; clock_unlock(); splx(s); } /* * clock_update_calendar: * * called by ntp timer to update scale factors. */ void clock_update_calendar(void) { uint64_t now, delta; struct bintime bt; spl_t s; int64_t adjustment; s = splclock(); clock_lock(); now = mach_absolute_time(); /* * scale the time elapsed since the last update and * add it to offset. */ bt = get_scaled_time(now); bintime_add(&clock_calend.offset, &bt); /* * update the base from which apply next scale factors. */ delta = now - clock_calend.offset_count; clock_calend.offset_count += delta; clock_calend.bintime = clock_calend.offset; bintime_add(&clock_calend.bintime, &clock_calend.boottime); /* * recompute next adjustment. */ ntp_update_second(&adjustment, clock_calend.bintime.sec); #if DEVELOPMENT || DEBUG if (g_should_log_clock_adjustments) { os_log(OS_LOG_DEFAULT, "%s adjustment %lld\n", __func__, adjustment); } #endif /* * recomputing scale factors. */ get_scale_factors_from_adj(adjustment, &clock_calend.tick_scale_x, &clock_calend.s_scale_ns, &clock_calend.s_adj_nsx); clock_gettimeofday_set_commpage(now, clock_calend.bintime.sec, clock_calend.bintime.frac, clock_calend.tick_scale_x, ticks_per_sec); #if DEVELOPMENT || DEBUG struct clock_calend calend_cp = clock_calend; #endif clock_unlock(); splx(s); print_all_clock_variables(__func__, NULL, NULL, NULL, NULL, &calend_cp); } #if DEVELOPMENT || DEBUG void print_all_clock_variables_internal(const char* func, struct clock_calend* clock_calend_cp) { clock_sec_t offset_secs; clock_usec_t offset_microsecs; clock_sec_t bintime_secs; clock_usec_t bintime_microsecs; clock_sec_t bootime_secs; clock_usec_t bootime_microsecs; if (!g_should_log_clock_adjustments) { return; } bintime2usclock(&clock_calend_cp->offset, &offset_secs, &offset_microsecs); bintime2usclock(&clock_calend_cp->bintime, &bintime_secs, &bintime_microsecs); bintime2usclock(&clock_calend_cp->boottime, &bootime_secs, &bootime_microsecs); os_log(OS_LOG_DEFAULT, "%s s_scale_ns %llu s_adj_nsx %lld tick_scale_x %llu offset_count %llu\n", func, clock_calend_cp->s_scale_ns, clock_calend_cp->s_adj_nsx, clock_calend_cp->tick_scale_x, clock_calend_cp->offset_count); os_log(OS_LOG_DEFAULT, "%s offset.sec %ld offset.frac %llu offset_secs %lu offset_microsecs %d\n", func, clock_calend_cp->offset.sec, clock_calend_cp->offset.frac, (unsigned long)offset_secs, offset_microsecs); os_log(OS_LOG_DEFAULT, "%s bintime.sec %ld bintime.frac %llu bintime_secs %lu bintime_microsecs %d\n", func, clock_calend_cp->bintime.sec, clock_calend_cp->bintime.frac, (unsigned long)bintime_secs, bintime_microsecs); os_log(OS_LOG_DEFAULT, "%s bootime.sec %ld bootime.frac %llu bootime_secs %lu bootime_microsecs %d\n", func, clock_calend_cp->boottime.sec, clock_calend_cp->boottime.frac, (unsigned long)bootime_secs, bootime_microsecs); #if !HAS_CONTINUOUS_HWCLOCK clock_sec_t basesleep_secs; clock_usec_t basesleep_microsecs; bintime2usclock(&clock_calend_cp->basesleep, &basesleep_secs, &basesleep_microsecs); os_log(OS_LOG_DEFAULT, "%s basesleep.sec %ld basesleep.frac %llu basesleep_secs %lu basesleep_microsecs %d\n", func, clock_calend_cp->basesleep.sec, clock_calend_cp->basesleep.frac, (unsigned long)basesleep_secs, basesleep_microsecs); #endif } void print_all_clock_variables(const char* func, clock_sec_t* pmu_secs, clock_usec_t* pmu_usec, clock_sec_t* sys_secs, clock_usec_t* sys_usec, struct clock_calend* clock_calend_cp) { if (!g_should_log_clock_adjustments) { return; } struct bintime bt; clock_sec_t wall_secs; clock_usec_t wall_microsecs; uint64_t now; uint64_t delta; if (pmu_secs) { os_log(OS_LOG_DEFAULT, "%s PMU %lu s %d u \n", func, (unsigned long)*pmu_secs, *pmu_usec); } if (sys_secs) { os_log(OS_LOG_DEFAULT, "%s sys %lu s %d u \n", func, (unsigned long)*sys_secs, *sys_usec); } print_all_clock_variables_internal(func, clock_calend_cp); now = mach_absolute_time(); delta = now - clock_calend_cp->offset_count; bt = scale_delta(delta, clock_calend_cp->tick_scale_x, clock_calend_cp->s_scale_ns, clock_calend_cp->s_adj_nsx); bintime_add(&bt, &clock_calend_cp->bintime); bintime2usclock(&bt, &wall_secs, &wall_microsecs); os_log(OS_LOG_DEFAULT, "%s wall %lu s %d u computed with %llu abs\n", func, (unsigned long)wall_secs, wall_microsecs, now); } #endif /* DEVELOPMENT || DEBUG */ /* * clock_initialize_calendar: * * Set the calendar and related clocks * from the platform clock at boot. * * Also sends host notifications. */ void clock_initialize_calendar(void) { clock_sec_t sys; // sleepless time since boot in seconds clock_sec_t secs; // Current UTC time clock_sec_t utc_offset_secs; // Difference in current UTC time and sleepless time since boot clock_usec_t microsys; clock_usec_t microsecs; clock_usec_t utc_offset_microsecs; spl_t s; struct bintime bt; #if ENABLE_LEGACY_CLOCK_CODE struct bintime monotonic_bt; struct latched_time monotonic_time; uint64_t monotonic_usec_total; clock_sec_t sys2, monotonic_sec; clock_usec_t microsys2, monotonic_usec; size_t size; #endif /* ENABLE_LEGACY_CLOCK_CODE */ //Get the UTC time and corresponding sys time PEGetUTCTimeOfDay(&secs, µsecs); clock_get_system_microtime(&sys, µsys); #if ENABLE_LEGACY_CLOCK_CODE /* * If the platform has a monotonic clock, use kern.monotonicclock_usecs * to estimate the sleep/wake time, otherwise use the UTC time to estimate * the sleep time. */ size = sizeof(monotonic_time); if (kernel_sysctlbyname("kern.monotonicclock_usecs", &monotonic_time, &size, NULL, 0) != 0) { has_monotonic_clock = 0; os_log(OS_LOG_DEFAULT, "%s system does not have monotonic clock\n", __func__); } else { has_monotonic_clock = 1; monotonic_usec_total = monotonic_time.monotonic_time_usec; absolutetime_to_microtime(monotonic_time.mach_time, &sys2, µsys2); os_log(OS_LOG_DEFAULT, "%s system has monotonic clock\n", __func__); } #endif /* ENABLE_LEGACY_CLOCK_CODE */ s = splclock(); clock_lock(); commpage_disable_timestamp(); utc_offset_secs = secs; utc_offset_microsecs = microsecs; /* * We normally expect the UTC clock to be always-on and produce * greater readings than the tick counter. There may be corner cases * due to differing clock resolutions (UTC clock is likely lower) and * and errors reading the UTC clock (some implementations return 0 * on error) in which that doesn't hold true. Bring the UTC measurements * in-line with the tick counter measurements as a best effort in that case. */ if ((sys > secs) || ((sys == secs) && (microsys > microsecs))) { os_log(OS_LOG_DEFAULT, "%s WARNING: UTC time is less then sys time, (%lu s %d u) UTC (%lu s %d u) sys\n", __func__, (unsigned long) secs, microsecs, (unsigned long)sys, microsys); secs = utc_offset_secs = sys; microsecs = utc_offset_microsecs = microsys; } // UTC - sys // This macro stores the subtraction result in utc_offset_secs and utc_offset_microsecs TIME_SUB(utc_offset_secs, sys, utc_offset_microsecs, microsys, USEC_PER_SEC); // This function converts utc_offset_secs and utc_offset_microsecs in bintime clock2bintime(&utc_offset_secs, &utc_offset_microsecs, &bt); /* * Initialize the boot time based on the platform clock. */ clock_boottime = secs; clock_boottime_usec = microsecs; commpage_update_boottime(clock_boottime * USEC_PER_SEC + clock_boottime_usec); nanoseconds_to_absolutetime((uint64_t)NSEC_PER_SEC, &ticks_per_sec); clock_calend.boottime = bt; clock_calend.bintime = bt; clock_calend.offset.sec = 0; clock_calend.offset.frac = 0; clock_calend.tick_scale_x = (uint64_t)1 << 63; clock_calend.tick_scale_x /= ticks_per_sec; clock_calend.tick_scale_x *= 2; clock_calend.s_scale_ns = NSEC_PER_SEC; clock_calend.s_adj_nsx = 0; #if ENABLE_LEGACY_CLOCK_CODE if (has_monotonic_clock) { OS_ANALYZER_SUPPRESS("82347749") monotonic_sec = monotonic_usec_total / (clock_sec_t)USEC_PER_SEC; monotonic_usec = monotonic_usec_total % (clock_usec_t)USEC_PER_SEC; // monotonic clock - sys // This macro stores the subtraction result in monotonic_sec and monotonic_usec TIME_SUB(monotonic_sec, sys2, monotonic_usec, microsys2, USEC_PER_SEC); clock2bintime(&monotonic_sec, &monotonic_usec, &monotonic_bt); // set the baseleep as the difference between monotonic clock - sys clock_calend.basesleep = monotonic_bt; } #endif /* ENABLE_LEGACY_CLOCK_CODE */ commpage_update_mach_continuous_time(mach_absolutetime_asleep); #if DEVELOPMENT || DEBUG struct clock_calend clock_calend_cp = clock_calend; #endif clock_unlock(); splx(s); print_all_clock_variables(__func__, &secs, µsecs, &sys, µsys, &clock_calend_cp); /* * Send host notifications. */ host_notify_calendar_change(); #if CONFIG_DTRACE clock_track_calend_nowait(); #endif } #if HAS_CONTINUOUS_HWCLOCK static void scale_sleep_time(void) { /* Apply the current NTP frequency adjustment to the time slept. * The frequency adjustment remains stable between calls to ntp_adjtime(), * and should thus provide a reasonable approximation of the total adjustment * required for the time slept. */ struct bintime sleep_time; uint64_t tick_scale_x, s_scale_ns; int64_t s_adj_nsx; int64_t sleep_adj = ntp_get_freq(); if (sleep_adj) { get_scale_factors_from_adj(sleep_adj, &tick_scale_x, &s_scale_ns, &s_adj_nsx); sleep_time = scale_delta(mach_absolutetime_last_sleep, tick_scale_x, s_scale_ns, s_adj_nsx); } else { tick_scale_x = (uint64_t)1 << 63; tick_scale_x /= ticks_per_sec; tick_scale_x *= 2; sleep_time.sec = mach_absolutetime_last_sleep / ticks_per_sec; sleep_time.frac = (mach_absolutetime_last_sleep % ticks_per_sec) * tick_scale_x; } bintime_add(&clock_calend.offset, &sleep_time); bintime_add(&clock_calend.bintime, &sleep_time); } static void clock_wakeup_calendar_hwclock(void) { spl_t s; s = splclock(); clock_lock(); commpage_disable_timestamp(); uint64_t abstime = mach_absolute_time(); uint64_t total_sleep_time = mach_continuous_time() - abstime; mach_absolutetime_last_sleep = total_sleep_time - mach_absolutetime_asleep; mach_absolutetime_asleep = total_sleep_time; scale_sleep_time(); KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_EPOCH_CHANGE), (uintptr_t)mach_absolutetime_last_sleep, (uintptr_t)mach_absolutetime_asleep, (uintptr_t)(mach_absolutetime_last_sleep >> 32), (uintptr_t)(mach_absolutetime_asleep >> 32)); commpage_update_mach_continuous_time(mach_absolutetime_asleep); #if HIBERNATION commpage_update_mach_continuous_time_hw_offset(hwclock_conttime_offset); #endif adjust_cont_time_thread_calls(); clock_unlock(); splx(s); host_notify_calendar_change(); #if CONFIG_DTRACE clock_track_calend_nowait(); #endif } #endif /* HAS_CONTINUOUS_HWCLOCK */ #if ENABLE_LEGACY_CLOCK_CODE static void clock_wakeup_calendar_legacy(void) { clock_sec_t wake_sys_sec; clock_usec_t wake_sys_usec; clock_sec_t wake_sec; clock_usec_t wake_usec; clock_sec_t wall_time_sec; clock_usec_t wall_time_usec; clock_sec_t diff_sec; clock_usec_t diff_usec; clock_sec_t var_s; clock_usec_t var_us; spl_t s; struct bintime bt, last_sleep_bt; struct latched_time monotonic_time; uint64_t monotonic_usec_total; uint64_t wake_abs; size_t size; /* * If the platform has the monotonic clock use that to * compute the sleep time. The monotonic clock does not have an offset * that can be modified, so nor kernel or userspace can change the time * of this clock, it can only monotonically increase over time. * During sleep mach_absolute_time (sys time) does not tick, * so the sleep time is the difference between the current monotonic time * less the absolute time and the previous difference stored at wake time. * * basesleep = (monotonic - sys) ---> computed at last wake * sleep_time = (monotonic - sys) - basesleep * * If the platform does not support monotonic clock we set the wall time to what the * UTC clock returns us. * Setting the wall time to UTC time implies that we loose all the adjustments * done during wake time through adjtime/ntp_adjustime. * The UTC time is the monotonic clock + an offset that can be set * by kernel. * The time slept in this case is the difference between wall time and UTC * at wake. * * IMPORTANT: * We assume that only the kernel is setting the offset of the PMU/RTC and that * it is doing it only througth the settimeofday interface. */ if (has_monotonic_clock) { #if DEVELOPMENT || DEBUG /* * Just for debugging, get the wake UTC time. */ PEGetUTCTimeOfDay(&var_s, &var_us); #endif /* * Get monotonic time with corresponding sys time */ size = sizeof(monotonic_time); if (kernel_sysctlbyname("kern.monotonicclock_usecs", &monotonic_time, &size, NULL, 0) != 0) { panic("%s: could not call kern.monotonicclock_usecs", __func__); } wake_abs = monotonic_time.mach_time; absolutetime_to_microtime(wake_abs, &wake_sys_sec, &wake_sys_usec); monotonic_usec_total = monotonic_time.monotonic_time_usec; wake_sec = monotonic_usec_total / (clock_sec_t)USEC_PER_SEC; wake_usec = monotonic_usec_total % (clock_usec_t)USEC_PER_SEC; } else { /* * Get UTC time and corresponding sys time */ PEGetUTCTimeOfDay(&wake_sec, &wake_usec); wake_abs = mach_absolute_time(); absolutetime_to_microtime(wake_abs, &wake_sys_sec, &wake_sys_usec); } #if DEVELOPMENT || DEBUG os_log(OS_LOG_DEFAULT, "time at wake %lu s %d u from %s clock, abs %llu\n", (unsigned long)wake_sec, wake_usec, (has_monotonic_clock)?"monotonic":"UTC", wake_abs); if (has_monotonic_clock) { OS_ANALYZER_SUPPRESS("82347749") os_log(OS_LOG_DEFAULT, "UTC time %lu s %d u\n", (unsigned long)var_s, var_us); } #endif /* DEVELOPMENT || DEBUG */ s = splclock(); clock_lock(); commpage_disable_timestamp(); #if DEVELOPMENT || DEBUG struct clock_calend clock_calend_cp1 = clock_calend; #endif /* DEVELOPMENT || DEBUG */ /* * We normally expect the UTC/monotonic clock to be always-on and produce * greater readings than the sys counter. There may be corner cases * due to differing clock resolutions (UTC/monotonic clock is likely lower) and * and errors reading the UTC/monotonic clock (some implementations return 0 * on error) in which that doesn't hold true. */ if ((wake_sys_sec > wake_sec) || ((wake_sys_sec == wake_sec) && (wake_sys_usec > wake_usec))) { os_log_error(OS_LOG_DEFAULT, "WARNING: %s clock is less then sys clock at wake: %lu s %d u vs %lu s %d u, defaulting sleep time to zero\n", (has_monotonic_clock)?"monotonic":"UTC", (unsigned long)wake_sec, wake_usec, (unsigned long)wake_sys_sec, wake_sys_usec); mach_absolutetime_last_sleep = 0; goto done; } if (has_monotonic_clock) { /* * computer the difference monotonic - sys * we already checked that monotonic time is * greater than sys. */ diff_sec = wake_sec; diff_usec = wake_usec; // This macro stores the subtraction result in diff_sec and diff_usec TIME_SUB(diff_sec, wake_sys_sec, diff_usec, wake_sys_usec, USEC_PER_SEC); //This function converts diff_sec and diff_usec in bintime clock2bintime(&diff_sec, &diff_usec, &bt); /* * Safety belt: the monotonic clock will likely have a lower resolution than the sys counter. * It's also possible that the device didn't fully transition to the powered-off state on * the most recent sleep, so the sys counter may not have reset or may have only briefly * turned off. In that case it's possible for the difference between the monotonic clock and the * sys counter to be less than the previously recorded value in clock.calend.basesleep. * In that case simply record that we slept for 0 ticks. */ if ((bt.sec > clock_calend.basesleep.sec) || ((bt.sec == clock_calend.basesleep.sec) && (bt.frac > clock_calend.basesleep.frac))) { //last_sleep is the difference between (current monotonic - abs) and (last wake monotonic - abs) last_sleep_bt = bt; bintime_sub(&last_sleep_bt, &clock_calend.basesleep); bintime2absolutetime(&last_sleep_bt, &mach_absolutetime_last_sleep); mach_absolutetime_asleep += mach_absolutetime_last_sleep; //set basesleep to current monotonic - abs clock_calend.basesleep = bt; //update wall time bintime_add(&clock_calend.offset, &last_sleep_bt); bintime_add(&clock_calend.bintime, &last_sleep_bt); bintime2usclock(&last_sleep_bt, &var_s, &var_us); os_log(OS_LOG_DEFAULT, "time_slept (%lu s %d u)\n", (unsigned long) var_s, var_us); } else { bintime2usclock(&clock_calend.basesleep, &var_s, &var_us); os_log_error(OS_LOG_DEFAULT, "WARNING: last wake monotonic-sys time (%lu s %d u) is greater then current monotonic-sys time(%lu s %d u), defaulting sleep time to zero\n", (unsigned long) var_s, var_us, (unsigned long) diff_sec, diff_usec); mach_absolutetime_last_sleep = 0; } } else { /* * set the wall time to UTC value */ bt = get_scaled_time(wake_abs); bintime_add(&bt, &clock_calend.bintime); bintime2usclock(&bt, &wall_time_sec, &wall_time_usec); if (wall_time_sec > wake_sec || (wall_time_sec == wake_sec && wall_time_usec > wake_usec)) { os_log(OS_LOG_DEFAULT, "WARNING: wall time (%lu s %d u) is greater than current UTC time (%lu s %d u), defaulting sleep time to zero\n", (unsigned long) wall_time_sec, wall_time_usec, (unsigned long) wake_sec, wake_usec); mach_absolutetime_last_sleep = 0; } else { diff_sec = wake_sec; diff_usec = wake_usec; // This macro stores the subtraction result in diff_sec and diff_usec TIME_SUB(diff_sec, wall_time_sec, diff_usec, wall_time_usec, USEC_PER_SEC); //This function converts diff_sec and diff_usec in bintime clock2bintime(&diff_sec, &diff_usec, &bt); //time slept in this case is the difference between PMU/RTC and wall time last_sleep_bt = bt; bintime2absolutetime(&last_sleep_bt, &mach_absolutetime_last_sleep); mach_absolutetime_asleep += mach_absolutetime_last_sleep; //update wall time bintime_add(&clock_calend.offset, &last_sleep_bt); bintime_add(&clock_calend.bintime, &last_sleep_bt); bintime2usclock(&last_sleep_bt, &var_s, &var_us); os_log(OS_LOG_DEFAULT, "time_slept (%lu s %d u)\n", (unsigned long)var_s, var_us); } } done: KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_EPOCH_CHANGE), (uintptr_t)mach_absolutetime_last_sleep, (uintptr_t)mach_absolutetime_asleep, (uintptr_t)(mach_absolutetime_last_sleep >> 32), (uintptr_t)(mach_absolutetime_asleep >> 32)); commpage_update_mach_continuous_time(mach_absolutetime_asleep); adjust_cont_time_thread_calls(); #if DEVELOPMENT || DEBUG struct clock_calend clock_calend_cp = clock_calend; #endif clock_unlock(); splx(s); #if DEVELOPMENT || DEBUG if (g_should_log_clock_adjustments) { print_all_clock_variables("clock_wakeup_calendar: BEFORE", NULL, NULL, NULL, NULL, &clock_calend_cp1); print_all_clock_variables("clock_wakeup_calendar: AFTER", NULL, NULL, NULL, NULL, &clock_calend_cp); } #endif /* DEVELOPMENT || DEBUG */ host_notify_calendar_change(); #if CONFIG_DTRACE clock_track_calend_nowait(); #endif } #endif /* ENABLE_LEGACY_CLOCK_CODE */ void clock_wakeup_calendar(void) { #if HAS_CONTINUOUS_HWCLOCK #if HIBERNATION_USES_LEGACY_CLOCK if (gIOHibernateState) { // if we're resuming from hibernation, we have to take the legacy wakeup path return clock_wakeup_calendar_legacy(); } #endif /* HIBERNATION_USES_LEGACY_CLOCK */ // use the hwclock wakeup path return clock_wakeup_calendar_hwclock(); #elif ENABLE_LEGACY_CLOCK_CODE return clock_wakeup_calendar_legacy(); #else #error "can't determine which clock code to run" #endif } /* * clock_get_boottime_nanotime: * * Return the boottime, used by sysctl. */ void clock_get_boottime_nanotime( clock_sec_t *secs, clock_nsec_t *nanosecs) { spl_t s; s = splclock(); clock_lock(); *secs = (clock_sec_t)clock_boottime; *nanosecs = (clock_nsec_t)clock_boottime_usec * NSEC_PER_USEC; clock_unlock(); splx(s); } /* * clock_get_boottime_nanotime: * * Return the boottime, used by sysctl. */ void clock_get_boottime_microtime( clock_sec_t *secs, clock_usec_t *microsecs) { spl_t s; s = splclock(); clock_lock(); *secs = (clock_sec_t)clock_boottime; *microsecs = (clock_nsec_t)clock_boottime_usec; clock_unlock(); splx(s); } /* * Wait / delay routines. */ static void mach_wait_until_continue( __unused void *parameter, wait_result_t wresult) { thread_syscall_return((wresult == THREAD_INTERRUPTED)? KERN_ABORTED: KERN_SUCCESS); /*NOTREACHED*/ } /* * mach_wait_until_trap: Suspend execution of calling thread until the specified time has passed * * Parameters: args->deadline Amount of time to wait * * Returns: 0 Success * !0 Not success * */ kern_return_t mach_wait_until_trap( struct mach_wait_until_trap_args *args) { uint64_t deadline = args->deadline; wait_result_t wresult; wresult = assert_wait_deadline_with_leeway((event_t)mach_wait_until_trap, THREAD_ABORTSAFE, TIMEOUT_URGENCY_USER_NORMAL, deadline, 0); if (wresult == THREAD_WAITING) { wresult = thread_block(mach_wait_until_continue); } return (wresult == THREAD_INTERRUPTED)? KERN_ABORTED: KERN_SUCCESS; } void clock_delay_until( uint64_t deadline) { uint64_t now = mach_absolute_time(); if (now >= deadline) { return; } _clock_delay_until_deadline(deadline - now, deadline); } /* * Preserve the original precise interval that the client * requested for comparison to the spin threshold. */ void _clock_delay_until_deadline( uint64_t interval, uint64_t deadline) { _clock_delay_until_deadline_with_leeway(interval, deadline, 0); } /* * Like _clock_delay_until_deadline, but it accepts a * leeway value. */ void _clock_delay_until_deadline_with_leeway( uint64_t interval, uint64_t deadline, uint64_t leeway) { if (interval == 0) { return; } if (ml_delay_should_spin(interval) || get_preemption_level() != 0 || ml_get_interrupts_enabled() == FALSE) { machine_delay_until(interval, deadline); } else { /* * For now, assume a leeway request of 0 means the client does not want a leeway * value. We may want to change this interpretation in the future. */ if (leeway) { assert_wait_deadline_with_leeway((event_t)clock_delay_until, THREAD_UNINT, TIMEOUT_URGENCY_LEEWAY, deadline, leeway); } else { assert_wait_deadline((event_t)clock_delay_until, THREAD_UNINT, deadline); } thread_block(THREAD_CONTINUE_NULL); } } void delay_for_interval( uint32_t interval, uint32_t scale_factor) { uint64_t abstime; clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime); _clock_delay_until_deadline(abstime, mach_absolute_time() + abstime); } void delay_for_interval_with_leeway( uint32_t interval, uint32_t leeway, uint32_t scale_factor) { uint64_t abstime_interval; uint64_t abstime_leeway; clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime_interval); clock_interval_to_absolutetime_interval(leeway, scale_factor, &abstime_leeway); _clock_delay_until_deadline_with_leeway(abstime_interval, mach_absolute_time() + abstime_interval, abstime_leeway); } void delay( int usec) { delay_for_interval((usec < 0)? -usec: usec, NSEC_PER_USEC); } /* * Miscellaneous routines. */ void clock_interval_to_deadline( uint32_t interval, uint32_t scale_factor, uint64_t *result) { uint64_t abstime; clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime); if (os_add_overflow(mach_absolute_time(), abstime, result)) { *result = UINT64_MAX; } } void nanoseconds_to_deadline( uint64_t interval, uint64_t *result) { uint64_t abstime; nanoseconds_to_absolutetime(interval, &abstime); if (os_add_overflow(mach_absolute_time(), abstime, result)) { *result = UINT64_MAX; } } void clock_absolutetime_interval_to_deadline( uint64_t abstime, uint64_t *result) { if (os_add_overflow(mach_absolute_time(), abstime, result)) { *result = UINT64_MAX; } } void clock_continuoustime_interval_to_deadline( uint64_t conttime, uint64_t *result) { if (os_add_overflow(mach_continuous_time(), conttime, result)) { *result = UINT64_MAX; } } void clock_get_uptime( uint64_t *result) { *result = mach_absolute_time(); } void clock_deadline_for_periodic_event( uint64_t interval, uint64_t abstime, uint64_t *deadline) { assert(interval != 0); // *deadline += interval; if (os_add_overflow(*deadline, interval, deadline)) { *deadline = UINT64_MAX; } if (*deadline <= abstime) { // *deadline = abstime + interval; if (os_add_overflow(abstime, interval, deadline)) { *deadline = UINT64_MAX; } abstime = mach_absolute_time(); if (*deadline <= abstime) { // *deadline = abstime + interval; if (os_add_overflow(abstime, interval, deadline)) { *deadline = UINT64_MAX; } } } } uint64_t mach_continuous_time(void) { #if HIBERNATION && HAS_CONTINUOUS_HWCLOCK return ml_get_hwclock() + hwclock_conttime_offset; #elif HAS_CONTINUOUS_HWCLOCK return ml_get_hwclock(); #else while (1) { uint64_t read1 = mach_absolutetime_asleep; uint64_t absolute = mach_absolute_time(); OSMemoryBarrier(); uint64_t read2 = mach_absolutetime_asleep; if (__builtin_expect(read1 == read2, 1)) { return absolute + read1; } } #endif } uint64_t mach_continuous_approximate_time(void) { #if HAS_CONTINUOUS_HWCLOCK return mach_continuous_time(); #else while (1) { uint64_t read1 = mach_absolutetime_asleep; uint64_t absolute = mach_approximate_time(); OSMemoryBarrier(); uint64_t read2 = mach_absolutetime_asleep; if (__builtin_expect(read1 == read2, 1)) { return absolute + read1; } } #endif } /* * continuoustime_to_absolutetime * Must be called with interrupts disabled * Returned value is only valid until the next update to * mach_continuous_time */ uint64_t continuoustime_to_absolutetime(uint64_t conttime) { if (conttime <= mach_absolutetime_asleep) { return 0; } else { return conttime - mach_absolutetime_asleep; } } /* * absolutetime_to_continuoustime * Must be called with interrupts disabled * Returned value is only valid until the next update to * mach_continuous_time */ uint64_t absolutetime_to_continuoustime(uint64_t abstime) { return abstime + mach_absolutetime_asleep; } #if CONFIG_DTRACE /* * clock_get_calendar_nanotime_nowait * * Description: Non-blocking version of clock_get_calendar_nanotime() * * Notes: This function operates by separately tracking calendar time * updates using a two element structure to copy the calendar * state, which may be asynchronously modified. It utilizes * barrier instructions in the tracking process and in the local * stable snapshot process in order to ensure that a consistent * snapshot is used to perform the calculation. */ void clock_get_calendar_nanotime_nowait( clock_sec_t *secs, clock_nsec_t *nanosecs) { int i = 0; uint64_t now; struct unlocked_clock_calend stable; struct bintime bt; for (;;) { stable = flipflop[i]; /* take snapshot */ /* * Use a barrier instructions to ensure atomicity. We AND * off the "in progress" bit to get the current generation * count. */ os_atomic_andnot(&stable.gen, 1, relaxed); /* * If an update _is_ in progress, the generation count will be * off by one, if it _was_ in progress, it will be off by two, * and if we caught it at a good time, it will be equal (and * our snapshot is threfore stable). */ if (flipflop[i].gen == stable.gen) { break; } /* Switch to the other element of the flipflop, and try again. */ i ^= 1; } now = mach_absolute_time(); bt = get_scaled_time(now); bintime_add(&bt, &clock_calend.bintime); bintime2nsclock(&bt, secs, nanosecs); } static void clock_track_calend_nowait(void) { int i; for (i = 0; i < 2; i++) { struct clock_calend tmp = clock_calend; /* * Set the low bit if the generation count; since we use a * barrier instruction to do this, we are guaranteed that this * will flag an update in progress to an async caller trying * to examine the contents. */ os_atomic_or(&flipflop[i].gen, 1, relaxed); flipflop[i].calend = tmp; /* * Increment the generation count to clear the low bit to * signal completion. If a caller compares the generation * count after taking a copy while in progress, the count * will be off by two. */ os_atomic_inc(&flipflop[i].gen, relaxed); } } #endif /* CONFIG_DTRACE */