osfmk/i386/rtclock.c

/*
 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */

/*
 *	File:		i386/rtclock.c
 *	Purpose:	Routines for handling the machine dependent
 *			real-time clock. Historically, this clock is
 *			generated by the Intel 8254 Programmable Interval
 *			Timer, but local apic timers are now used for
 *			this purpose with the master time reference being
 *			the cpu clock counted by the timestamp MSR.
 */


#include <mach/mach_types.h>

#include <kern/cpu_data.h>
#include <kern/cpu_number.h>
#include <kern/clock.h>
#include <kern/host_notify.h>
#include <kern/macro_help.h>
#include <kern/misc_protos.h>
#include <kern/spl.h>
#include <kern/assert.h>
#include <kern/timer_queue.h>
#include <mach/vm_prot.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>         /* for kernel_map */
#include <architecture/i386/pio.h>
#include <i386/machine_cpu.h>
#include <i386/cpuid.h>
#include <i386/cpu_threads.h>
#include <i386/mp.h>
#include <i386/machine_routines.h>
#include <i386/pal_routines.h>
#include <i386/proc_reg.h>
#include <i386/misc_protos.h>
#include <pexpert/pexpert.h>
#include <machine/limits.h>
#include <machine/commpage.h>
#include <sys/kdebug.h>
#include <i386/tsc.h>
#include <i386/rtclock_protos.h>
#define UI_CPUFREQ_ROUNDING_FACTOR      10000000

int             rtclock_init(void);

uint64_t        tsc_rebase_abs_time = 0;

volatile uint64_t gAcpiLastSleepTscBase = 0;
volatile uint64_t gAcpiLastSleepNanoBase = 0;
volatile uint64_t gAcpiLastWakeTscBase = 0;
volatile uint64_t gAcpiLastWakeNanoBase = 0;

static void     rtc_set_timescale(uint64_t cycles);
static uint64_t rtc_export_speed(uint64_t cycles);

void
rtc_timer_start(void)
{
	/*
	 * Force a complete re-evaluation of timer deadlines.
	 */
	x86_lcpu()->rtcDeadline = EndOfAllTime;
	timer_resync_deadlines();
}

static inline uint32_t
_absolutetime_to_microtime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs)
{
	uint32_t remain;
	*secs = abstime / (uint64_t)NSEC_PER_SEC;
	remain = (uint32_t)(abstime % (uint64_t)NSEC_PER_SEC);
	*microsecs = remain / NSEC_PER_USEC;
	return remain;
}

static inline void
_absolutetime_to_nanotime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *nanosecs)
{
	*secs = abstime / (uint64_t)NSEC_PER_SEC;
	*nanosecs = (clock_usec_t)(abstime % (uint64_t)NSEC_PER_SEC);
}

/*
 * Nanotime/mach_absolutime_time
 * -----------------------------
 * The timestamp counter (TSC) - which counts cpu clock cycles and can be read
 * efficiently by the kernel and in userspace - is the reference for all timing.
 * The cpu clock rate is platform-dependent and may stop or be reset when the
 * processor is napped/slept.  As a result, nanotime is the software abstraction
 * used to maintain a monotonic clock, adjusted from an outside reference as needed.
 *
 * The kernel maintains nanotime information recording:
 *      - the ratio of tsc to nanoseconds
 *	  with this ratio expressed as a 32-bit scale and shift
 *	  (power of 2 divider);
 *	- { tsc_base, ns_base } pair of corresponding timestamps.
 *
 * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage
 * for the userspace nanotime routine to read.
 *
 * All of the routines which update the nanotime data are non-reentrant.  This must
 * be guaranteed by the caller.
 */
static inline void
rtc_nanotime_set_commpage(pal_rtc_nanotime_t *rntp)
{
	commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift);
}

/*
 * rtc_nanotime_init:
 *
 * Intialize the nanotime info from the base time.
 */
static inline void
_rtc_nanotime_init(pal_rtc_nanotime_t *rntp, uint64_t base)
{
	uint64_t        tsc = rdtsc64();

	_pal_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
}

void
rtc_nanotime_init(uint64_t base)
{
	gAcpiLastSleepTscBase = pal_rtc_nanotime_info.tsc_base;
	gAcpiLastSleepNanoBase = pal_rtc_nanotime_info.ns_base;

	_rtc_nanotime_init(&pal_rtc_nanotime_info, base);

	gAcpiLastWakeTscBase = pal_rtc_nanotime_info.tsc_base;
	gAcpiLastWakeNanoBase = pal_rtc_nanotime_info.ns_base;

	rtc_nanotime_set_commpage(&pal_rtc_nanotime_info);
}

/*
 * rtc_nanotime_init_commpage:
 *
 * Call back from the commpage initialization to
 * cause the commpage data to be filled in once the
 * commpages have been created.
 */
void
rtc_nanotime_init_commpage(void)
{
	spl_t                   s = splclock();

	rtc_nanotime_set_commpage(&pal_rtc_nanotime_info);
	splx(s);
}

/*
 * rtc_nanotime_read:
 *
 * Returns the current nanotime value, accessable from any
 * context.
 */
static inline uint64_t
rtc_nanotime_read(void)
{
	return _rtc_nanotime_read(&pal_rtc_nanotime_info);
}

/*
 * rtc_clock_napped:
 *
 * Invoked from power management when we exit from a low C-State (>= C4)
 * and the TSC has stopped counting.  The nanotime data is updated according
 * to the provided value which represents the new value for nanotime.
 */
void
rtc_clock_napped(uint64_t base, uint64_t tsc_base)
{
	pal_rtc_nanotime_t      *rntp = &pal_rtc_nanotime_info;
	uint64_t        oldnsecs;
	uint64_t        newnsecs;
	uint64_t        tsc;

	assert(!ml_get_interrupts_enabled());
	tsc = rdtsc64();
	oldnsecs = rntp->ns_base + _rtc_tsc_to_nanoseconds(tsc - rntp->tsc_base, rntp);
	newnsecs = base + _rtc_tsc_to_nanoseconds(tsc - tsc_base, rntp);

	/*
	 * Only update the base values if time using the new base values
	 * is later than the time using the old base values.
	 */
	if (oldnsecs < newnsecs) {
		_pal_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp);
		rtc_nanotime_set_commpage(rntp);
	}
}

/*
 * Invoked from power management to correct the SFLM TSC entry drift problem:
 * a small delta is added to the tsc_base.  This is equivalent to nudgin time
 * backwards.  We require this to be on the order of a TSC quantum which won't
 * cause callers of mach_absolute_time() to see time going backwards!
 */
void
rtc_clock_adjust(uint64_t tsc_base_delta)
{
	pal_rtc_nanotime_t  *rntp = &pal_rtc_nanotime_info;

	assert(!ml_get_interrupts_enabled());
	assert(tsc_base_delta < 100ULL); /* i.e. it's small */
	_rtc_nanotime_adjust(tsc_base_delta, rntp);
	rtc_nanotime_set_commpage(rntp);
}

/*
 * rtc_sleep_wakeup:
 *
 * Invoked from power management when we have awoken from a sleep (S3)
 * and the TSC has been reset, or from Deep Idle (S0) sleep when the TSC
 * has progressed.  The nanotime data is updated based on the passed-in value.
 *
 * The caller must guarantee non-reentrancy.
 */
void
rtc_sleep_wakeup(
	uint64_t                base)
{
	/* Set fixed configuration for lapic timers */
	rtc_timer->rtc_config();

	/*
	 * Reset nanotime.
	 * The timestamp counter will have been reset
	 * but nanotime (uptime) marches onward.
	 */
	rtc_nanotime_init(base);
}

void
rtc_decrementer_configure(void)
{
	rtc_timer->rtc_config();
}
/*
 * rtclock_early_init() is called very early at boot to
 * establish mach_absolute_time() and set it to zero.
 */
void
rtclock_early_init(void)
{
	assert(tscFreq);
	rtc_set_timescale(tscFreq);
}

/*
 * Initialize the real-time clock device.
 * In addition, various variables used to support the clock are initialized.
 */
int
rtclock_init(void)
{
	uint64_t        cycles;

	assert(!ml_get_interrupts_enabled());

	if (cpu_number() == master_cpu) {
		assert(tscFreq);

		/*
		 * Adjust and set the exported cpu speed.
		 */
		cycles = rtc_export_speed(tscFreq);

		/*
		 * Set min/max to actual.
		 * ACPI may update these later if speed-stepping is detected.
		 */
		gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles;
		gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles;

		rtc_timer_init();
		clock_timebase_init();
		ml_init_delay_spin_threshold(10);
	}

	/* Set fixed configuration for lapic timers */
	rtc_timer->rtc_config();
	rtc_timer_start();

	return 1;
}

// utility routine
// Code to calculate how many processor cycles are in a second...

static void
rtc_set_timescale(uint64_t cycles)
{
	pal_rtc_nanotime_t      *rntp = &pal_rtc_nanotime_info;
	uint32_t    shift = 0;

	/* the "scale" factor will overflow unless cycles>SLOW_TSC_THRESHOLD */

	while (cycles <= SLOW_TSC_THRESHOLD) {
		shift++;
		cycles <<= 1;
	}

	rntp->scale = (uint32_t)(((uint64_t)NSEC_PER_SEC << 32) / cycles);

	rntp->shift = shift;

	/*
	 * On some platforms, the TSC is not reset at warm boot. But the
	 * rebase time must be relative to the current boot so we can't use
	 * mach_absolute_time(). Instead, we convert the TSC delta since boot
	 * to nanoseconds.
	 */
	if (tsc_rebase_abs_time == 0) {
		tsc_rebase_abs_time = _rtc_tsc_to_nanoseconds(
			rdtsc64() - tsc_at_boot, rntp);
	}

	rtc_nanotime_init(0);
}

static uint64_t
rtc_export_speed(uint64_t cyc_per_sec)
{
	pal_rtc_nanotime_t      *rntp = &pal_rtc_nanotime_info;
	uint64_t        cycles;

	if (rntp->shift != 0) {
		printf("Slow TSC, rtc_nanotime.shift == %d\n", rntp->shift);
	}

	/* Round: */
	cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR / 2))
	    / UI_CPUFREQ_ROUNDING_FACTOR)
	    * UI_CPUFREQ_ROUNDING_FACTOR;

	/*
	 * Set current measured speed.
	 */
	if (cycles >= 0x100000000ULL) {
		gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL;
	} else {
		gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles;
	}
	gPEClockFrequencyInfo.cpu_frequency_hz = cycles;

	kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec);
	return cycles;
}

void
clock_get_system_microtime(
	clock_sec_t                     *secs,
	clock_usec_t            *microsecs)
{
	uint64_t        now = rtc_nanotime_read();

	_absolutetime_to_microtime(now, secs, microsecs);
}

void
clock_get_system_nanotime(
	clock_sec_t                     *secs,
	clock_nsec_t            *nanosecs)
{
	uint64_t        now = rtc_nanotime_read();

	_absolutetime_to_nanotime(now, secs, nanosecs);
}

void
clock_gettimeofday_set_commpage(uint64_t abstime, uint64_t sec, uint64_t frac, uint64_t scale, uint64_t tick_per_sec)
{
	commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
}

void
clock_timebase_info(
	mach_timebase_info_t    info)
{
	info->numer = info->denom =  1;
}

/*
 * Real-time clock device interrupt.
 */
int
rtclock_intr(
	x86_saved_state_t       *tregs)
{
	uint64_t        rip;
	boolean_t       user_mode = FALSE;

	assert(get_preemption_level() > 0);
	assert(!ml_get_interrupts_enabled());

	if (is_saved_state64(tregs) == TRUE) {
		x86_saved_state64_t     *regs;

		regs = saved_state64(tregs);

		if (regs->isf.cs & 0x03) {
			user_mode = TRUE;
		}
		rip = regs->isf.rip;
	} else {
		x86_saved_state32_t     *regs;

		regs = saved_state32(tregs);

		if (regs->cs & 0x03) {
			user_mode = TRUE;
		}
		rip = regs->eip;
	}

	/* call the generic etimer */
	timer_intr(user_mode, rip);

	return 0;
}


/*
 *	Request timer pop from the hardware
 */

uint64_t
setPop(uint64_t time)
{
	uint64_t        now;
	uint64_t        pop;

	/* 0 and EndOfAllTime are special-cases for "clear the timer" */
	if (time == 0 || time == EndOfAllTime) {
		time = EndOfAllTime;
		now = 0;
		pop = rtc_timer->rtc_set(0, 0);
	} else {
		now = rtc_nanotime_read();      /* The time in nanoseconds */
		pop = rtc_timer->rtc_set(time, now);
	}

	/* Record requested and actual deadlines set */
	x86_lcpu()->rtcDeadline = time;
	x86_lcpu()->rtcPop      = pop;

	return pop - now;
}

uint64_t
mach_absolute_time(void)
{
	return rtc_nanotime_read();
}

uint64_t
mach_approximate_time(void)
{
	return rtc_nanotime_read();
}

void
clock_interval_to_absolutetime_interval(
	uint32_t                interval,
	uint32_t                scale_factor,
	uint64_t                *result)
{
	*result = (uint64_t)interval * scale_factor;
}

void
absolutetime_to_microtime(
	uint64_t                        abstime,
	clock_sec_t                     *secs,
	clock_usec_t            *microsecs)
{
	_absolutetime_to_microtime(abstime, secs, microsecs);
}

void
nanotime_to_absolutetime(
	clock_sec_t                     secs,
	clock_nsec_t            nanosecs,
	uint64_t                        *result)
{
	*result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs;
}

void
absolutetime_to_nanoseconds(
	uint64_t                abstime,
	uint64_t                *result)
{
	*result = abstime;
}

void
nanoseconds_to_absolutetime(
	uint64_t                nanoseconds,
	uint64_t                *result)
{
	*result = nanoseconds;
}

void
machine_delay_until(
	uint64_t interval,
	uint64_t                deadline)
{
	(void)interval;
	while (mach_absolute_time() < deadline) {
		cpu_pause();
	}
}