/*- * Copyright (c) 2009 Adrian Chadd * Copyright (c) 2012 Spectra Logic Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /** * \file dev/xen/timer/timer.c * \brief A timer driver for the Xen hypervisor's PV clock. */ #include __FBSDID("$FreeBSD: head/sys/dev/xen/timer/timer.c 267528 2014-06-16 08:44:33Z royger $"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "clock_if.h" static devclass_t xentimer_devclass; #define NSEC_IN_SEC 1000000000ULL #define NSEC_IN_USEC 1000ULL /* 18446744073 = int(2^64 / NSEC_IN_SC) = 1 ns in 64-bit fractions */ #define FRAC_IN_NSEC 18446744073LL /* Xen timers may fire up to 100us off */ #define XENTIMER_MIN_PERIOD_IN_NSEC 100*NSEC_IN_USEC #define XENCLOCK_RESOLUTION 10000000 #define ETIME 62 /* Xen "bad time" error */ #define XENTIMER_QUALITY 950 struct xentimer_pcpu_data { uint64_t timer; uint64_t last_processed; void *irq_handle; }; DPCPU_DEFINE(struct xentimer_pcpu_data, xentimer_pcpu); DPCPU_DECLARE(struct vcpu_info *, vcpu_info); struct xentimer_softc { device_t dev; struct timecounter tc; struct eventtimer et; }; /* Last time; this guarantees a monotonically increasing clock. */ volatile uint64_t xen_timer_last_time = 0; static void xentimer_identify(driver_t *driver, device_t parent) { if (!xen_domain()) return; /* Handle all Xen PV timers in one device instance. */ if (devclass_get_device(xentimer_devclass, 0)) return; BUS_ADD_CHILD(parent, 0, "xen_et", 0); } static int xentimer_probe(device_t dev) { KASSERT((xen_domain()), ("Trying to use Xen timer on bare metal")); /* * In order to attach, this driver requires the following: * - Vector callback support by the hypervisor, in order to deliver * timer interrupts to the correct CPU for CPUs other than 0. * - Access to the hypervisor shared info page, in order to look up * each VCPU's timer information and the Xen wallclock time. * - The hypervisor must say its PV clock is "safe" to use. * - The hypervisor must support VCPUOP hypercalls. * - The maximum number of CPUs supported by FreeBSD must not exceed * the number of VCPUs supported by the hypervisor. */ #define XTREQUIRES(condition, reason...) \ if (!(condition)) { \ device_printf(dev, ## reason); \ device_detach(dev); \ return (ENXIO); \ } if (xen_hvm_domain()) { XTREQUIRES(xen_vector_callback_enabled, "vector callbacks unavailable\n"); XTREQUIRES(xen_feature(XENFEAT_hvm_safe_pvclock), "HVM safe pvclock unavailable\n"); } XTREQUIRES(HYPERVISOR_shared_info != NULL, "shared info page unavailable\n"); XTREQUIRES(HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, 0, NULL) == 0, "VCPUOPs interface unavailable\n"); #undef XTREQUIRES device_set_desc(dev, "Xen PV Clock"); return (BUS_PROBE_NOWILDCARD); } /* * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction, * yielding a 64-bit result. */ static inline uint64_t scale_delta(uint64_t delta, uint32_t mul_frac, int shift) { uint64_t product; if (shift < 0) delta >>= -shift; else delta <<= shift; #if defined(__i386__) { uint32_t tmp1, tmp2; /** * For i386, the formula looks like: * * lower = (mul_frac * (delta & UINT_MAX)) >> 32 * upper = mul_frac * (delta >> 32) * product = lower + upper */ __asm__ ( "mul %5 ; " "mov %4,%%eax ; " "mov %%edx,%4 ; " "mul %5 ; " "xor %5,%5 ; " "add %4,%%eax ; " "adc %5,%%edx ; " : "=A" (product), "=r" (tmp1), "=r" (tmp2) : "a" ((uint32_t)delta), "1" ((uint32_t)(delta >> 32)), "2" (mul_frac) ); } #elif defined(__amd64__) { unsigned long tmp; __asm__ ( "mulq %[mul_frac] ; shrd $32, %[hi], %[lo]" : [lo]"=a" (product), [hi]"=d" (tmp) : "0" (delta), [mul_frac]"rm"((uint64_t)mul_frac)); } #else #error "xentimer: unsupported architecture" #endif return (product); } static uint64_t get_nsec_offset(struct vcpu_time_info *tinfo) { return (scale_delta(rdtsc() - tinfo->tsc_timestamp, tinfo->tsc_to_system_mul, tinfo->tsc_shift)); } /* * Read the current hypervisor system uptime value from Xen. * See for a description of how this works. */ static uint32_t xen_fetch_vcpu_tinfo(struct vcpu_time_info *dst, struct vcpu_time_info *src) { do { dst->version = src->version; rmb(); dst->tsc_timestamp = src->tsc_timestamp; dst->system_time = src->system_time; dst->tsc_to_system_mul = src->tsc_to_system_mul; dst->tsc_shift = src->tsc_shift; rmb(); } while ((src->version & 1) | (dst->version ^ src->version)); return (dst->version); } /** * \brief Get the current time, in nanoseconds, since the hypervisor booted. * * \param vcpu vcpu_info structure to fetch the time from. * * \note This function returns the current CPU's idea of this value, unless * it happens to be less than another CPU's previously determined value. */ static uint64_t xen_fetch_vcpu_time(struct vcpu_info *vcpu) { struct vcpu_time_info dst; struct vcpu_time_info *src; uint32_t pre_version; uint64_t now; volatile uint64_t last; src = &vcpu->time; do { pre_version = xen_fetch_vcpu_tinfo(&dst, src); barrier(); now = dst.system_time + get_nsec_offset(&dst); barrier(); } while (pre_version != src->version); /* * Enforce a monotonically increasing clock time across all * VCPUs. If our time is too old, use the last time and return. * Otherwise, try to update the last time. */ do { last = xen_timer_last_time; if (last > now) { now = last; break; } } while (!atomic_cmpset_64(&xen_timer_last_time, last, now)); return (now); } static uint32_t xentimer_get_timecount(struct timecounter *tc) { uint64_t vcpu_time; /* * We don't disable preemption here because the worst that can * happen is reading the vcpu_info area of a different CPU than * the one we are currently running on, but that would also * return a valid tc (and we avoid the overhead of * critical_{enter/exit} calls). */ vcpu_time = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); return (vcpu_time & UINT32_MAX); } /** * \brief Fetch the hypervisor boot time, known as the "Xen wallclock". * * \param ts Timespec to store the current stable value. * \param version Pointer to store the corresponding wallclock version. * * \note This value is updated when Domain-0 shifts its clock to follow * clock drift, e.g. as detected by NTP. */ static void xen_fetch_wallclock(struct timespec *ts) { shared_info_t *src = HYPERVISOR_shared_info; uint32_t version = 0; do { version = src->wc_version; rmb(); ts->tv_sec = src->wc_sec; ts->tv_nsec = src->wc_nsec; rmb(); } while ((src->wc_version & 1) | (version ^ src->wc_version)); } static void xen_fetch_uptime(struct timespec *ts) { uint64_t uptime; uptime = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); ts->tv_sec = uptime / NSEC_IN_SEC; ts->tv_nsec = uptime % NSEC_IN_SEC; } static int xentimer_settime(device_t dev __unused, struct timespec *ts) { /* * Don't return EINVAL here; just silently fail if the domain isn't * privileged enough to set the TOD. */ return (0); } /** * \brief Return current time according to the Xen Hypervisor wallclock. * * \param dev Xentimer device. * \param ts Pointer to store the wallclock time. * * \note The Xen time structures document the hypervisor start time and the * uptime-since-hypervisor-start (in nsec.) They need to be combined * in order to calculate a TOD clock. */ static int xentimer_gettime(device_t dev, struct timespec *ts) { struct timespec u_ts; timespecclear(ts); xen_fetch_wallclock(ts); xen_fetch_uptime(&u_ts); timespecadd(ts, &u_ts); return (0); } /** * \brief Handle a timer interrupt for the Xen PV timer driver. * * \param arg Xen timer driver softc that is expecting the interrupt. */ static int xentimer_intr(void *arg) { struct xentimer_softc *sc = (struct xentimer_softc *)arg; struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu); pcpu->last_processed = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); if (pcpu->timer != 0 && sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } static int xentimer_vcpu_start_timer(int vcpu, uint64_t next_time) { struct vcpu_set_singleshot_timer single; single.timeout_abs_ns = next_time; single.flags = VCPU_SSHOTTMR_future; return (HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, vcpu, &single)); } static int xentimer_vcpu_stop_timer(int vcpu) { return (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, vcpu, NULL)); } /** * \brief Set the next oneshot time for the current CPU. * * \param et Xen timer driver event timer to schedule on. * \param first Delta to the next time to schedule the interrupt for. * \param period Not used. * * \note See eventtimers(9) for more information. * \note * * \returns 0 */ static int xentimer_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { int error = 0, i = 0; struct xentimer_softc *sc = et->et_priv; int cpu = PCPU_GET(vcpu_id); struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu); struct vcpu_info *vcpu = DPCPU_GET(vcpu_info); uint64_t first_in_ns, next_time; #ifdef INVARIANTS struct thread *td = curthread; #endif KASSERT(td->td_critnest != 0, ("xentimer_et_start called without preemption disabled")); /* See sbttots() for this formula. */ first_in_ns = (((first >> 32) * NSEC_IN_SEC) + (((uint64_t)NSEC_IN_SEC * (uint32_t)first) >> 32)); /* * Retry any timer scheduling failures, where the hypervisor * returns -ETIME. Sometimes even a 100us timer period isn't large * enough, but larger period instances are relatively uncommon. * * XXX Remove the panics once et_start() and its consumers are * equipped to deal with start failures. */ do { if (++i == 60) panic("can't schedule timer"); next_time = xen_fetch_vcpu_time(vcpu) + first_in_ns; error = xentimer_vcpu_start_timer(cpu, next_time); } while (error == -ETIME); if (error) panic("%s: Error %d setting singleshot timer to %"PRIu64"\n", device_get_nameunit(sc->dev), error, next_time); pcpu->timer = next_time; return (error); } /** * \brief Cancel the event timer's currently running timer, if any. */ static int xentimer_et_stop(struct eventtimer *et) { int cpu = PCPU_GET(vcpu_id); struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu); pcpu->timer = 0; return (xentimer_vcpu_stop_timer(cpu)); } /** * \brief Attach a Xen PV timer driver instance. * * \param dev Bus device object to attach. * * \note * \returns EINVAL */ static int xentimer_attach(device_t dev) { struct xentimer_softc *sc = device_get_softc(dev); int error, i; sc->dev = dev; /* Bind an event channel to a VIRQ on each VCPU. */ CPU_FOREACH(i) { struct xentimer_pcpu_data *pcpu; pcpu = DPCPU_ID_PTR(i, xentimer_pcpu); error = HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, i, NULL); if (error) { device_printf(dev, "Error disabling Xen periodic timer " "on CPU %d\n", i); return (error); } error = xen_intr_bind_virq(dev, VIRQ_TIMER, i, xentimer_intr, NULL, sc, INTR_TYPE_CLK, &pcpu->irq_handle); if (error) { device_printf(dev, "Error %d binding VIRQ_TIMER " "to VCPU %d\n", error, i); return (error); } xen_intr_describe(pcpu->irq_handle, "c%d", i); } /* Register the event timer. */ sc->et.et_name = "XENTIMER"; sc->et.et_quality = XENTIMER_QUALITY; sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_frequency = NSEC_IN_SEC; /* See tstosbt() for this formula */ sc->et.et_min_period = (XENTIMER_MIN_PERIOD_IN_NSEC * (((uint64_t)1 << 63) / 500000000) >> 32); sc->et.et_max_period = ((sbintime_t)4 << 32); sc->et.et_start = xentimer_et_start; sc->et.et_stop = xentimer_et_stop; sc->et.et_priv = sc; et_register(&sc->et); /* Register the timecounter. */ sc->tc.tc_name = "XENTIMER"; sc->tc.tc_quality = XENTIMER_QUALITY; sc->tc.tc_flags = TC_FLAGS_SUSPEND_SAFE; /* * The underlying resolution is in nanoseconds, since the timer info * scales TSC frequencies using a fraction that represents time in * terms of nanoseconds. */ sc->tc.tc_frequency = NSEC_IN_SEC; sc->tc.tc_counter_mask = ~0u; sc->tc.tc_get_timecount = xentimer_get_timecount; sc->tc.tc_priv = sc; tc_init(&sc->tc); /* Register the Hypervisor wall clock */ clock_register(dev, XENCLOCK_RESOLUTION); return (0); } static int xentimer_detach(device_t dev) { /* Implement Xen PV clock teardown - XXX see hpet_detach ? */ /* If possible: * 1. need to deregister timecounter * 2. need to deregister event timer * 3. need to deregister virtual IRQ event channels */ return (EBUSY); } static void xentimer_percpu_resume(void *arg) { device_t dev = (device_t) arg; struct xentimer_softc *sc = device_get_softc(dev); xentimer_et_start(&sc->et, sc->et.et_min_period, 0); } static int xentimer_resume(device_t dev) { int error; int i; /* Disable the periodic timer */ CPU_FOREACH(i) { error = HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, i, NULL); if (error != 0) { device_printf(dev, "Error disabling Xen periodic timer on CPU %d\n", i); return (error); } } /* Reset the last uptime value */ xen_timer_last_time = 0; /* Reset the RTC clock */ inittodr(time_second); /* Kick the timers on all CPUs */ smp_rendezvous(NULL, xentimer_percpu_resume, NULL, dev); if (bootverbose) device_printf(dev, "resumed operation after suspension\n"); return (0); } static int xentimer_suspend(device_t dev) { return (0); } /* * Xen early clock init */ void xen_clock_init(void) { } /* * Xen PV DELAY function * * When running on PVH mode we don't have an emulated i8524, so * make use of the Xen time info in order to code a simple DELAY * function that can be used during early boot. */ void xen_delay(int n) { struct vcpu_info *vcpu = &HYPERVISOR_shared_info->vcpu_info[0]; uint64_t end_ns; uint64_t current; end_ns = xen_fetch_vcpu_time(vcpu); end_ns += n * NSEC_IN_USEC; for (;;) { current = xen_fetch_vcpu_time(vcpu); if (current >= end_ns) break; } } static device_method_t xentimer_methods[] = { DEVMETHOD(device_identify, xentimer_identify), DEVMETHOD(device_probe, xentimer_probe), DEVMETHOD(device_attach, xentimer_attach), DEVMETHOD(device_detach, xentimer_detach), DEVMETHOD(device_suspend, xentimer_suspend), DEVMETHOD(device_resume, xentimer_resume), /* clock interface */ DEVMETHOD(clock_gettime, xentimer_gettime), DEVMETHOD(clock_settime, xentimer_settime), DEVMETHOD_END }; static driver_t xentimer_driver = { "xen_et", xentimer_methods, sizeof(struct xentimer_softc), }; DRIVER_MODULE(xentimer, xenpv, xentimer_driver, xentimer_devclass, 0, 0); MODULE_DEPEND(xentimer, xenpv, 1, 1, 1);