2 * Xen time implementation.
4 * This is implemented in terms of a clocksource driver which uses
5 * the hypervisor clock as a nanosecond timebase, and a clockevent
6 * driver which uses the hypervisor's timer mechanism.
8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
10 #include <linux/kernel.h>
11 #include <linux/interrupt.h>
12 #include <linux/clocksource.h>
13 #include <linux/clockchips.h>
14 #include <linux/kernel_stat.h>
16 #include <asm/xen/hypervisor.h>
17 #include <asm/xen/hypercall.h>
19 #include <xen/events.h>
20 #include <xen/interface/xen.h>
21 #include <xen/interface/vcpu.h>
27 /* Xen may fire a timer up to this many ns early */
28 #define TIMER_SLOP 100000
29 #define NS_PER_TICK (1000000000LL / HZ)
31 static cycle_t xen_clocksource_read(void);
33 /* These are perodically updated in shared_info, and then copied here. */
34 struct shadow_time_info {
35 u64 tsc_timestamp; /* TSC at last update of time vals. */
36 u64 system_timestamp; /* Time, in nanosecs, since boot. */
42 static DEFINE_PER_CPU(struct shadow_time_info, shadow_time);
44 /* runstate info updated by Xen */
45 static DEFINE_PER_CPU(struct vcpu_runstate_info, runstate);
47 /* snapshots of runstate info */
48 static DEFINE_PER_CPU(struct vcpu_runstate_info, runstate_snapshot);
50 /* unused ns of stolen and blocked time */
51 static DEFINE_PER_CPU(u64, residual_stolen);
52 static DEFINE_PER_CPU(u64, residual_blocked);
54 /* return an consistent snapshot of 64-bit time/counter value */
55 static u64 get64(const u64 *p)
59 if (BITS_PER_LONG < 64) {
64 * Read high then low, and then make sure high is
65 * still the same; this will only loop if low wraps
66 * and carries into high.
67 * XXX some clean way to make this endian-proof?
74 } while (p32[1] != h);
76 ret = (((u64)h) << 32) | l;
86 static void get_runstate_snapshot(struct vcpu_runstate_info *res)
89 struct vcpu_runstate_info *state;
91 BUG_ON(preemptible());
93 state = &__get_cpu_var(runstate);
96 * The runstate info is always updated by the hypervisor on
97 * the current CPU, so there's no need to use anything
98 * stronger than a compiler barrier when fetching it.
101 state_time = get64(&state->state_entry_time);
105 } while (get64(&state->state_entry_time) != state_time);
108 static void setup_runstate_info(int cpu)
110 struct vcpu_register_runstate_memory_area area;
112 area.addr.v = &per_cpu(runstate, cpu);
114 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
119 static void do_stolen_accounting(void)
121 struct vcpu_runstate_info state;
122 struct vcpu_runstate_info *snap;
123 s64 blocked, runnable, offline, stolen;
126 get_runstate_snapshot(&state);
128 WARN_ON(state.state != RUNSTATE_running);
130 snap = &__get_cpu_var(runstate_snapshot);
132 /* work out how much time the VCPU has not been runn*ing* */
133 blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked];
134 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
135 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
139 /* Add the appropriate number of ticks of stolen time,
140 including any left-overs from last time. Passing NULL to
141 account_steal_time accounts the time as stolen. */
142 stolen = runnable + offline + __get_cpu_var(residual_stolen);
148 while (stolen >= NS_PER_TICK) {
150 stolen -= NS_PER_TICK;
152 __get_cpu_var(residual_stolen) = stolen;
153 account_steal_time(NULL, ticks);
155 /* Add the appropriate number of ticks of blocked time,
156 including any left-overs from last time. Passing idle to
157 account_steal_time accounts the time as idle/wait. */
158 blocked += __get_cpu_var(residual_blocked);
164 while (blocked >= NS_PER_TICK) {
166 blocked -= NS_PER_TICK;
168 __get_cpu_var(residual_blocked) = blocked;
169 account_steal_time(idle_task(smp_processor_id()), ticks);
173 * Xen sched_clock implementation. Returns the number of unstolen
174 * nanoseconds, which is nanoseconds the VCPU spent in RUNNING+BLOCKED
177 unsigned long long xen_sched_clock(void)
179 struct vcpu_runstate_info state;
185 * Ideally sched_clock should be called on a per-cpu basis
186 * anyway, so preempt should already be disabled, but that's
187 * not current practice at the moment.
191 now = xen_clocksource_read();
193 get_runstate_snapshot(&state);
195 WARN_ON(state.state != RUNSTATE_running);
197 offset = now - state.state_entry_time;
201 ret = state.time[RUNSTATE_blocked] +
202 state.time[RUNSTATE_running] +
211 /* Get the CPU speed from Xen */
212 unsigned long xen_cpu_khz(void)
214 u64 cpu_khz = 1000000ULL << 32;
215 const struct vcpu_time_info *info =
216 &HYPERVISOR_shared_info->vcpu_info[0].time;
218 do_div(cpu_khz, info->tsc_to_system_mul);
219 if (info->tsc_shift < 0)
220 cpu_khz <<= -info->tsc_shift;
222 cpu_khz >>= info->tsc_shift;
228 * Reads a consistent set of time-base values from Xen, into a shadow data
231 static unsigned get_time_values_from_xen(void)
233 struct vcpu_time_info *src;
234 struct shadow_time_info *dst;
236 /* src is shared memory with the hypervisor, so we need to
237 make sure we get a consistent snapshot, even in the face of
239 src = &__get_cpu_var(xen_vcpu)->time;
240 dst = &__get_cpu_var(shadow_time);
243 dst->version = src->version;
244 rmb(); /* fetch version before data */
245 dst->tsc_timestamp = src->tsc_timestamp;
246 dst->system_timestamp = src->system_time;
247 dst->tsc_to_nsec_mul = src->tsc_to_system_mul;
248 dst->tsc_shift = src->tsc_shift;
249 rmb(); /* test version after fetching data */
250 } while ((src->version & 1) | (dst->version ^ src->version));
256 * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
257 * yielding a 64-bit result.
259 static inline u64 scale_delta(u64 delta, u32 mul_frac, int shift)
280 : "=A" (product), "=r" (tmp1), "=r" (tmp2)
281 : "a" ((u32)delta), "1" ((u32)(delta >> 32)), "2" (mul_frac) );
284 "mul %%rdx ; shrd $32,%%rdx,%%rax"
285 : "=a" (product) : "0" (delta), "d" ((u64)mul_frac) );
293 static u64 get_nsec_offset(struct shadow_time_info *shadow)
296 now = native_read_tsc();
297 delta = now - shadow->tsc_timestamp;
298 return scale_delta(delta, shadow->tsc_to_nsec_mul, shadow->tsc_shift);
301 static cycle_t xen_clocksource_read(void)
303 struct shadow_time_info *shadow = &get_cpu_var(shadow_time);
308 version = get_time_values_from_xen();
310 ret = shadow->system_timestamp + get_nsec_offset(shadow);
312 } while (version != __get_cpu_var(xen_vcpu)->time.version);
314 put_cpu_var(shadow_time);
319 static void xen_read_wallclock(struct timespec *ts)
321 const struct shared_info *s = HYPERVISOR_shared_info;
326 /* get wallclock at system boot */
328 version = s->wc_version;
329 rmb(); /* fetch version before time */
330 now.tv_sec = s->wc_sec;
331 now.tv_nsec = s->wc_nsec;
332 rmb(); /* fetch time before checking version */
333 } while ((s->wc_version & 1) | (version ^ s->wc_version));
335 delta = xen_clocksource_read(); /* time since system boot */
336 delta += now.tv_sec * (u64)NSEC_PER_SEC + now.tv_nsec;
338 now.tv_nsec = do_div(delta, NSEC_PER_SEC);
341 set_normalized_timespec(ts, now.tv_sec, now.tv_nsec);
344 unsigned long xen_get_wallclock(void)
348 xen_read_wallclock(&ts);
353 int xen_set_wallclock(unsigned long now)
355 /* do nothing for domU */
359 static struct clocksource xen_clocksource __read_mostly = {
362 .read = xen_clocksource_read,
364 .mult = 1<<XEN_SHIFT, /* time directly in nanoseconds */
366 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
370 Xen clockevent implementation
372 Xen has two clockevent implementations:
374 The old timer_op one works with all released versions of Xen prior
375 to version 3.0.4. This version of the hypervisor provides a
376 single-shot timer with nanosecond resolution. However, sharing the
377 same event channel is a 100Hz tick which is delivered while the
378 vcpu is running. We don't care about or use this tick, but it will
379 cause the core time code to think the timer fired too soon, and
380 will end up resetting it each time. It could be filtered, but
381 doing so has complications when the ktime clocksource is not yet
382 the xen clocksource (ie, at boot time).
384 The new vcpu_op-based timer interface allows the tick timer period
385 to be changed or turned off. The tick timer is not useful as a
386 periodic timer because events are only delivered to running vcpus.
387 The one-shot timer can report when a timeout is in the past, so
388 set_next_event is capable of returning -ETIME when appropriate.
389 This interface is used when available.
394 Get a hypervisor absolute time. In theory we could maintain an
395 offset between the kernel's time and the hypervisor's time, and
396 apply that to a kernel's absolute timeout. Unfortunately the
397 hypervisor and kernel times can drift even if the kernel is using
398 the Xen clocksource, because ntp can warp the kernel's clocksource.
400 static s64 get_abs_timeout(unsigned long delta)
402 return xen_clocksource_read() + delta;
405 static void xen_timerop_set_mode(enum clock_event_mode mode,
406 struct clock_event_device *evt)
409 case CLOCK_EVT_MODE_PERIODIC:
414 case CLOCK_EVT_MODE_ONESHOT:
417 case CLOCK_EVT_MODE_UNUSED:
418 case CLOCK_EVT_MODE_SHUTDOWN:
419 HYPERVISOR_set_timer_op(0); /* cancel timeout */
424 static int xen_timerop_set_next_event(unsigned long delta,
425 struct clock_event_device *evt)
427 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
429 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
432 /* We may have missed the deadline, but there's no real way of
433 knowing for sure. If the event was in the past, then we'll
434 get an immediate interrupt. */
439 static const struct clock_event_device xen_timerop_clockevent = {
441 .features = CLOCK_EVT_FEAT_ONESHOT,
443 .max_delta_ns = 0xffffffff,
444 .min_delta_ns = TIMER_SLOP,
450 .set_mode = xen_timerop_set_mode,
451 .set_next_event = xen_timerop_set_next_event,
456 static void xen_vcpuop_set_mode(enum clock_event_mode mode,
457 struct clock_event_device *evt)
459 int cpu = smp_processor_id();
462 case CLOCK_EVT_MODE_PERIODIC:
463 WARN_ON(1); /* unsupported */
466 case CLOCK_EVT_MODE_ONESHOT:
467 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
471 case CLOCK_EVT_MODE_UNUSED:
472 case CLOCK_EVT_MODE_SHUTDOWN:
473 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
474 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
480 static int xen_vcpuop_set_next_event(unsigned long delta,
481 struct clock_event_device *evt)
483 int cpu = smp_processor_id();
484 struct vcpu_set_singleshot_timer single;
487 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
489 single.timeout_abs_ns = get_abs_timeout(delta);
490 single.flags = VCPU_SSHOTTMR_future;
492 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
494 BUG_ON(ret != 0 && ret != -ETIME);
499 static const struct clock_event_device xen_vcpuop_clockevent = {
501 .features = CLOCK_EVT_FEAT_ONESHOT,
503 .max_delta_ns = 0xffffffff,
504 .min_delta_ns = TIMER_SLOP,
510 .set_mode = xen_vcpuop_set_mode,
511 .set_next_event = xen_vcpuop_set_next_event,
514 static const struct clock_event_device *xen_clockevent =
515 &xen_timerop_clockevent;
516 static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events);
518 static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
520 struct clock_event_device *evt = &__get_cpu_var(xen_clock_events);
524 if (evt->event_handler) {
525 evt->event_handler(evt);
529 do_stolen_accounting();
534 void xen_setup_timer(int cpu)
537 struct clock_event_device *evt;
540 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
542 name = kasprintf(GFP_KERNEL, "timer%d", cpu);
544 name = "<timer kasprintf failed>";
546 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
547 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
550 evt = &per_cpu(xen_clock_events, cpu);
551 memcpy(evt, xen_clockevent, sizeof(*evt));
553 evt->cpumask = cpumask_of_cpu(cpu);
556 setup_runstate_info(cpu);
559 void xen_setup_cpu_clockevents(void)
561 BUG_ON(preemptible());
563 clockevents_register_device(&__get_cpu_var(xen_clock_events));
566 __init void xen_time_init(void)
568 int cpu = smp_processor_id();
570 get_time_values_from_xen();
572 clocksource_register(&xen_clocksource);
574 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
575 /* Successfully turned off 100Hz tick, so we have the
576 vcpuop-based timer interface */
577 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
578 xen_clockevent = &xen_vcpuop_clockevent;
581 /* Set initial system time with full resolution */
582 xen_read_wallclock(&xtime);
583 set_normalized_timespec(&wall_to_monotonic,
584 -xtime.tv_sec, -xtime.tv_nsec);
588 xen_setup_timer(cpu);
589 xen_setup_cpu_clockevents();