xref: /openbmc/linux/arch/x86/xen/time.c (revision f35e839a)
1 /*
2  * Xen time implementation.
3  *
4  * This is implemented in terms of a clocksource driver which uses
5  * the hypervisor clock as a nanosecond timebase, and a clockevent
6  * driver which uses the hypervisor's timer mechanism.
7  *
8  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
9  */
10 #include <linux/kernel.h>
11 #include <linux/interrupt.h>
12 #include <linux/clocksource.h>
13 #include <linux/clockchips.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/math64.h>
16 #include <linux/gfp.h>
17 
18 #include <asm/pvclock.h>
19 #include <asm/xen/hypervisor.h>
20 #include <asm/xen/hypercall.h>
21 
22 #include <xen/events.h>
23 #include <xen/features.h>
24 #include <xen/interface/xen.h>
25 #include <xen/interface/vcpu.h>
26 
27 #include "xen-ops.h"
28 
29 /* Xen may fire a timer up to this many ns early */
30 #define TIMER_SLOP	100000
31 #define NS_PER_TICK	(1000000000LL / HZ)
32 
33 /* runstate info updated by Xen */
34 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
35 
36 /* snapshots of runstate info */
37 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
38 
39 /* unused ns of stolen and blocked time */
40 static DEFINE_PER_CPU(u64, xen_residual_stolen);
41 static DEFINE_PER_CPU(u64, xen_residual_blocked);
42 
43 /* return an consistent snapshot of 64-bit time/counter value */
44 static u64 get64(const u64 *p)
45 {
46 	u64 ret;
47 
48 	if (BITS_PER_LONG < 64) {
49 		u32 *p32 = (u32 *)p;
50 		u32 h, l;
51 
52 		/*
53 		 * Read high then low, and then make sure high is
54 		 * still the same; this will only loop if low wraps
55 		 * and carries into high.
56 		 * XXX some clean way to make this endian-proof?
57 		 */
58 		do {
59 			h = p32[1];
60 			barrier();
61 			l = p32[0];
62 			barrier();
63 		} while (p32[1] != h);
64 
65 		ret = (((u64)h) << 32) | l;
66 	} else
67 		ret = *p;
68 
69 	return ret;
70 }
71 
72 /*
73  * Runstate accounting
74  */
75 static void get_runstate_snapshot(struct vcpu_runstate_info *res)
76 {
77 	u64 state_time;
78 	struct vcpu_runstate_info *state;
79 
80 	BUG_ON(preemptible());
81 
82 	state = &__get_cpu_var(xen_runstate);
83 
84 	/*
85 	 * The runstate info is always updated by the hypervisor on
86 	 * the current CPU, so there's no need to use anything
87 	 * stronger than a compiler barrier when fetching it.
88 	 */
89 	do {
90 		state_time = get64(&state->state_entry_time);
91 		barrier();
92 		*res = *state;
93 		barrier();
94 	} while (get64(&state->state_entry_time) != state_time);
95 }
96 
97 /* return true when a vcpu could run but has no real cpu to run on */
98 bool xen_vcpu_stolen(int vcpu)
99 {
100 	return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
101 }
102 
103 void xen_setup_runstate_info(int cpu)
104 {
105 	struct vcpu_register_runstate_memory_area area;
106 
107 	area.addr.v = &per_cpu(xen_runstate, cpu);
108 
109 	if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
110 			       cpu, &area))
111 		BUG();
112 }
113 
114 static void do_stolen_accounting(void)
115 {
116 	struct vcpu_runstate_info state;
117 	struct vcpu_runstate_info *snap;
118 	s64 blocked, runnable, offline, stolen;
119 	cputime_t ticks;
120 
121 	get_runstate_snapshot(&state);
122 
123 	WARN_ON(state.state != RUNSTATE_running);
124 
125 	snap = &__get_cpu_var(xen_runstate_snapshot);
126 
127 	/* work out how much time the VCPU has not been runn*ing*  */
128 	blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked];
129 	runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
130 	offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
131 
132 	*snap = state;
133 
134 	/* Add the appropriate number of ticks of stolen time,
135 	   including any left-overs from last time. */
136 	stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
137 
138 	if (stolen < 0)
139 		stolen = 0;
140 
141 	ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
142 	__this_cpu_write(xen_residual_stolen, stolen);
143 	account_steal_ticks(ticks);
144 
145 	/* Add the appropriate number of ticks of blocked time,
146 	   including any left-overs from last time. */
147 	blocked += __this_cpu_read(xen_residual_blocked);
148 
149 	if (blocked < 0)
150 		blocked = 0;
151 
152 	ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked);
153 	__this_cpu_write(xen_residual_blocked, blocked);
154 	account_idle_ticks(ticks);
155 }
156 
157 /* Get the TSC speed from Xen */
158 static unsigned long xen_tsc_khz(void)
159 {
160 	struct pvclock_vcpu_time_info *info =
161 		&HYPERVISOR_shared_info->vcpu_info[0].time;
162 
163 	return pvclock_tsc_khz(info);
164 }
165 
166 cycle_t xen_clocksource_read(void)
167 {
168         struct pvclock_vcpu_time_info *src;
169 	cycle_t ret;
170 
171 	preempt_disable_notrace();
172 	src = &__get_cpu_var(xen_vcpu)->time;
173 	ret = pvclock_clocksource_read(src);
174 	preempt_enable_notrace();
175 	return ret;
176 }
177 
178 static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
179 {
180 	return xen_clocksource_read();
181 }
182 
183 static void xen_read_wallclock(struct timespec *ts)
184 {
185 	struct shared_info *s = HYPERVISOR_shared_info;
186 	struct pvclock_wall_clock *wall_clock = &(s->wc);
187         struct pvclock_vcpu_time_info *vcpu_time;
188 
189 	vcpu_time = &get_cpu_var(xen_vcpu)->time;
190 	pvclock_read_wallclock(wall_clock, vcpu_time, ts);
191 	put_cpu_var(xen_vcpu);
192 }
193 
194 static unsigned long xen_get_wallclock(void)
195 {
196 	struct timespec ts;
197 
198 	xen_read_wallclock(&ts);
199 	return ts.tv_sec;
200 }
201 
202 static int xen_set_wallclock(unsigned long now)
203 {
204 	struct xen_platform_op op;
205 	int rc;
206 
207 	/* do nothing for domU */
208 	if (!xen_initial_domain())
209 		return -1;
210 
211 	op.cmd = XENPF_settime;
212 	op.u.settime.secs = now;
213 	op.u.settime.nsecs = 0;
214 	op.u.settime.system_time = xen_clocksource_read();
215 
216 	rc = HYPERVISOR_dom0_op(&op);
217 	WARN(rc != 0, "XENPF_settime failed: now=%ld\n", now);
218 
219 	return rc;
220 }
221 
222 static struct clocksource xen_clocksource __read_mostly = {
223 	.name = "xen",
224 	.rating = 400,
225 	.read = xen_clocksource_get_cycles,
226 	.mask = ~0,
227 	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
228 };
229 
230 /*
231    Xen clockevent implementation
232 
233    Xen has two clockevent implementations:
234 
235    The old timer_op one works with all released versions of Xen prior
236    to version 3.0.4.  This version of the hypervisor provides a
237    single-shot timer with nanosecond resolution.  However, sharing the
238    same event channel is a 100Hz tick which is delivered while the
239    vcpu is running.  We don't care about or use this tick, but it will
240    cause the core time code to think the timer fired too soon, and
241    will end up resetting it each time.  It could be filtered, but
242    doing so has complications when the ktime clocksource is not yet
243    the xen clocksource (ie, at boot time).
244 
245    The new vcpu_op-based timer interface allows the tick timer period
246    to be changed or turned off.  The tick timer is not useful as a
247    periodic timer because events are only delivered to running vcpus.
248    The one-shot timer can report when a timeout is in the past, so
249    set_next_event is capable of returning -ETIME when appropriate.
250    This interface is used when available.
251 */
252 
253 
254 /*
255   Get a hypervisor absolute time.  In theory we could maintain an
256   offset between the kernel's time and the hypervisor's time, and
257   apply that to a kernel's absolute timeout.  Unfortunately the
258   hypervisor and kernel times can drift even if the kernel is using
259   the Xen clocksource, because ntp can warp the kernel's clocksource.
260 */
261 static s64 get_abs_timeout(unsigned long delta)
262 {
263 	return xen_clocksource_read() + delta;
264 }
265 
266 static void xen_timerop_set_mode(enum clock_event_mode mode,
267 				 struct clock_event_device *evt)
268 {
269 	switch (mode) {
270 	case CLOCK_EVT_MODE_PERIODIC:
271 		/* unsupported */
272 		WARN_ON(1);
273 		break;
274 
275 	case CLOCK_EVT_MODE_ONESHOT:
276 	case CLOCK_EVT_MODE_RESUME:
277 		break;
278 
279 	case CLOCK_EVT_MODE_UNUSED:
280 	case CLOCK_EVT_MODE_SHUTDOWN:
281 		HYPERVISOR_set_timer_op(0);  /* cancel timeout */
282 		break;
283 	}
284 }
285 
286 static int xen_timerop_set_next_event(unsigned long delta,
287 				      struct clock_event_device *evt)
288 {
289 	WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
290 
291 	if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
292 		BUG();
293 
294 	/* We may have missed the deadline, but there's no real way of
295 	   knowing for sure.  If the event was in the past, then we'll
296 	   get an immediate interrupt. */
297 
298 	return 0;
299 }
300 
301 static const struct clock_event_device xen_timerop_clockevent = {
302 	.name = "xen",
303 	.features = CLOCK_EVT_FEAT_ONESHOT,
304 
305 	.max_delta_ns = 0xffffffff,
306 	.min_delta_ns = TIMER_SLOP,
307 
308 	.mult = 1,
309 	.shift = 0,
310 	.rating = 500,
311 
312 	.set_mode = xen_timerop_set_mode,
313 	.set_next_event = xen_timerop_set_next_event,
314 };
315 
316 
317 
318 static void xen_vcpuop_set_mode(enum clock_event_mode mode,
319 				struct clock_event_device *evt)
320 {
321 	int cpu = smp_processor_id();
322 
323 	switch (mode) {
324 	case CLOCK_EVT_MODE_PERIODIC:
325 		WARN_ON(1);	/* unsupported */
326 		break;
327 
328 	case CLOCK_EVT_MODE_ONESHOT:
329 		if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
330 			BUG();
331 		break;
332 
333 	case CLOCK_EVT_MODE_UNUSED:
334 	case CLOCK_EVT_MODE_SHUTDOWN:
335 		if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
336 		    HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
337 			BUG();
338 		break;
339 	case CLOCK_EVT_MODE_RESUME:
340 		break;
341 	}
342 }
343 
344 static int xen_vcpuop_set_next_event(unsigned long delta,
345 				     struct clock_event_device *evt)
346 {
347 	int cpu = smp_processor_id();
348 	struct vcpu_set_singleshot_timer single;
349 	int ret;
350 
351 	WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
352 
353 	single.timeout_abs_ns = get_abs_timeout(delta);
354 	single.flags = VCPU_SSHOTTMR_future;
355 
356 	ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
357 
358 	BUG_ON(ret != 0 && ret != -ETIME);
359 
360 	return ret;
361 }
362 
363 static const struct clock_event_device xen_vcpuop_clockevent = {
364 	.name = "xen",
365 	.features = CLOCK_EVT_FEAT_ONESHOT,
366 
367 	.max_delta_ns = 0xffffffff,
368 	.min_delta_ns = TIMER_SLOP,
369 
370 	.mult = 1,
371 	.shift = 0,
372 	.rating = 500,
373 
374 	.set_mode = xen_vcpuop_set_mode,
375 	.set_next_event = xen_vcpuop_set_next_event,
376 };
377 
378 static const struct clock_event_device *xen_clockevent =
379 	&xen_timerop_clockevent;
380 static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events) = { .irq = -1 };
381 
382 static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
383 {
384 	struct clock_event_device *evt = &__get_cpu_var(xen_clock_events);
385 	irqreturn_t ret;
386 
387 	ret = IRQ_NONE;
388 	if (evt->event_handler) {
389 		evt->event_handler(evt);
390 		ret = IRQ_HANDLED;
391 	}
392 
393 	do_stolen_accounting();
394 
395 	return ret;
396 }
397 
398 void xen_setup_timer(int cpu)
399 {
400 	const char *name;
401 	struct clock_event_device *evt;
402 	int irq;
403 
404 	evt = &per_cpu(xen_clock_events, cpu);
405 	WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
406 
407 	printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
408 
409 	name = kasprintf(GFP_KERNEL, "timer%d", cpu);
410 	if (!name)
411 		name = "<timer kasprintf failed>";
412 
413 	irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
414 				      IRQF_DISABLED|IRQF_PERCPU|
415 				      IRQF_NOBALANCING|IRQF_TIMER|
416 				      IRQF_FORCE_RESUME,
417 				      name, NULL);
418 
419 	memcpy(evt, xen_clockevent, sizeof(*evt));
420 
421 	evt->cpumask = cpumask_of(cpu);
422 	evt->irq = irq;
423 }
424 
425 void xen_teardown_timer(int cpu)
426 {
427 	struct clock_event_device *evt;
428 	BUG_ON(cpu == 0);
429 	evt = &per_cpu(xen_clock_events, cpu);
430 	unbind_from_irqhandler(evt->irq, NULL);
431 	evt->irq = -1;
432 }
433 
434 void xen_setup_cpu_clockevents(void)
435 {
436 	BUG_ON(preemptible());
437 
438 	clockevents_register_device(&__get_cpu_var(xen_clock_events));
439 }
440 
441 void xen_timer_resume(void)
442 {
443 	int cpu;
444 
445 	pvclock_resume();
446 
447 	if (xen_clockevent != &xen_vcpuop_clockevent)
448 		return;
449 
450 	for_each_online_cpu(cpu) {
451 		if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
452 			BUG();
453 	}
454 }
455 
456 static const struct pv_time_ops xen_time_ops __initconst = {
457 	.sched_clock = xen_clocksource_read,
458 };
459 
460 static void __init xen_time_init(void)
461 {
462 	int cpu = smp_processor_id();
463 	struct timespec tp;
464 
465 	clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
466 
467 	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
468 		/* Successfully turned off 100Hz tick, so we have the
469 		   vcpuop-based timer interface */
470 		printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
471 		xen_clockevent = &xen_vcpuop_clockevent;
472 	}
473 
474 	/* Set initial system time with full resolution */
475 	xen_read_wallclock(&tp);
476 	do_settimeofday(&tp);
477 
478 	setup_force_cpu_cap(X86_FEATURE_TSC);
479 
480 	xen_setup_runstate_info(cpu);
481 	xen_setup_timer(cpu);
482 	xen_setup_cpu_clockevents();
483 }
484 
485 void __init xen_init_time_ops(void)
486 {
487 	pv_time_ops = xen_time_ops;
488 
489 	x86_init.timers.timer_init = xen_time_init;
490 	x86_init.timers.setup_percpu_clockev = x86_init_noop;
491 	x86_cpuinit.setup_percpu_clockev = x86_init_noop;
492 
493 	x86_platform.calibrate_tsc = xen_tsc_khz;
494 	x86_platform.get_wallclock = xen_get_wallclock;
495 	x86_platform.set_wallclock = xen_set_wallclock;
496 }
497 
498 #ifdef CONFIG_XEN_PVHVM
499 static void xen_hvm_setup_cpu_clockevents(void)
500 {
501 	int cpu = smp_processor_id();
502 	xen_setup_runstate_info(cpu);
503 	/*
504 	 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
505 	 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
506 	 * early bootup and also during CPU hotplug events).
507 	 */
508 	xen_setup_cpu_clockevents();
509 }
510 
511 void __init xen_hvm_init_time_ops(void)
512 {
513 	/* vector callback is needed otherwise we cannot receive interrupts
514 	 * on cpu > 0 and at this point we don't know how many cpus are
515 	 * available */
516 	if (!xen_have_vector_callback)
517 		return;
518 	if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
519 		printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
520 				"disable pv timer\n");
521 		return;
522 	}
523 
524 	pv_time_ops = xen_time_ops;
525 	x86_init.timers.setup_percpu_clockev = xen_time_init;
526 	x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
527 
528 	x86_platform.calibrate_tsc = xen_tsc_khz;
529 	x86_platform.get_wallclock = xen_get_wallclock;
530 	x86_platform.set_wallclock = xen_set_wallclock;
531 }
532 #endif
533