1 /* 2 * Xen time implementation. 3 * 4 * This is implemented in terms of a clocksource driver which uses 5 * the hypervisor clock as a nanosecond timebase, and a clockevent 6 * driver which uses the hypervisor's timer mechanism. 7 * 8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 9 */ 10 #include <linux/kernel.h> 11 #include <linux/interrupt.h> 12 #include <linux/clocksource.h> 13 #include <linux/clockchips.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/math64.h> 16 #include <linux/gfp.h> 17 18 #include <asm/pvclock.h> 19 #include <asm/xen/hypervisor.h> 20 #include <asm/xen/hypercall.h> 21 22 #include <xen/events.h> 23 #include <xen/features.h> 24 #include <xen/interface/xen.h> 25 #include <xen/interface/vcpu.h> 26 27 #include "xen-ops.h" 28 29 /* Xen may fire a timer up to this many ns early */ 30 #define TIMER_SLOP 100000 31 #define NS_PER_TICK (1000000000LL / HZ) 32 33 /* runstate info updated by Xen */ 34 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); 35 36 /* snapshots of runstate info */ 37 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot); 38 39 /* unused ns of stolen and blocked time */ 40 static DEFINE_PER_CPU(u64, xen_residual_stolen); 41 static DEFINE_PER_CPU(u64, xen_residual_blocked); 42 43 /* return an consistent snapshot of 64-bit time/counter value */ 44 static u64 get64(const u64 *p) 45 { 46 u64 ret; 47 48 if (BITS_PER_LONG < 64) { 49 u32 *p32 = (u32 *)p; 50 u32 h, l; 51 52 /* 53 * Read high then low, and then make sure high is 54 * still the same; this will only loop if low wraps 55 * and carries into high. 56 * XXX some clean way to make this endian-proof? 57 */ 58 do { 59 h = p32[1]; 60 barrier(); 61 l = p32[0]; 62 barrier(); 63 } while (p32[1] != h); 64 65 ret = (((u64)h) << 32) | l; 66 } else 67 ret = *p; 68 69 return ret; 70 } 71 72 /* 73 * Runstate accounting 74 */ 75 static void get_runstate_snapshot(struct vcpu_runstate_info *res) 76 { 77 u64 state_time; 78 struct vcpu_runstate_info *state; 79 80 BUG_ON(preemptible()); 81 82 state = &__get_cpu_var(xen_runstate); 83 84 /* 85 * The runstate info is always updated by the hypervisor on 86 * the current CPU, so there's no need to use anything 87 * stronger than a compiler barrier when fetching it. 88 */ 89 do { 90 state_time = get64(&state->state_entry_time); 91 barrier(); 92 *res = *state; 93 barrier(); 94 } while (get64(&state->state_entry_time) != state_time); 95 } 96 97 /* return true when a vcpu could run but has no real cpu to run on */ 98 bool xen_vcpu_stolen(int vcpu) 99 { 100 return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; 101 } 102 103 void xen_setup_runstate_info(int cpu) 104 { 105 struct vcpu_register_runstate_memory_area area; 106 107 area.addr.v = &per_cpu(xen_runstate, cpu); 108 109 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, 110 cpu, &area)) 111 BUG(); 112 } 113 114 static void do_stolen_accounting(void) 115 { 116 struct vcpu_runstate_info state; 117 struct vcpu_runstate_info *snap; 118 s64 blocked, runnable, offline, stolen; 119 cputime_t ticks; 120 121 get_runstate_snapshot(&state); 122 123 WARN_ON(state.state != RUNSTATE_running); 124 125 snap = &__get_cpu_var(xen_runstate_snapshot); 126 127 /* work out how much time the VCPU has not been runn*ing* */ 128 blocked = state.time[RUNSTATE_blocked] - snap->time[RUNSTATE_blocked]; 129 runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable]; 130 offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline]; 131 132 *snap = state; 133 134 /* Add the appropriate number of ticks of stolen time, 135 including any left-overs from last time. */ 136 stolen = runnable + offline + __this_cpu_read(xen_residual_stolen); 137 138 if (stolen < 0) 139 stolen = 0; 140 141 ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen); 142 __this_cpu_write(xen_residual_stolen, stolen); 143 account_steal_ticks(ticks); 144 145 /* Add the appropriate number of ticks of blocked time, 146 including any left-overs from last time. */ 147 blocked += __this_cpu_read(xen_residual_blocked); 148 149 if (blocked < 0) 150 blocked = 0; 151 152 ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked); 153 __this_cpu_write(xen_residual_blocked, blocked); 154 account_idle_ticks(ticks); 155 } 156 157 /* Get the TSC speed from Xen */ 158 static unsigned long xen_tsc_khz(void) 159 { 160 struct pvclock_vcpu_time_info *info = 161 &HYPERVISOR_shared_info->vcpu_info[0].time; 162 163 return pvclock_tsc_khz(info); 164 } 165 166 cycle_t xen_clocksource_read(void) 167 { 168 struct pvclock_vcpu_time_info *src; 169 cycle_t ret; 170 171 preempt_disable_notrace(); 172 src = &__get_cpu_var(xen_vcpu)->time; 173 ret = pvclock_clocksource_read(src); 174 preempt_enable_notrace(); 175 return ret; 176 } 177 178 static cycle_t xen_clocksource_get_cycles(struct clocksource *cs) 179 { 180 return xen_clocksource_read(); 181 } 182 183 static void xen_read_wallclock(struct timespec *ts) 184 { 185 struct shared_info *s = HYPERVISOR_shared_info; 186 struct pvclock_wall_clock *wall_clock = &(s->wc); 187 struct pvclock_vcpu_time_info *vcpu_time; 188 189 vcpu_time = &get_cpu_var(xen_vcpu)->time; 190 pvclock_read_wallclock(wall_clock, vcpu_time, ts); 191 put_cpu_var(xen_vcpu); 192 } 193 194 static unsigned long xen_get_wallclock(void) 195 { 196 struct timespec ts; 197 198 xen_read_wallclock(&ts); 199 return ts.tv_sec; 200 } 201 202 static int xen_set_wallclock(unsigned long now) 203 { 204 struct xen_platform_op op; 205 int rc; 206 207 /* do nothing for domU */ 208 if (!xen_initial_domain()) 209 return -1; 210 211 op.cmd = XENPF_settime; 212 op.u.settime.secs = now; 213 op.u.settime.nsecs = 0; 214 op.u.settime.system_time = xen_clocksource_read(); 215 216 rc = HYPERVISOR_dom0_op(&op); 217 WARN(rc != 0, "XENPF_settime failed: now=%ld\n", now); 218 219 return rc; 220 } 221 222 static struct clocksource xen_clocksource __read_mostly = { 223 .name = "xen", 224 .rating = 400, 225 .read = xen_clocksource_get_cycles, 226 .mask = ~0, 227 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 228 }; 229 230 /* 231 Xen clockevent implementation 232 233 Xen has two clockevent implementations: 234 235 The old timer_op one works with all released versions of Xen prior 236 to version 3.0.4. This version of the hypervisor provides a 237 single-shot timer with nanosecond resolution. However, sharing the 238 same event channel is a 100Hz tick which is delivered while the 239 vcpu is running. We don't care about or use this tick, but it will 240 cause the core time code to think the timer fired too soon, and 241 will end up resetting it each time. It could be filtered, but 242 doing so has complications when the ktime clocksource is not yet 243 the xen clocksource (ie, at boot time). 244 245 The new vcpu_op-based timer interface allows the tick timer period 246 to be changed or turned off. The tick timer is not useful as a 247 periodic timer because events are only delivered to running vcpus. 248 The one-shot timer can report when a timeout is in the past, so 249 set_next_event is capable of returning -ETIME when appropriate. 250 This interface is used when available. 251 */ 252 253 254 /* 255 Get a hypervisor absolute time. In theory we could maintain an 256 offset between the kernel's time and the hypervisor's time, and 257 apply that to a kernel's absolute timeout. Unfortunately the 258 hypervisor and kernel times can drift even if the kernel is using 259 the Xen clocksource, because ntp can warp the kernel's clocksource. 260 */ 261 static s64 get_abs_timeout(unsigned long delta) 262 { 263 return xen_clocksource_read() + delta; 264 } 265 266 static void xen_timerop_set_mode(enum clock_event_mode mode, 267 struct clock_event_device *evt) 268 { 269 switch (mode) { 270 case CLOCK_EVT_MODE_PERIODIC: 271 /* unsupported */ 272 WARN_ON(1); 273 break; 274 275 case CLOCK_EVT_MODE_ONESHOT: 276 case CLOCK_EVT_MODE_RESUME: 277 break; 278 279 case CLOCK_EVT_MODE_UNUSED: 280 case CLOCK_EVT_MODE_SHUTDOWN: 281 HYPERVISOR_set_timer_op(0); /* cancel timeout */ 282 break; 283 } 284 } 285 286 static int xen_timerop_set_next_event(unsigned long delta, 287 struct clock_event_device *evt) 288 { 289 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT); 290 291 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0) 292 BUG(); 293 294 /* We may have missed the deadline, but there's no real way of 295 knowing for sure. If the event was in the past, then we'll 296 get an immediate interrupt. */ 297 298 return 0; 299 } 300 301 static const struct clock_event_device xen_timerop_clockevent = { 302 .name = "xen", 303 .features = CLOCK_EVT_FEAT_ONESHOT, 304 305 .max_delta_ns = 0xffffffff, 306 .min_delta_ns = TIMER_SLOP, 307 308 .mult = 1, 309 .shift = 0, 310 .rating = 500, 311 312 .set_mode = xen_timerop_set_mode, 313 .set_next_event = xen_timerop_set_next_event, 314 }; 315 316 317 318 static void xen_vcpuop_set_mode(enum clock_event_mode mode, 319 struct clock_event_device *evt) 320 { 321 int cpu = smp_processor_id(); 322 323 switch (mode) { 324 case CLOCK_EVT_MODE_PERIODIC: 325 WARN_ON(1); /* unsupported */ 326 break; 327 328 case CLOCK_EVT_MODE_ONESHOT: 329 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) 330 BUG(); 331 break; 332 333 case CLOCK_EVT_MODE_UNUSED: 334 case CLOCK_EVT_MODE_SHUTDOWN: 335 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) || 336 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) 337 BUG(); 338 break; 339 case CLOCK_EVT_MODE_RESUME: 340 break; 341 } 342 } 343 344 static int xen_vcpuop_set_next_event(unsigned long delta, 345 struct clock_event_device *evt) 346 { 347 int cpu = smp_processor_id(); 348 struct vcpu_set_singleshot_timer single; 349 int ret; 350 351 WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT); 352 353 single.timeout_abs_ns = get_abs_timeout(delta); 354 single.flags = VCPU_SSHOTTMR_future; 355 356 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single); 357 358 BUG_ON(ret != 0 && ret != -ETIME); 359 360 return ret; 361 } 362 363 static const struct clock_event_device xen_vcpuop_clockevent = { 364 .name = "xen", 365 .features = CLOCK_EVT_FEAT_ONESHOT, 366 367 .max_delta_ns = 0xffffffff, 368 .min_delta_ns = TIMER_SLOP, 369 370 .mult = 1, 371 .shift = 0, 372 .rating = 500, 373 374 .set_mode = xen_vcpuop_set_mode, 375 .set_next_event = xen_vcpuop_set_next_event, 376 }; 377 378 static const struct clock_event_device *xen_clockevent = 379 &xen_timerop_clockevent; 380 static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events) = { .irq = -1 }; 381 382 static irqreturn_t xen_timer_interrupt(int irq, void *dev_id) 383 { 384 struct clock_event_device *evt = &__get_cpu_var(xen_clock_events); 385 irqreturn_t ret; 386 387 ret = IRQ_NONE; 388 if (evt->event_handler) { 389 evt->event_handler(evt); 390 ret = IRQ_HANDLED; 391 } 392 393 do_stolen_accounting(); 394 395 return ret; 396 } 397 398 void xen_setup_timer(int cpu) 399 { 400 const char *name; 401 struct clock_event_device *evt; 402 int irq; 403 404 evt = &per_cpu(xen_clock_events, cpu); 405 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu); 406 407 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu); 408 409 name = kasprintf(GFP_KERNEL, "timer%d", cpu); 410 if (!name) 411 name = "<timer kasprintf failed>"; 412 413 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt, 414 IRQF_DISABLED|IRQF_PERCPU| 415 IRQF_NOBALANCING|IRQF_TIMER| 416 IRQF_FORCE_RESUME, 417 name, NULL); 418 419 memcpy(evt, xen_clockevent, sizeof(*evt)); 420 421 evt->cpumask = cpumask_of(cpu); 422 evt->irq = irq; 423 } 424 425 void xen_teardown_timer(int cpu) 426 { 427 struct clock_event_device *evt; 428 BUG_ON(cpu == 0); 429 evt = &per_cpu(xen_clock_events, cpu); 430 unbind_from_irqhandler(evt->irq, NULL); 431 evt->irq = -1; 432 } 433 434 void xen_setup_cpu_clockevents(void) 435 { 436 BUG_ON(preemptible()); 437 438 clockevents_register_device(&__get_cpu_var(xen_clock_events)); 439 } 440 441 void xen_timer_resume(void) 442 { 443 int cpu; 444 445 pvclock_resume(); 446 447 if (xen_clockevent != &xen_vcpuop_clockevent) 448 return; 449 450 for_each_online_cpu(cpu) { 451 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) 452 BUG(); 453 } 454 } 455 456 static const struct pv_time_ops xen_time_ops __initconst = { 457 .sched_clock = xen_clocksource_read, 458 }; 459 460 static void __init xen_time_init(void) 461 { 462 int cpu = smp_processor_id(); 463 struct timespec tp; 464 465 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); 466 467 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { 468 /* Successfully turned off 100Hz tick, so we have the 469 vcpuop-based timer interface */ 470 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n"); 471 xen_clockevent = &xen_vcpuop_clockevent; 472 } 473 474 /* Set initial system time with full resolution */ 475 xen_read_wallclock(&tp); 476 do_settimeofday(&tp); 477 478 setup_force_cpu_cap(X86_FEATURE_TSC); 479 480 xen_setup_runstate_info(cpu); 481 xen_setup_timer(cpu); 482 xen_setup_cpu_clockevents(); 483 } 484 485 void __init xen_init_time_ops(void) 486 { 487 pv_time_ops = xen_time_ops; 488 489 x86_init.timers.timer_init = xen_time_init; 490 x86_init.timers.setup_percpu_clockev = x86_init_noop; 491 x86_cpuinit.setup_percpu_clockev = x86_init_noop; 492 493 x86_platform.calibrate_tsc = xen_tsc_khz; 494 x86_platform.get_wallclock = xen_get_wallclock; 495 x86_platform.set_wallclock = xen_set_wallclock; 496 } 497 498 #ifdef CONFIG_XEN_PVHVM 499 static void xen_hvm_setup_cpu_clockevents(void) 500 { 501 int cpu = smp_processor_id(); 502 xen_setup_runstate_info(cpu); 503 /* 504 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence 505 * doing it xen_hvm_cpu_notify (which gets called by smp_init during 506 * early bootup and also during CPU hotplug events). 507 */ 508 xen_setup_cpu_clockevents(); 509 } 510 511 void __init xen_hvm_init_time_ops(void) 512 { 513 /* vector callback is needed otherwise we cannot receive interrupts 514 * on cpu > 0 and at this point we don't know how many cpus are 515 * available */ 516 if (!xen_have_vector_callback) 517 return; 518 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) { 519 printk(KERN_INFO "Xen doesn't support pvclock on HVM," 520 "disable pv timer\n"); 521 return; 522 } 523 524 pv_time_ops = xen_time_ops; 525 x86_init.timers.setup_percpu_clockev = xen_time_init; 526 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; 527 528 x86_platform.calibrate_tsc = xen_tsc_khz; 529 x86_platform.get_wallclock = xen_get_wallclock; 530 x86_platform.set_wallclock = xen_set_wallclock; 531 } 532 #endif 533