1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Clocksource driver for the synthetic counter and timers 5 * provided by the Hyper-V hypervisor to guest VMs, as described 6 * in the Hyper-V Top Level Functional Spec (TLFS). This driver 7 * is instruction set architecture independent. 8 * 9 * Copyright (C) 2019, Microsoft, Inc. 10 * 11 * Author: Michael Kelley <mikelley@microsoft.com> 12 */ 13 14 #include <linux/percpu.h> 15 #include <linux/cpumask.h> 16 #include <linux/clockchips.h> 17 #include <linux/clocksource.h> 18 #include <linux/sched_clock.h> 19 #include <linux/mm.h> 20 #include <linux/cpuhotplug.h> 21 #include <linux/interrupt.h> 22 #include <linux/irq.h> 23 #include <linux/acpi.h> 24 #include <linux/hyperv.h> 25 #include <clocksource/hyperv_timer.h> 26 #include <asm/hyperv-tlfs.h> 27 #include <asm/mshyperv.h> 28 29 static struct clock_event_device __percpu *hv_clock_event; 30 static u64 hv_sched_clock_offset __ro_after_init; 31 32 /* 33 * If false, we're using the old mechanism for stimer0 interrupts 34 * where it sends a VMbus message when it expires. The old 35 * mechanism is used when running on older versions of Hyper-V 36 * that don't support Direct Mode. While Hyper-V provides 37 * four stimer's per CPU, Linux uses only stimer0. 38 * 39 * Because Direct Mode does not require processing a VMbus 40 * message, stimer interrupts can be enabled earlier in the 41 * process of booting a CPU, and consistent with when timer 42 * interrupts are enabled for other clocksource drivers. 43 * However, for legacy versions of Hyper-V when Direct Mode 44 * is not enabled, setting up stimer interrupts must be 45 * delayed until VMbus is initialized and can process the 46 * interrupt message. 47 */ 48 static bool direct_mode_enabled; 49 50 static int stimer0_irq = -1; 51 static int stimer0_message_sint; 52 static DEFINE_PER_CPU(long, stimer0_evt); 53 54 /* 55 * Common code for stimer0 interrupts coming via Direct Mode or 56 * as a VMbus message. 57 */ 58 void hv_stimer0_isr(void) 59 { 60 struct clock_event_device *ce; 61 62 ce = this_cpu_ptr(hv_clock_event); 63 ce->event_handler(ce); 64 } 65 EXPORT_SYMBOL_GPL(hv_stimer0_isr); 66 67 /* 68 * stimer0 interrupt handler for architectures that support 69 * per-cpu interrupts, which also implies Direct Mode. 70 */ 71 static irqreturn_t hv_stimer0_percpu_isr(int irq, void *dev_id) 72 { 73 hv_stimer0_isr(); 74 return IRQ_HANDLED; 75 } 76 77 static int hv_ce_set_next_event(unsigned long delta, 78 struct clock_event_device *evt) 79 { 80 u64 current_tick; 81 82 current_tick = hv_read_reference_counter(); 83 current_tick += delta; 84 hv_set_register(HV_REGISTER_STIMER0_COUNT, current_tick); 85 return 0; 86 } 87 88 static int hv_ce_shutdown(struct clock_event_device *evt) 89 { 90 hv_set_register(HV_REGISTER_STIMER0_COUNT, 0); 91 hv_set_register(HV_REGISTER_STIMER0_CONFIG, 0); 92 if (direct_mode_enabled && stimer0_irq >= 0) 93 disable_percpu_irq(stimer0_irq); 94 95 return 0; 96 } 97 98 static int hv_ce_set_oneshot(struct clock_event_device *evt) 99 { 100 union hv_stimer_config timer_cfg; 101 102 timer_cfg.as_uint64 = 0; 103 timer_cfg.enable = 1; 104 timer_cfg.auto_enable = 1; 105 if (direct_mode_enabled) { 106 /* 107 * When it expires, the timer will directly interrupt 108 * on the specified hardware vector/IRQ. 109 */ 110 timer_cfg.direct_mode = 1; 111 timer_cfg.apic_vector = HYPERV_STIMER0_VECTOR; 112 if (stimer0_irq >= 0) 113 enable_percpu_irq(stimer0_irq, IRQ_TYPE_NONE); 114 } else { 115 /* 116 * When it expires, the timer will generate a VMbus message, 117 * to be handled by the normal VMbus interrupt handler. 118 */ 119 timer_cfg.direct_mode = 0; 120 timer_cfg.sintx = stimer0_message_sint; 121 } 122 hv_set_register(HV_REGISTER_STIMER0_CONFIG, timer_cfg.as_uint64); 123 return 0; 124 } 125 126 /* 127 * hv_stimer_init - Per-cpu initialization of the clockevent 128 */ 129 static int hv_stimer_init(unsigned int cpu) 130 { 131 struct clock_event_device *ce; 132 133 if (!hv_clock_event) 134 return 0; 135 136 ce = per_cpu_ptr(hv_clock_event, cpu); 137 ce->name = "Hyper-V clockevent"; 138 ce->features = CLOCK_EVT_FEAT_ONESHOT; 139 ce->cpumask = cpumask_of(cpu); 140 ce->rating = 1000; 141 ce->set_state_shutdown = hv_ce_shutdown; 142 ce->set_state_oneshot = hv_ce_set_oneshot; 143 ce->set_next_event = hv_ce_set_next_event; 144 145 clockevents_config_and_register(ce, 146 HV_CLOCK_HZ, 147 HV_MIN_DELTA_TICKS, 148 HV_MAX_MAX_DELTA_TICKS); 149 return 0; 150 } 151 152 /* 153 * hv_stimer_cleanup - Per-cpu cleanup of the clockevent 154 */ 155 int hv_stimer_cleanup(unsigned int cpu) 156 { 157 struct clock_event_device *ce; 158 159 if (!hv_clock_event) 160 return 0; 161 162 /* 163 * In the legacy case where Direct Mode is not enabled 164 * (which can only be on x86/64), stimer cleanup happens 165 * relatively early in the CPU offlining process. We 166 * must unbind the stimer-based clockevent device so 167 * that the LAPIC timer can take over until clockevents 168 * are no longer needed in the offlining process. Note 169 * that clockevents_unbind_device() eventually calls 170 * hv_ce_shutdown(). 171 * 172 * The unbind should not be done when Direct Mode is 173 * enabled because we may be on an architecture where 174 * there are no other clockevent devices to fallback to. 175 */ 176 ce = per_cpu_ptr(hv_clock_event, cpu); 177 if (direct_mode_enabled) 178 hv_ce_shutdown(ce); 179 else 180 clockevents_unbind_device(ce, cpu); 181 182 return 0; 183 } 184 EXPORT_SYMBOL_GPL(hv_stimer_cleanup); 185 186 /* 187 * These placeholders are overridden by arch specific code on 188 * architectures that need special setup of the stimer0 IRQ because 189 * they don't support per-cpu IRQs (such as x86/x64). 190 */ 191 void __weak hv_setup_stimer0_handler(void (*handler)(void)) 192 { 193 }; 194 195 void __weak hv_remove_stimer0_handler(void) 196 { 197 }; 198 199 /* Called only on architectures with per-cpu IRQs (i.e., not x86/x64) */ 200 static int hv_setup_stimer0_irq(void) 201 { 202 int ret; 203 204 ret = acpi_register_gsi(NULL, HYPERV_STIMER0_VECTOR, 205 ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH); 206 if (ret < 0) { 207 pr_err("Can't register Hyper-V stimer0 GSI. Error %d", ret); 208 return ret; 209 } 210 stimer0_irq = ret; 211 212 ret = request_percpu_irq(stimer0_irq, hv_stimer0_percpu_isr, 213 "Hyper-V stimer0", &stimer0_evt); 214 if (ret) { 215 pr_err("Can't request Hyper-V stimer0 IRQ %d. Error %d", 216 stimer0_irq, ret); 217 acpi_unregister_gsi(stimer0_irq); 218 stimer0_irq = -1; 219 } 220 return ret; 221 } 222 223 static void hv_remove_stimer0_irq(void) 224 { 225 if (stimer0_irq == -1) { 226 hv_remove_stimer0_handler(); 227 } else { 228 free_percpu_irq(stimer0_irq, &stimer0_evt); 229 acpi_unregister_gsi(stimer0_irq); 230 stimer0_irq = -1; 231 } 232 } 233 234 /* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */ 235 int hv_stimer_alloc(bool have_percpu_irqs) 236 { 237 int ret; 238 239 /* 240 * Synthetic timers are always available except on old versions of 241 * Hyper-V on x86. In that case, return as error as Linux will use a 242 * clockevent based on emulated LAPIC timer hardware. 243 */ 244 if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) 245 return -EINVAL; 246 247 hv_clock_event = alloc_percpu(struct clock_event_device); 248 if (!hv_clock_event) 249 return -ENOMEM; 250 251 direct_mode_enabled = ms_hyperv.misc_features & 252 HV_STIMER_DIRECT_MODE_AVAILABLE; 253 254 /* 255 * If Direct Mode isn't enabled, the remainder of the initialization 256 * is done later by hv_stimer_legacy_init() 257 */ 258 if (!direct_mode_enabled) 259 return 0; 260 261 if (have_percpu_irqs) { 262 ret = hv_setup_stimer0_irq(); 263 if (ret) 264 goto free_clock_event; 265 } else { 266 hv_setup_stimer0_handler(hv_stimer0_isr); 267 } 268 269 /* 270 * Since we are in Direct Mode, stimer initialization 271 * can be done now with a CPUHP value in the same range 272 * as other clockevent devices. 273 */ 274 ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING, 275 "clockevents/hyperv/stimer:starting", 276 hv_stimer_init, hv_stimer_cleanup); 277 if (ret < 0) { 278 hv_remove_stimer0_irq(); 279 goto free_clock_event; 280 } 281 return ret; 282 283 free_clock_event: 284 free_percpu(hv_clock_event); 285 hv_clock_event = NULL; 286 return ret; 287 } 288 EXPORT_SYMBOL_GPL(hv_stimer_alloc); 289 290 /* 291 * hv_stimer_legacy_init -- Called from the VMbus driver to handle 292 * the case when Direct Mode is not enabled, and the stimer 293 * must be initialized late in the CPU onlining process. 294 * 295 */ 296 void hv_stimer_legacy_init(unsigned int cpu, int sint) 297 { 298 if (direct_mode_enabled) 299 return; 300 301 /* 302 * This function gets called by each vCPU, so setting the 303 * global stimer_message_sint value each time is conceptually 304 * not ideal, but the value passed in is always the same and 305 * it avoids introducing yet another interface into this 306 * clocksource driver just to set the sint in the legacy case. 307 */ 308 stimer0_message_sint = sint; 309 (void)hv_stimer_init(cpu); 310 } 311 EXPORT_SYMBOL_GPL(hv_stimer_legacy_init); 312 313 /* 314 * hv_stimer_legacy_cleanup -- Called from the VMbus driver to 315 * handle the case when Direct Mode is not enabled, and the 316 * stimer must be cleaned up early in the CPU offlining 317 * process. 318 */ 319 void hv_stimer_legacy_cleanup(unsigned int cpu) 320 { 321 if (direct_mode_enabled) 322 return; 323 (void)hv_stimer_cleanup(cpu); 324 } 325 EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup); 326 327 /* 328 * Do a global cleanup of clockevents for the cases of kexec and 329 * vmbus exit 330 */ 331 void hv_stimer_global_cleanup(void) 332 { 333 int cpu; 334 335 /* 336 * hv_stime_legacy_cleanup() will stop the stimer if Direct 337 * Mode is not enabled, and fallback to the LAPIC timer. 338 */ 339 for_each_present_cpu(cpu) { 340 hv_stimer_legacy_cleanup(cpu); 341 } 342 343 if (!hv_clock_event) 344 return; 345 346 if (direct_mode_enabled) { 347 cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING); 348 hv_remove_stimer0_irq(); 349 stimer0_irq = -1; 350 } 351 free_percpu(hv_clock_event); 352 hv_clock_event = NULL; 353 354 } 355 EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); 356 357 /* 358 * Code and definitions for the Hyper-V clocksources. Two 359 * clocksources are defined: one that reads the Hyper-V defined MSR, and 360 * the other that uses the TSC reference page feature as defined in the 361 * TLFS. The MSR version is for compatibility with old versions of 362 * Hyper-V and 32-bit x86. The TSC reference page version is preferred. 363 */ 364 365 static union { 366 struct ms_hyperv_tsc_page page; 367 u8 reserved[PAGE_SIZE]; 368 } tsc_pg __aligned(PAGE_SIZE); 369 370 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 371 { 372 return &tsc_pg.page; 373 } 374 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 375 376 static u64 notrace read_hv_clock_tsc(void) 377 { 378 u64 current_tick = hv_read_tsc_page(hv_get_tsc_page()); 379 380 if (current_tick == U64_MAX) 381 current_tick = hv_get_register(HV_REGISTER_TIME_REF_COUNT); 382 383 return current_tick; 384 } 385 386 static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg) 387 { 388 return read_hv_clock_tsc(); 389 } 390 391 static u64 notrace read_hv_sched_clock_tsc(void) 392 { 393 return (read_hv_clock_tsc() - hv_sched_clock_offset) * 394 (NSEC_PER_SEC / HV_CLOCK_HZ); 395 } 396 397 static void suspend_hv_clock_tsc(struct clocksource *arg) 398 { 399 union hv_reference_tsc_msr tsc_msr; 400 401 /* Disable the TSC page */ 402 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 403 tsc_msr.enable = 0; 404 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 405 } 406 407 408 static void resume_hv_clock_tsc(struct clocksource *arg) 409 { 410 phys_addr_t phys_addr = virt_to_phys(&tsc_pg); 411 union hv_reference_tsc_msr tsc_msr; 412 413 /* Re-enable the TSC page */ 414 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 415 tsc_msr.enable = 1; 416 tsc_msr.pfn = HVPFN_DOWN(phys_addr); 417 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 418 } 419 420 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 421 static int hv_cs_enable(struct clocksource *cs) 422 { 423 vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); 424 return 0; 425 } 426 #endif 427 428 static struct clocksource hyperv_cs_tsc = { 429 .name = "hyperv_clocksource_tsc_page", 430 .rating = 500, 431 .read = read_hv_clock_tsc_cs, 432 .mask = CLOCKSOURCE_MASK(64), 433 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 434 .suspend= suspend_hv_clock_tsc, 435 .resume = resume_hv_clock_tsc, 436 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 437 .enable = hv_cs_enable, 438 .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK, 439 #else 440 .vdso_clock_mode = VDSO_CLOCKMODE_NONE, 441 #endif 442 }; 443 444 static u64 notrace read_hv_clock_msr(void) 445 { 446 /* 447 * Read the partition counter to get the current tick count. This count 448 * is set to 0 when the partition is created and is incremented in 449 * 100 nanosecond units. 450 */ 451 return hv_get_register(HV_REGISTER_TIME_REF_COUNT); 452 } 453 454 static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg) 455 { 456 return read_hv_clock_msr(); 457 } 458 459 static u64 notrace read_hv_sched_clock_msr(void) 460 { 461 return (read_hv_clock_msr() - hv_sched_clock_offset) * 462 (NSEC_PER_SEC / HV_CLOCK_HZ); 463 } 464 465 static struct clocksource hyperv_cs_msr = { 466 .name = "hyperv_clocksource_msr", 467 .rating = 500, 468 .read = read_hv_clock_msr_cs, 469 .mask = CLOCKSOURCE_MASK(64), 470 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 471 }; 472 473 /* 474 * Reference to pv_ops must be inline so objtool 475 * detection of noinstr violations can work correctly. 476 */ 477 #ifdef CONFIG_GENERIC_SCHED_CLOCK 478 static __always_inline void hv_setup_sched_clock(void *sched_clock) 479 { 480 /* 481 * We're on an architecture with generic sched clock (not x86/x64). 482 * The Hyper-V sched clock read function returns nanoseconds, not 483 * the normal 100ns units of the Hyper-V synthetic clock. 484 */ 485 sched_clock_register(sched_clock, 64, NSEC_PER_SEC); 486 } 487 #elif defined CONFIG_PARAVIRT 488 static __always_inline void hv_setup_sched_clock(void *sched_clock) 489 { 490 /* We're on x86/x64 *and* using PV ops */ 491 paravirt_set_sched_clock(sched_clock); 492 } 493 #else /* !CONFIG_GENERIC_SCHED_CLOCK && !CONFIG_PARAVIRT */ 494 static __always_inline void hv_setup_sched_clock(void *sched_clock) {} 495 #endif /* CONFIG_GENERIC_SCHED_CLOCK */ 496 497 static bool __init hv_init_tsc_clocksource(void) 498 { 499 union hv_reference_tsc_msr tsc_msr; 500 phys_addr_t phys_addr; 501 502 if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) 503 return false; 504 505 if (hv_root_partition) 506 return false; 507 508 /* 509 * If Hyper-V offers TSC_INVARIANT, then the virtualized TSC correctly 510 * handles frequency and offset changes due to live migration, 511 * pause/resume, and other VM management operations. So lower the 512 * Hyper-V Reference TSC rating, causing the generic TSC to be used. 513 * TSC_INVARIANT is not offered on ARM64, so the Hyper-V Reference 514 * TSC will be preferred over the virtualized ARM64 arch counter. 515 * While the Hyper-V MSR clocksource won't be used since the 516 * Reference TSC clocksource is present, change its rating as 517 * well for consistency. 518 */ 519 if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) { 520 hyperv_cs_tsc.rating = 250; 521 hyperv_cs_msr.rating = 250; 522 } 523 524 hv_read_reference_counter = read_hv_clock_tsc; 525 phys_addr = virt_to_phys(hv_get_tsc_page()); 526 527 /* 528 * The Hyper-V TLFS specifies to preserve the value of reserved 529 * bits in registers. So read the existing value, preserve the 530 * low order 12 bits, and add in the guest physical address 531 * (which already has at least the low 12 bits set to zero since 532 * it is page aligned). Also set the "enable" bit, which is bit 0. 533 */ 534 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 535 tsc_msr.enable = 1; 536 tsc_msr.pfn = HVPFN_DOWN(phys_addr); 537 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 538 539 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 540 541 hv_sched_clock_offset = hv_read_reference_counter(); 542 hv_setup_sched_clock(read_hv_sched_clock_tsc); 543 544 return true; 545 } 546 547 void __init hv_init_clocksource(void) 548 { 549 /* 550 * Try to set up the TSC page clocksource. If it succeeds, we're 551 * done. Otherwise, set up the MSR clocksource. At least one of 552 * these will always be available except on very old versions of 553 * Hyper-V on x86. In that case we won't have a Hyper-V 554 * clocksource, but Linux will still run with a clocksource based 555 * on the emulated PIT or LAPIC timer. 556 */ 557 if (hv_init_tsc_clocksource()) 558 return; 559 560 if (!(ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE)) 561 return; 562 563 hv_read_reference_counter = read_hv_clock_msr; 564 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 565 566 hv_sched_clock_offset = hv_read_reference_counter(); 567 hv_setup_sched_clock(read_hv_sched_clock_msr); 568 } 569