1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Clocksource driver for the synthetic counter and timers
5  * provided by the Hyper-V hypervisor to guest VMs, as described
6  * in the Hyper-V Top Level Functional Spec (TLFS). This driver
7  * is instruction set architecture independent.
8  *
9  * Copyright (C) 2019, Microsoft, Inc.
10  *
11  * Author:  Michael Kelley <mikelley@microsoft.com>
12  */
13 
14 #include <linux/percpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/clockchips.h>
17 #include <linux/clocksource.h>
18 #include <linux/sched_clock.h>
19 #include <linux/mm.h>
20 #include <linux/cpuhotplug.h>
21 #include <linux/interrupt.h>
22 #include <linux/irq.h>
23 #include <linux/acpi.h>
24 #include <linux/hyperv.h>
25 #include <clocksource/hyperv_timer.h>
26 #include <asm/hyperv-tlfs.h>
27 #include <asm/mshyperv.h>
28 
29 static struct clock_event_device __percpu *hv_clock_event;
30 static u64 hv_sched_clock_offset __ro_after_init;
31 
32 /*
33  * If false, we're using the old mechanism for stimer0 interrupts
34  * where it sends a VMbus message when it expires. The old
35  * mechanism is used when running on older versions of Hyper-V
36  * that don't support Direct Mode. While Hyper-V provides
37  * four stimer's per CPU, Linux uses only stimer0.
38  *
39  * Because Direct Mode does not require processing a VMbus
40  * message, stimer interrupts can be enabled earlier in the
41  * process of booting a CPU, and consistent with when timer
42  * interrupts are enabled for other clocksource drivers.
43  * However, for legacy versions of Hyper-V when Direct Mode
44  * is not enabled, setting up stimer interrupts must be
45  * delayed until VMbus is initialized and can process the
46  * interrupt message.
47  */
48 static bool direct_mode_enabled;
49 
50 static int stimer0_irq = -1;
51 static int stimer0_message_sint;
52 static __maybe_unused DEFINE_PER_CPU(long, stimer0_evt);
53 
54 /*
55  * Common code for stimer0 interrupts coming via Direct Mode or
56  * as a VMbus message.
57  */
58 void hv_stimer0_isr(void)
59 {
60 	struct clock_event_device *ce;
61 
62 	ce = this_cpu_ptr(hv_clock_event);
63 	ce->event_handler(ce);
64 }
65 EXPORT_SYMBOL_GPL(hv_stimer0_isr);
66 
67 /*
68  * stimer0 interrupt handler for architectures that support
69  * per-cpu interrupts, which also implies Direct Mode.
70  */
71 static irqreturn_t __maybe_unused hv_stimer0_percpu_isr(int irq, void *dev_id)
72 {
73 	hv_stimer0_isr();
74 	return IRQ_HANDLED;
75 }
76 
77 static int hv_ce_set_next_event(unsigned long delta,
78 				struct clock_event_device *evt)
79 {
80 	u64 current_tick;
81 
82 	current_tick = hv_read_reference_counter();
83 	current_tick += delta;
84 	hv_set_register(HV_REGISTER_STIMER0_COUNT, current_tick);
85 	return 0;
86 }
87 
88 static int hv_ce_shutdown(struct clock_event_device *evt)
89 {
90 	hv_set_register(HV_REGISTER_STIMER0_COUNT, 0);
91 	hv_set_register(HV_REGISTER_STIMER0_CONFIG, 0);
92 	if (direct_mode_enabled && stimer0_irq >= 0)
93 		disable_percpu_irq(stimer0_irq);
94 
95 	return 0;
96 }
97 
98 static int hv_ce_set_oneshot(struct clock_event_device *evt)
99 {
100 	union hv_stimer_config timer_cfg;
101 
102 	timer_cfg.as_uint64 = 0;
103 	timer_cfg.enable = 1;
104 	timer_cfg.auto_enable = 1;
105 	if (direct_mode_enabled) {
106 		/*
107 		 * When it expires, the timer will directly interrupt
108 		 * on the specified hardware vector/IRQ.
109 		 */
110 		timer_cfg.direct_mode = 1;
111 		timer_cfg.apic_vector = HYPERV_STIMER0_VECTOR;
112 		if (stimer0_irq >= 0)
113 			enable_percpu_irq(stimer0_irq, IRQ_TYPE_NONE);
114 	} else {
115 		/*
116 		 * When it expires, the timer will generate a VMbus message,
117 		 * to be handled by the normal VMbus interrupt handler.
118 		 */
119 		timer_cfg.direct_mode = 0;
120 		timer_cfg.sintx = stimer0_message_sint;
121 	}
122 	hv_set_register(HV_REGISTER_STIMER0_CONFIG, timer_cfg.as_uint64);
123 	return 0;
124 }
125 
126 /*
127  * hv_stimer_init - Per-cpu initialization of the clockevent
128  */
129 static int hv_stimer_init(unsigned int cpu)
130 {
131 	struct clock_event_device *ce;
132 
133 	if (!hv_clock_event)
134 		return 0;
135 
136 	ce = per_cpu_ptr(hv_clock_event, cpu);
137 	ce->name = "Hyper-V clockevent";
138 	ce->features = CLOCK_EVT_FEAT_ONESHOT;
139 	ce->cpumask = cpumask_of(cpu);
140 	ce->rating = 1000;
141 	ce->set_state_shutdown = hv_ce_shutdown;
142 	ce->set_state_oneshot = hv_ce_set_oneshot;
143 	ce->set_next_event = hv_ce_set_next_event;
144 
145 	clockevents_config_and_register(ce,
146 					HV_CLOCK_HZ,
147 					HV_MIN_DELTA_TICKS,
148 					HV_MAX_MAX_DELTA_TICKS);
149 	return 0;
150 }
151 
152 /*
153  * hv_stimer_cleanup - Per-cpu cleanup of the clockevent
154  */
155 int hv_stimer_cleanup(unsigned int cpu)
156 {
157 	struct clock_event_device *ce;
158 
159 	if (!hv_clock_event)
160 		return 0;
161 
162 	/*
163 	 * In the legacy case where Direct Mode is not enabled
164 	 * (which can only be on x86/64), stimer cleanup happens
165 	 * relatively early in the CPU offlining process. We
166 	 * must unbind the stimer-based clockevent device so
167 	 * that the LAPIC timer can take over until clockevents
168 	 * are no longer needed in the offlining process. Note
169 	 * that clockevents_unbind_device() eventually calls
170 	 * hv_ce_shutdown().
171 	 *
172 	 * The unbind should not be done when Direct Mode is
173 	 * enabled because we may be on an architecture where
174 	 * there are no other clockevent devices to fallback to.
175 	 */
176 	ce = per_cpu_ptr(hv_clock_event, cpu);
177 	if (direct_mode_enabled)
178 		hv_ce_shutdown(ce);
179 	else
180 		clockevents_unbind_device(ce, cpu);
181 
182 	return 0;
183 }
184 EXPORT_SYMBOL_GPL(hv_stimer_cleanup);
185 
186 /*
187  * These placeholders are overridden by arch specific code on
188  * architectures that need special setup of the stimer0 IRQ because
189  * they don't support per-cpu IRQs (such as x86/x64).
190  */
191 void __weak hv_setup_stimer0_handler(void (*handler)(void))
192 {
193 };
194 
195 void __weak hv_remove_stimer0_handler(void)
196 {
197 };
198 
199 #ifdef CONFIG_ACPI
200 /* Called only on architectures with per-cpu IRQs (i.e., not x86/x64) */
201 static int hv_setup_stimer0_irq(void)
202 {
203 	int ret;
204 
205 	ret = acpi_register_gsi(NULL, HYPERV_STIMER0_VECTOR,
206 			ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH);
207 	if (ret < 0) {
208 		pr_err("Can't register Hyper-V stimer0 GSI. Error %d", ret);
209 		return ret;
210 	}
211 	stimer0_irq = ret;
212 
213 	ret = request_percpu_irq(stimer0_irq, hv_stimer0_percpu_isr,
214 		"Hyper-V stimer0", &stimer0_evt);
215 	if (ret) {
216 		pr_err("Can't request Hyper-V stimer0 IRQ %d. Error %d",
217 			stimer0_irq, ret);
218 		acpi_unregister_gsi(stimer0_irq);
219 		stimer0_irq = -1;
220 	}
221 	return ret;
222 }
223 
224 static void hv_remove_stimer0_irq(void)
225 {
226 	if (stimer0_irq == -1) {
227 		hv_remove_stimer0_handler();
228 	} else {
229 		free_percpu_irq(stimer0_irq, &stimer0_evt);
230 		acpi_unregister_gsi(stimer0_irq);
231 		stimer0_irq = -1;
232 	}
233 }
234 #else
235 static int hv_setup_stimer0_irq(void)
236 {
237 	return 0;
238 }
239 
240 static void hv_remove_stimer0_irq(void)
241 {
242 }
243 #endif
244 
245 /* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */
246 int hv_stimer_alloc(bool have_percpu_irqs)
247 {
248 	int ret;
249 
250 	/*
251 	 * Synthetic timers are always available except on old versions of
252 	 * Hyper-V on x86.  In that case, return as error as Linux will use a
253 	 * clockevent based on emulated LAPIC timer hardware.
254 	 */
255 	if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE))
256 		return -EINVAL;
257 
258 	hv_clock_event = alloc_percpu(struct clock_event_device);
259 	if (!hv_clock_event)
260 		return -ENOMEM;
261 
262 	direct_mode_enabled = ms_hyperv.misc_features &
263 			HV_STIMER_DIRECT_MODE_AVAILABLE;
264 
265 	/*
266 	 * If Direct Mode isn't enabled, the remainder of the initialization
267 	 * is done later by hv_stimer_legacy_init()
268 	 */
269 	if (!direct_mode_enabled)
270 		return 0;
271 
272 	if (have_percpu_irqs) {
273 		ret = hv_setup_stimer0_irq();
274 		if (ret)
275 			goto free_clock_event;
276 	} else {
277 		hv_setup_stimer0_handler(hv_stimer0_isr);
278 	}
279 
280 	/*
281 	 * Since we are in Direct Mode, stimer initialization
282 	 * can be done now with a CPUHP value in the same range
283 	 * as other clockevent devices.
284 	 */
285 	ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING,
286 			"clockevents/hyperv/stimer:starting",
287 			hv_stimer_init, hv_stimer_cleanup);
288 	if (ret < 0) {
289 		hv_remove_stimer0_irq();
290 		goto free_clock_event;
291 	}
292 	return ret;
293 
294 free_clock_event:
295 	free_percpu(hv_clock_event);
296 	hv_clock_event = NULL;
297 	return ret;
298 }
299 EXPORT_SYMBOL_GPL(hv_stimer_alloc);
300 
301 /*
302  * hv_stimer_legacy_init -- Called from the VMbus driver to handle
303  * the case when Direct Mode is not enabled, and the stimer
304  * must be initialized late in the CPU onlining process.
305  *
306  */
307 void hv_stimer_legacy_init(unsigned int cpu, int sint)
308 {
309 	if (direct_mode_enabled)
310 		return;
311 
312 	/*
313 	 * This function gets called by each vCPU, so setting the
314 	 * global stimer_message_sint value each time is conceptually
315 	 * not ideal, but the value passed in is always the same and
316 	 * it avoids introducing yet another interface into this
317 	 * clocksource driver just to set the sint in the legacy case.
318 	 */
319 	stimer0_message_sint = sint;
320 	(void)hv_stimer_init(cpu);
321 }
322 EXPORT_SYMBOL_GPL(hv_stimer_legacy_init);
323 
324 /*
325  * hv_stimer_legacy_cleanup -- Called from the VMbus driver to
326  * handle the case when Direct Mode is not enabled, and the
327  * stimer must be cleaned up early in the CPU offlining
328  * process.
329  */
330 void hv_stimer_legacy_cleanup(unsigned int cpu)
331 {
332 	if (direct_mode_enabled)
333 		return;
334 	(void)hv_stimer_cleanup(cpu);
335 }
336 EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup);
337 
338 /*
339  * Do a global cleanup of clockevents for the cases of kexec and
340  * vmbus exit
341  */
342 void hv_stimer_global_cleanup(void)
343 {
344 	int	cpu;
345 
346 	/*
347 	 * hv_stime_legacy_cleanup() will stop the stimer if Direct
348 	 * Mode is not enabled, and fallback to the LAPIC timer.
349 	 */
350 	for_each_present_cpu(cpu) {
351 		hv_stimer_legacy_cleanup(cpu);
352 	}
353 
354 	if (!hv_clock_event)
355 		return;
356 
357 	if (direct_mode_enabled) {
358 		cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING);
359 		hv_remove_stimer0_irq();
360 		stimer0_irq = -1;
361 	}
362 	free_percpu(hv_clock_event);
363 	hv_clock_event = NULL;
364 
365 }
366 EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup);
367 
368 static __always_inline u64 read_hv_clock_msr(void)
369 {
370 	/*
371 	 * Read the partition counter to get the current tick count. This count
372 	 * is set to 0 when the partition is created and is incremented in 100
373 	 * nanosecond units.
374 	 *
375 	 * Use hv_raw_get_register() because this function is used from
376 	 * noinstr. Notable; while HV_REGISTER_TIME_REF_COUNT is a synthetic
377 	 * register it doesn't need the GHCB path.
378 	 */
379 	return hv_raw_get_register(HV_REGISTER_TIME_REF_COUNT);
380 }
381 
382 /*
383  * Code and definitions for the Hyper-V clocksources.  Two
384  * clocksources are defined: one that reads the Hyper-V defined MSR, and
385  * the other that uses the TSC reference page feature as defined in the
386  * TLFS.  The MSR version is for compatibility with old versions of
387  * Hyper-V and 32-bit x86.  The TSC reference page version is preferred.
388  */
389 
390 static union {
391 	struct ms_hyperv_tsc_page page;
392 	u8 reserved[PAGE_SIZE];
393 } tsc_pg __bss_decrypted __aligned(PAGE_SIZE);
394 
395 static struct ms_hyperv_tsc_page *tsc_page = &tsc_pg.page;
396 static unsigned long tsc_pfn;
397 
398 unsigned long hv_get_tsc_pfn(void)
399 {
400 	return tsc_pfn;
401 }
402 EXPORT_SYMBOL_GPL(hv_get_tsc_pfn);
403 
404 struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
405 {
406 	return tsc_page;
407 }
408 EXPORT_SYMBOL_GPL(hv_get_tsc_page);
409 
410 static __always_inline u64 read_hv_clock_tsc(void)
411 {
412 	u64 cur_tsc, time;
413 
414 	/*
415 	 * The Hyper-V Top-Level Function Spec (TLFS), section Timers,
416 	 * subsection Refererence Counter, guarantees that the TSC and MSR
417 	 * times are in sync and monotonic. Therefore we can fall back
418 	 * to the MSR in case the TSC page indicates unavailability.
419 	 */
420 	if (!hv_read_tsc_page_tsc(tsc_page, &cur_tsc, &time))
421 		time = read_hv_clock_msr();
422 
423 	return time;
424 }
425 
426 static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg)
427 {
428 	return read_hv_clock_tsc();
429 }
430 
431 static u64 noinstr read_hv_sched_clock_tsc(void)
432 {
433 	return (read_hv_clock_tsc() - hv_sched_clock_offset) *
434 		(NSEC_PER_SEC / HV_CLOCK_HZ);
435 }
436 
437 static void suspend_hv_clock_tsc(struct clocksource *arg)
438 {
439 	union hv_reference_tsc_msr tsc_msr;
440 
441 	/* Disable the TSC page */
442 	tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC);
443 	tsc_msr.enable = 0;
444 	hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64);
445 }
446 
447 
448 static void resume_hv_clock_tsc(struct clocksource *arg)
449 {
450 	union hv_reference_tsc_msr tsc_msr;
451 
452 	/* Re-enable the TSC page */
453 	tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC);
454 	tsc_msr.enable = 1;
455 	tsc_msr.pfn = tsc_pfn;
456 	hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64);
457 }
458 
459 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK
460 static int hv_cs_enable(struct clocksource *cs)
461 {
462 	vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK);
463 	return 0;
464 }
465 #endif
466 
467 static struct clocksource hyperv_cs_tsc = {
468 	.name	= "hyperv_clocksource_tsc_page",
469 	.rating	= 500,
470 	.read	= read_hv_clock_tsc_cs,
471 	.mask	= CLOCKSOURCE_MASK(64),
472 	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
473 	.suspend= suspend_hv_clock_tsc,
474 	.resume	= resume_hv_clock_tsc,
475 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK
476 	.enable = hv_cs_enable,
477 	.vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK,
478 #else
479 	.vdso_clock_mode = VDSO_CLOCKMODE_NONE,
480 #endif
481 };
482 
483 static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg)
484 {
485 	return read_hv_clock_msr();
486 }
487 
488 static struct clocksource hyperv_cs_msr = {
489 	.name	= "hyperv_clocksource_msr",
490 	.rating	= 495,
491 	.read	= read_hv_clock_msr_cs,
492 	.mask	= CLOCKSOURCE_MASK(64),
493 	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
494 };
495 
496 /*
497  * Reference to pv_ops must be inline so objtool
498  * detection of noinstr violations can work correctly.
499  */
500 #ifdef CONFIG_GENERIC_SCHED_CLOCK
501 static __always_inline void hv_setup_sched_clock(void *sched_clock)
502 {
503 	/*
504 	 * We're on an architecture with generic sched clock (not x86/x64).
505 	 * The Hyper-V sched clock read function returns nanoseconds, not
506 	 * the normal 100ns units of the Hyper-V synthetic clock.
507 	 */
508 	sched_clock_register(sched_clock, 64, NSEC_PER_SEC);
509 }
510 #elif defined CONFIG_PARAVIRT
511 static __always_inline void hv_setup_sched_clock(void *sched_clock)
512 {
513 	/* We're on x86/x64 *and* using PV ops */
514 	paravirt_set_sched_clock(sched_clock);
515 }
516 #else /* !CONFIG_GENERIC_SCHED_CLOCK && !CONFIG_PARAVIRT */
517 static __always_inline void hv_setup_sched_clock(void *sched_clock) {}
518 #endif /* CONFIG_GENERIC_SCHED_CLOCK */
519 
520 static void __init hv_init_tsc_clocksource(void)
521 {
522 	union hv_reference_tsc_msr tsc_msr;
523 
524 	/*
525 	 * If Hyper-V offers TSC_INVARIANT, then the virtualized TSC correctly
526 	 * handles frequency and offset changes due to live migration,
527 	 * pause/resume, and other VM management operations.  So lower the
528 	 * Hyper-V Reference TSC rating, causing the generic TSC to be used.
529 	 * TSC_INVARIANT is not offered on ARM64, so the Hyper-V Reference
530 	 * TSC will be preferred over the virtualized ARM64 arch counter.
531 	 */
532 	if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) {
533 		hyperv_cs_tsc.rating = 250;
534 		hyperv_cs_msr.rating = 245;
535 	}
536 
537 	if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
538 		return;
539 
540 	hv_read_reference_counter = read_hv_clock_tsc;
541 
542 	/*
543 	 * TSC page mapping works differently in root compared to guest.
544 	 * - In guest partition the guest PFN has to be passed to the
545 	 *   hypervisor.
546 	 * - In root partition it's other way around: it has to map the PFN
547 	 *   provided by the hypervisor.
548 	 *   But it can't be mapped right here as it's too early and MMU isn't
549 	 *   ready yet. So, we only set the enable bit here and will remap the
550 	 *   page later in hv_remap_tsc_clocksource().
551 	 *
552 	 * It worth mentioning, that TSC clocksource read function
553 	 * (read_hv_clock_tsc) has a MSR-based fallback mechanism, used when
554 	 * TSC page is zeroed (which is the case until the PFN is remapped) and
555 	 * thus TSC clocksource will work even without the real TSC page
556 	 * mapped.
557 	 */
558 	tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC);
559 	if (hv_root_partition)
560 		tsc_pfn = tsc_msr.pfn;
561 	else
562 		tsc_pfn = HVPFN_DOWN(virt_to_phys(tsc_page));
563 	tsc_msr.enable = 1;
564 	tsc_msr.pfn = tsc_pfn;
565 	hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64);
566 
567 	clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
568 
569 	/*
570 	 * If TSC is invariant, then let it stay as the sched clock since it
571 	 * will be faster than reading the TSC page. But if not invariant, use
572 	 * the TSC page so that live migrations across hosts with different
573 	 * frequencies is handled correctly.
574 	 */
575 	if (!(ms_hyperv.features & HV_ACCESS_TSC_INVARIANT)) {
576 		hv_sched_clock_offset = hv_read_reference_counter();
577 		hv_setup_sched_clock(read_hv_sched_clock_tsc);
578 	}
579 }
580 
581 void __init hv_init_clocksource(void)
582 {
583 	/*
584 	 * Try to set up the TSC page clocksource, then the MSR clocksource.
585 	 * At least one of these will always be available except on very old
586 	 * versions of Hyper-V on x86.  In that case we won't have a Hyper-V
587 	 * clocksource, but Linux will still run with a clocksource based
588 	 * on the emulated PIT or LAPIC timer.
589 	 *
590 	 * Never use the MSR clocksource as sched clock.  It's too slow.
591 	 * Better to use the native sched clock as the fallback.
592 	 */
593 	hv_init_tsc_clocksource();
594 
595 	if (ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE)
596 		clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
597 }
598 
599 void __init hv_remap_tsc_clocksource(void)
600 {
601 	if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
602 		return;
603 
604 	if (!hv_root_partition) {
605 		WARN(1, "%s: attempt to remap TSC page in guest partition\n",
606 		     __func__);
607 		return;
608 	}
609 
610 	tsc_page = memremap(tsc_pfn << HV_HYP_PAGE_SHIFT, sizeof(tsc_pg),
611 			    MEMREMAP_WB);
612 	if (!tsc_page)
613 		pr_err("Failed to remap Hyper-V TSC page.\n");
614 }
615