xref: /openbmc/linux/arch/x86/kernel/kvm.c (revision 6c33a6f4)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * KVM paravirt_ops implementation
4  *
5  * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6  * Copyright IBM Corporation, 2007
7  *   Authors: Anthony Liguori <aliguori@us.ibm.com>
8  */
9 
10 #include <linux/context_tracking.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/kvm_para.h>
14 #include <linux/cpu.h>
15 #include <linux/mm.h>
16 #include <linux/highmem.h>
17 #include <linux/hardirq.h>
18 #include <linux/notifier.h>
19 #include <linux/reboot.h>
20 #include <linux/hash.h>
21 #include <linux/sched.h>
22 #include <linux/slab.h>
23 #include <linux/kprobes.h>
24 #include <linux/debugfs.h>
25 #include <linux/nmi.h>
26 #include <linux/swait.h>
27 #include <asm/timer.h>
28 #include <asm/cpu.h>
29 #include <asm/traps.h>
30 #include <asm/desc.h>
31 #include <asm/tlbflush.h>
32 #include <asm/apic.h>
33 #include <asm/apicdef.h>
34 #include <asm/hypervisor.h>
35 #include <asm/tlb.h>
36 #include <asm/cpuidle_haltpoll.h>
37 
38 static int kvmapf = 1;
39 
40 static int __init parse_no_kvmapf(char *arg)
41 {
42         kvmapf = 0;
43         return 0;
44 }
45 
46 early_param("no-kvmapf", parse_no_kvmapf);
47 
48 static int steal_acc = 1;
49 static int __init parse_no_stealacc(char *arg)
50 {
51         steal_acc = 0;
52         return 0;
53 }
54 
55 early_param("no-steal-acc", parse_no_stealacc);
56 
57 static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
58 DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible;
59 static int has_steal_clock = 0;
60 
61 /*
62  * No need for any "IO delay" on KVM
63  */
64 static void kvm_io_delay(void)
65 {
66 }
67 
68 #define KVM_TASK_SLEEP_HASHBITS 8
69 #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
70 
71 struct kvm_task_sleep_node {
72 	struct hlist_node link;
73 	struct swait_queue_head wq;
74 	u32 token;
75 	int cpu;
76 	bool halted;
77 };
78 
79 static struct kvm_task_sleep_head {
80 	raw_spinlock_t lock;
81 	struct hlist_head list;
82 } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
83 
84 static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
85 						  u32 token)
86 {
87 	struct hlist_node *p;
88 
89 	hlist_for_each(p, &b->list) {
90 		struct kvm_task_sleep_node *n =
91 			hlist_entry(p, typeof(*n), link);
92 		if (n->token == token)
93 			return n;
94 	}
95 
96 	return NULL;
97 }
98 
99 /*
100  * @interrupt_kernel: Is this called from a routine which interrupts the kernel
101  * 		      (other than user space)?
102  */
103 void kvm_async_pf_task_wait(u32 token, int interrupt_kernel)
104 {
105 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
106 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
107 	struct kvm_task_sleep_node n, *e;
108 	DECLARE_SWAITQUEUE(wait);
109 
110 	rcu_irq_enter();
111 
112 	raw_spin_lock(&b->lock);
113 	e = _find_apf_task(b, token);
114 	if (e) {
115 		/* dummy entry exist -> wake up was delivered ahead of PF */
116 		hlist_del(&e->link);
117 		kfree(e);
118 		raw_spin_unlock(&b->lock);
119 
120 		rcu_irq_exit();
121 		return;
122 	}
123 
124 	n.token = token;
125 	n.cpu = smp_processor_id();
126 	n.halted = is_idle_task(current) ||
127 		   (IS_ENABLED(CONFIG_PREEMPT_COUNT)
128 		    ? preempt_count() > 1 || rcu_preempt_depth()
129 		    : interrupt_kernel);
130 	init_swait_queue_head(&n.wq);
131 	hlist_add_head(&n.link, &b->list);
132 	raw_spin_unlock(&b->lock);
133 
134 	for (;;) {
135 		if (!n.halted)
136 			prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
137 		if (hlist_unhashed(&n.link))
138 			break;
139 
140 		rcu_irq_exit();
141 
142 		if (!n.halted) {
143 			local_irq_enable();
144 			schedule();
145 			local_irq_disable();
146 		} else {
147 			/*
148 			 * We cannot reschedule. So halt.
149 			 */
150 			native_safe_halt();
151 			local_irq_disable();
152 		}
153 
154 		rcu_irq_enter();
155 	}
156 	if (!n.halted)
157 		finish_swait(&n.wq, &wait);
158 
159 	rcu_irq_exit();
160 	return;
161 }
162 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait);
163 
164 static void apf_task_wake_one(struct kvm_task_sleep_node *n)
165 {
166 	hlist_del_init(&n->link);
167 	if (n->halted)
168 		smp_send_reschedule(n->cpu);
169 	else if (swq_has_sleeper(&n->wq))
170 		swake_up_one(&n->wq);
171 }
172 
173 static void apf_task_wake_all(void)
174 {
175 	int i;
176 
177 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
178 		struct hlist_node *p, *next;
179 		struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
180 		raw_spin_lock(&b->lock);
181 		hlist_for_each_safe(p, next, &b->list) {
182 			struct kvm_task_sleep_node *n =
183 				hlist_entry(p, typeof(*n), link);
184 			if (n->cpu == smp_processor_id())
185 				apf_task_wake_one(n);
186 		}
187 		raw_spin_unlock(&b->lock);
188 	}
189 }
190 
191 void kvm_async_pf_task_wake(u32 token)
192 {
193 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
194 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
195 	struct kvm_task_sleep_node *n;
196 
197 	if (token == ~0) {
198 		apf_task_wake_all();
199 		return;
200 	}
201 
202 again:
203 	raw_spin_lock(&b->lock);
204 	n = _find_apf_task(b, token);
205 	if (!n) {
206 		/*
207 		 * async PF was not yet handled.
208 		 * Add dummy entry for the token.
209 		 */
210 		n = kzalloc(sizeof(*n), GFP_ATOMIC);
211 		if (!n) {
212 			/*
213 			 * Allocation failed! Busy wait while other cpu
214 			 * handles async PF.
215 			 */
216 			raw_spin_unlock(&b->lock);
217 			cpu_relax();
218 			goto again;
219 		}
220 		n->token = token;
221 		n->cpu = smp_processor_id();
222 		init_swait_queue_head(&n->wq);
223 		hlist_add_head(&n->link, &b->list);
224 	} else
225 		apf_task_wake_one(n);
226 	raw_spin_unlock(&b->lock);
227 	return;
228 }
229 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
230 
231 u32 kvm_read_and_reset_pf_reason(void)
232 {
233 	u32 reason = 0;
234 
235 	if (__this_cpu_read(apf_reason.enabled)) {
236 		reason = __this_cpu_read(apf_reason.reason);
237 		__this_cpu_write(apf_reason.reason, 0);
238 	}
239 
240 	return reason;
241 }
242 EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
243 NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
244 
245 dotraplinkage void
246 do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
247 {
248 	switch (kvm_read_and_reset_pf_reason()) {
249 	default:
250 		do_page_fault(regs, error_code, address);
251 		break;
252 	case KVM_PV_REASON_PAGE_NOT_PRESENT:
253 		/* page is swapped out by the host. */
254 		kvm_async_pf_task_wait((u32)address, !user_mode(regs));
255 		break;
256 	case KVM_PV_REASON_PAGE_READY:
257 		rcu_irq_enter();
258 		kvm_async_pf_task_wake((u32)address);
259 		rcu_irq_exit();
260 		break;
261 	}
262 }
263 NOKPROBE_SYMBOL(do_async_page_fault);
264 
265 static void __init paravirt_ops_setup(void)
266 {
267 	pv_info.name = "KVM";
268 
269 	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
270 		pv_ops.cpu.io_delay = kvm_io_delay;
271 
272 #ifdef CONFIG_X86_IO_APIC
273 	no_timer_check = 1;
274 #endif
275 }
276 
277 static void kvm_register_steal_time(void)
278 {
279 	int cpu = smp_processor_id();
280 	struct kvm_steal_time *st = &per_cpu(steal_time, cpu);
281 
282 	if (!has_steal_clock)
283 		return;
284 
285 	wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
286 	pr_info("kvm-stealtime: cpu %d, msr %llx\n",
287 		cpu, (unsigned long long) slow_virt_to_phys(st));
288 }
289 
290 static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
291 
292 static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val)
293 {
294 	/**
295 	 * This relies on __test_and_clear_bit to modify the memory
296 	 * in a way that is atomic with respect to the local CPU.
297 	 * The hypervisor only accesses this memory from the local CPU so
298 	 * there's no need for lock or memory barriers.
299 	 * An optimization barrier is implied in apic write.
300 	 */
301 	if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi)))
302 		return;
303 	apic->native_eoi_write(APIC_EOI, APIC_EOI_ACK);
304 }
305 
306 static void kvm_guest_cpu_init(void)
307 {
308 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
309 		u64 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason));
310 
311 #ifdef CONFIG_PREEMPTION
312 		pa |= KVM_ASYNC_PF_SEND_ALWAYS;
313 #endif
314 		pa |= KVM_ASYNC_PF_ENABLED;
315 
316 		if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
317 			pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
318 
319 		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
320 		__this_cpu_write(apf_reason.enabled, 1);
321 		printk(KERN_INFO"KVM setup async PF for cpu %d\n",
322 		       smp_processor_id());
323 	}
324 
325 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) {
326 		unsigned long pa;
327 		/* Size alignment is implied but just to make it explicit. */
328 		BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
329 		__this_cpu_write(kvm_apic_eoi, 0);
330 		pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi))
331 			| KVM_MSR_ENABLED;
332 		wrmsrl(MSR_KVM_PV_EOI_EN, pa);
333 	}
334 
335 	if (has_steal_clock)
336 		kvm_register_steal_time();
337 }
338 
339 static void kvm_pv_disable_apf(void)
340 {
341 	if (!__this_cpu_read(apf_reason.enabled))
342 		return;
343 
344 	wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
345 	__this_cpu_write(apf_reason.enabled, 0);
346 
347 	printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
348 	       smp_processor_id());
349 }
350 
351 static void kvm_pv_guest_cpu_reboot(void *unused)
352 {
353 	/*
354 	 * We disable PV EOI before we load a new kernel by kexec,
355 	 * since MSR_KVM_PV_EOI_EN stores a pointer into old kernel's memory.
356 	 * New kernel can re-enable when it boots.
357 	 */
358 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
359 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
360 	kvm_pv_disable_apf();
361 	kvm_disable_steal_time();
362 }
363 
364 static int kvm_pv_reboot_notify(struct notifier_block *nb,
365 				unsigned long code, void *unused)
366 {
367 	if (code == SYS_RESTART)
368 		on_each_cpu(kvm_pv_guest_cpu_reboot, NULL, 1);
369 	return NOTIFY_DONE;
370 }
371 
372 static struct notifier_block kvm_pv_reboot_nb = {
373 	.notifier_call = kvm_pv_reboot_notify,
374 };
375 
376 static u64 kvm_steal_clock(int cpu)
377 {
378 	u64 steal;
379 	struct kvm_steal_time *src;
380 	int version;
381 
382 	src = &per_cpu(steal_time, cpu);
383 	do {
384 		version = src->version;
385 		virt_rmb();
386 		steal = src->steal;
387 		virt_rmb();
388 	} while ((version & 1) || (version != src->version));
389 
390 	return steal;
391 }
392 
393 void kvm_disable_steal_time(void)
394 {
395 	if (!has_steal_clock)
396 		return;
397 
398 	wrmsr(MSR_KVM_STEAL_TIME, 0, 0);
399 }
400 
401 static inline void __set_percpu_decrypted(void *ptr, unsigned long size)
402 {
403 	early_set_memory_decrypted((unsigned long) ptr, size);
404 }
405 
406 /*
407  * Iterate through all possible CPUs and map the memory region pointed
408  * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once.
409  *
410  * Note: we iterate through all possible CPUs to ensure that CPUs
411  * hotplugged will have their per-cpu variable already mapped as
412  * decrypted.
413  */
414 static void __init sev_map_percpu_data(void)
415 {
416 	int cpu;
417 
418 	if (!sev_active())
419 		return;
420 
421 	for_each_possible_cpu(cpu) {
422 		__set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
423 		__set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
424 		__set_percpu_decrypted(&per_cpu(kvm_apic_eoi, cpu), sizeof(kvm_apic_eoi));
425 	}
426 }
427 
428 static bool pv_tlb_flush_supported(void)
429 {
430 	return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
431 		!kvm_para_has_hint(KVM_HINTS_REALTIME) &&
432 		kvm_para_has_feature(KVM_FEATURE_STEAL_TIME));
433 }
434 
435 static DEFINE_PER_CPU(cpumask_var_t, __pv_cpu_mask);
436 
437 #ifdef CONFIG_SMP
438 
439 static bool pv_ipi_supported(void)
440 {
441 	return kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI);
442 }
443 
444 static bool pv_sched_yield_supported(void)
445 {
446 	return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) &&
447 		!kvm_para_has_hint(KVM_HINTS_REALTIME) &&
448 	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME));
449 }
450 
451 #define KVM_IPI_CLUSTER_SIZE	(2 * BITS_PER_LONG)
452 
453 static void __send_ipi_mask(const struct cpumask *mask, int vector)
454 {
455 	unsigned long flags;
456 	int cpu, apic_id, icr;
457 	int min = 0, max = 0;
458 #ifdef CONFIG_X86_64
459 	__uint128_t ipi_bitmap = 0;
460 #else
461 	u64 ipi_bitmap = 0;
462 #endif
463 	long ret;
464 
465 	if (cpumask_empty(mask))
466 		return;
467 
468 	local_irq_save(flags);
469 
470 	switch (vector) {
471 	default:
472 		icr = APIC_DM_FIXED | vector;
473 		break;
474 	case NMI_VECTOR:
475 		icr = APIC_DM_NMI;
476 		break;
477 	}
478 
479 	for_each_cpu(cpu, mask) {
480 		apic_id = per_cpu(x86_cpu_to_apicid, cpu);
481 		if (!ipi_bitmap) {
482 			min = max = apic_id;
483 		} else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) {
484 			ipi_bitmap <<= min - apic_id;
485 			min = apic_id;
486 		} else if (apic_id < min + KVM_IPI_CLUSTER_SIZE) {
487 			max = apic_id < max ? max : apic_id;
488 		} else {
489 			ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
490 				(unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
491 			WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret);
492 			min = max = apic_id;
493 			ipi_bitmap = 0;
494 		}
495 		__set_bit(apic_id - min, (unsigned long *)&ipi_bitmap);
496 	}
497 
498 	if (ipi_bitmap) {
499 		ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
500 			(unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
501 		WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret);
502 	}
503 
504 	local_irq_restore(flags);
505 }
506 
507 static void kvm_send_ipi_mask(const struct cpumask *mask, int vector)
508 {
509 	__send_ipi_mask(mask, vector);
510 }
511 
512 static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
513 {
514 	unsigned int this_cpu = smp_processor_id();
515 	struct cpumask *new_mask = this_cpu_cpumask_var_ptr(__pv_cpu_mask);
516 	const struct cpumask *local_mask;
517 
518 	cpumask_copy(new_mask, mask);
519 	cpumask_clear_cpu(this_cpu, new_mask);
520 	local_mask = new_mask;
521 	__send_ipi_mask(local_mask, vector);
522 }
523 
524 /*
525  * Set the IPI entry points
526  */
527 static void kvm_setup_pv_ipi(void)
528 {
529 	apic->send_IPI_mask = kvm_send_ipi_mask;
530 	apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself;
531 	pr_info("KVM setup pv IPIs\n");
532 }
533 
534 static void kvm_smp_send_call_func_ipi(const struct cpumask *mask)
535 {
536 	int cpu;
537 
538 	native_send_call_func_ipi(mask);
539 
540 	/* Make sure other vCPUs get a chance to run if they need to. */
541 	for_each_cpu(cpu, mask) {
542 		if (vcpu_is_preempted(cpu)) {
543 			kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu));
544 			break;
545 		}
546 	}
547 }
548 
549 static void __init kvm_smp_prepare_cpus(unsigned int max_cpus)
550 {
551 	native_smp_prepare_cpus(max_cpus);
552 	if (kvm_para_has_hint(KVM_HINTS_REALTIME))
553 		static_branch_disable(&virt_spin_lock_key);
554 }
555 
556 static void __init kvm_smp_prepare_boot_cpu(void)
557 {
558 	/*
559 	 * Map the per-cpu variables as decrypted before kvm_guest_cpu_init()
560 	 * shares the guest physical address with the hypervisor.
561 	 */
562 	sev_map_percpu_data();
563 
564 	kvm_guest_cpu_init();
565 	native_smp_prepare_boot_cpu();
566 	kvm_spinlock_init();
567 }
568 
569 static void kvm_guest_cpu_offline(void)
570 {
571 	kvm_disable_steal_time();
572 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
573 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
574 	kvm_pv_disable_apf();
575 	apf_task_wake_all();
576 }
577 
578 static int kvm_cpu_online(unsigned int cpu)
579 {
580 	local_irq_disable();
581 	kvm_guest_cpu_init();
582 	local_irq_enable();
583 	return 0;
584 }
585 
586 static int kvm_cpu_down_prepare(unsigned int cpu)
587 {
588 	local_irq_disable();
589 	kvm_guest_cpu_offline();
590 	local_irq_enable();
591 	return 0;
592 }
593 #endif
594 
595 static void __init kvm_apf_trap_init(void)
596 {
597 	update_intr_gate(X86_TRAP_PF, async_page_fault);
598 }
599 
600 
601 static void kvm_flush_tlb_others(const struct cpumask *cpumask,
602 			const struct flush_tlb_info *info)
603 {
604 	u8 state;
605 	int cpu;
606 	struct kvm_steal_time *src;
607 	struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask);
608 
609 	cpumask_copy(flushmask, cpumask);
610 	/*
611 	 * We have to call flush only on online vCPUs. And
612 	 * queue flush_on_enter for pre-empted vCPUs
613 	 */
614 	for_each_cpu(cpu, flushmask) {
615 		src = &per_cpu(steal_time, cpu);
616 		state = READ_ONCE(src->preempted);
617 		if ((state & KVM_VCPU_PREEMPTED)) {
618 			if (try_cmpxchg(&src->preempted, &state,
619 					state | KVM_VCPU_FLUSH_TLB))
620 				__cpumask_clear_cpu(cpu, flushmask);
621 		}
622 	}
623 
624 	native_flush_tlb_others(flushmask, info);
625 }
626 
627 static void __init kvm_guest_init(void)
628 {
629 	int i;
630 
631 	paravirt_ops_setup();
632 	register_reboot_notifier(&kvm_pv_reboot_nb);
633 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
634 		raw_spin_lock_init(&async_pf_sleepers[i].lock);
635 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
636 		x86_init.irqs.trap_init = kvm_apf_trap_init;
637 
638 	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
639 		has_steal_clock = 1;
640 		pv_ops.time.steal_clock = kvm_steal_clock;
641 	}
642 
643 	if (pv_tlb_flush_supported()) {
644 		pv_ops.mmu.flush_tlb_others = kvm_flush_tlb_others;
645 		pv_ops.mmu.tlb_remove_table = tlb_remove_table;
646 		pr_info("KVM setup pv remote TLB flush\n");
647 	}
648 
649 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
650 		apic_set_eoi_write(kvm_guest_apic_eoi_write);
651 
652 #ifdef CONFIG_SMP
653 	smp_ops.smp_prepare_cpus = kvm_smp_prepare_cpus;
654 	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
655 	if (pv_sched_yield_supported()) {
656 		smp_ops.send_call_func_ipi = kvm_smp_send_call_func_ipi;
657 		pr_info("KVM setup pv sched yield\n");
658 	}
659 	if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/kvm:online",
660 				      kvm_cpu_online, kvm_cpu_down_prepare) < 0)
661 		pr_err("kvm_guest: Failed to install cpu hotplug callbacks\n");
662 #else
663 	sev_map_percpu_data();
664 	kvm_guest_cpu_init();
665 #endif
666 
667 	/*
668 	 * Hard lockup detection is enabled by default. Disable it, as guests
669 	 * can get false positives too easily, for example if the host is
670 	 * overcommitted.
671 	 */
672 	hardlockup_detector_disable();
673 }
674 
675 static noinline uint32_t __kvm_cpuid_base(void)
676 {
677 	if (boot_cpu_data.cpuid_level < 0)
678 		return 0;	/* So we don't blow up on old processors */
679 
680 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
681 		return hypervisor_cpuid_base("KVMKVMKVM\0\0\0", 0);
682 
683 	return 0;
684 }
685 
686 static inline uint32_t kvm_cpuid_base(void)
687 {
688 	static int kvm_cpuid_base = -1;
689 
690 	if (kvm_cpuid_base == -1)
691 		kvm_cpuid_base = __kvm_cpuid_base();
692 
693 	return kvm_cpuid_base;
694 }
695 
696 bool kvm_para_available(void)
697 {
698 	return kvm_cpuid_base() != 0;
699 }
700 EXPORT_SYMBOL_GPL(kvm_para_available);
701 
702 unsigned int kvm_arch_para_features(void)
703 {
704 	return cpuid_eax(kvm_cpuid_base() | KVM_CPUID_FEATURES);
705 }
706 
707 unsigned int kvm_arch_para_hints(void)
708 {
709 	return cpuid_edx(kvm_cpuid_base() | KVM_CPUID_FEATURES);
710 }
711 EXPORT_SYMBOL_GPL(kvm_arch_para_hints);
712 
713 static uint32_t __init kvm_detect(void)
714 {
715 	return kvm_cpuid_base();
716 }
717 
718 static void __init kvm_apic_init(void)
719 {
720 #if defined(CONFIG_SMP)
721 	if (pv_ipi_supported())
722 		kvm_setup_pv_ipi();
723 #endif
724 }
725 
726 static void __init kvm_init_platform(void)
727 {
728 	kvmclock_init();
729 	x86_platform.apic_post_init = kvm_apic_init;
730 }
731 
732 const __initconst struct hypervisor_x86 x86_hyper_kvm = {
733 	.name			= "KVM",
734 	.detect			= kvm_detect,
735 	.type			= X86_HYPER_KVM,
736 	.init.guest_late_init	= kvm_guest_init,
737 	.init.x2apic_available	= kvm_para_available,
738 	.init.init_platform	= kvm_init_platform,
739 };
740 
741 static __init int activate_jump_labels(void)
742 {
743 	if (has_steal_clock) {
744 		static_key_slow_inc(&paravirt_steal_enabled);
745 		if (steal_acc)
746 			static_key_slow_inc(&paravirt_steal_rq_enabled);
747 	}
748 
749 	return 0;
750 }
751 arch_initcall(activate_jump_labels);
752 
753 static __init int kvm_alloc_cpumask(void)
754 {
755 	int cpu;
756 	bool alloc = false;
757 
758 	if (!kvm_para_available() || nopv)
759 		return 0;
760 
761 	if (pv_tlb_flush_supported())
762 		alloc = true;
763 
764 #if defined(CONFIG_SMP)
765 	if (pv_ipi_supported())
766 		alloc = true;
767 #endif
768 
769 	if (alloc)
770 		for_each_possible_cpu(cpu) {
771 			zalloc_cpumask_var_node(per_cpu_ptr(&__pv_cpu_mask, cpu),
772 				GFP_KERNEL, cpu_to_node(cpu));
773 		}
774 
775 	return 0;
776 }
777 arch_initcall(kvm_alloc_cpumask);
778 
779 #ifdef CONFIG_PARAVIRT_SPINLOCKS
780 
781 /* Kick a cpu by its apicid. Used to wake up a halted vcpu */
782 static void kvm_kick_cpu(int cpu)
783 {
784 	int apicid;
785 	unsigned long flags = 0;
786 
787 	apicid = per_cpu(x86_cpu_to_apicid, cpu);
788 	kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
789 }
790 
791 #include <asm/qspinlock.h>
792 
793 static void kvm_wait(u8 *ptr, u8 val)
794 {
795 	unsigned long flags;
796 
797 	if (in_nmi())
798 		return;
799 
800 	local_irq_save(flags);
801 
802 	if (READ_ONCE(*ptr) != val)
803 		goto out;
804 
805 	/*
806 	 * halt until it's our turn and kicked. Note that we do safe halt
807 	 * for irq enabled case to avoid hang when lock info is overwritten
808 	 * in irq spinlock slowpath and no spurious interrupt occur to save us.
809 	 */
810 	if (arch_irqs_disabled_flags(flags))
811 		halt();
812 	else
813 		safe_halt();
814 
815 out:
816 	local_irq_restore(flags);
817 }
818 
819 #ifdef CONFIG_X86_32
820 __visible bool __kvm_vcpu_is_preempted(long cpu)
821 {
822 	struct kvm_steal_time *src = &per_cpu(steal_time, cpu);
823 
824 	return !!(src->preempted & KVM_VCPU_PREEMPTED);
825 }
826 PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted);
827 
828 #else
829 
830 #include <asm/asm-offsets.h>
831 
832 extern bool __raw_callee_save___kvm_vcpu_is_preempted(long);
833 
834 /*
835  * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and
836  * restoring to/from the stack.
837  */
838 asm(
839 ".pushsection .text;"
840 ".global __raw_callee_save___kvm_vcpu_is_preempted;"
841 ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
842 "__raw_callee_save___kvm_vcpu_is_preempted:"
843 "movq	__per_cpu_offset(,%rdi,8), %rax;"
844 "cmpb	$0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
845 "setne	%al;"
846 "ret;"
847 ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;"
848 ".popsection");
849 
850 #endif
851 
852 /*
853  * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
854  */
855 void __init kvm_spinlock_init(void)
856 {
857 	/* Does host kernel support KVM_FEATURE_PV_UNHALT? */
858 	if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
859 		return;
860 
861 	if (kvm_para_has_hint(KVM_HINTS_REALTIME))
862 		return;
863 
864 	/* Don't use the pvqspinlock code if there is only 1 vCPU. */
865 	if (num_possible_cpus() == 1)
866 		return;
867 
868 	__pv_init_lock_hash();
869 	pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
870 	pv_ops.lock.queued_spin_unlock =
871 		PV_CALLEE_SAVE(__pv_queued_spin_unlock);
872 	pv_ops.lock.wait = kvm_wait;
873 	pv_ops.lock.kick = kvm_kick_cpu;
874 
875 	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
876 		pv_ops.lock.vcpu_is_preempted =
877 			PV_CALLEE_SAVE(__kvm_vcpu_is_preempted);
878 	}
879 }
880 
881 #endif	/* CONFIG_PARAVIRT_SPINLOCKS */
882 
883 #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
884 
885 static void kvm_disable_host_haltpoll(void *i)
886 {
887 	wrmsrl(MSR_KVM_POLL_CONTROL, 0);
888 }
889 
890 static void kvm_enable_host_haltpoll(void *i)
891 {
892 	wrmsrl(MSR_KVM_POLL_CONTROL, 1);
893 }
894 
895 void arch_haltpoll_enable(unsigned int cpu)
896 {
897 	if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) {
898 		pr_err_once("kvm: host does not support poll control\n");
899 		pr_err_once("kvm: host upgrade recommended\n");
900 		return;
901 	}
902 
903 	/* Enable guest halt poll disables host halt poll */
904 	smp_call_function_single(cpu, kvm_disable_host_haltpoll, NULL, 1);
905 }
906 EXPORT_SYMBOL_GPL(arch_haltpoll_enable);
907 
908 void arch_haltpoll_disable(unsigned int cpu)
909 {
910 	if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
911 		return;
912 
913 	/* Enable guest halt poll disables host halt poll */
914 	smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1);
915 }
916 EXPORT_SYMBOL_GPL(arch_haltpoll_disable);
917 #endif
918