1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
4  */
5 
6 #include <linux/cpu.h>
7 #include <linux/kvm_host.h>
8 #include <linux/preempt.h>
9 #include <linux/export.h>
10 #include <linux/sched.h>
11 #include <linux/spinlock.h>
12 #include <linux/init.h>
13 #include <linux/memblock.h>
14 #include <linux/sizes.h>
15 #include <linux/cma.h>
16 #include <linux/bitops.h>
17 
18 #include <asm/cputable.h>
19 #include <asm/interrupt.h>
20 #include <asm/kvm_ppc.h>
21 #include <asm/kvm_book3s.h>
22 #include <asm/machdep.h>
23 #include <asm/xics.h>
24 #include <asm/xive.h>
25 #include <asm/dbell.h>
26 #include <asm/cputhreads.h>
27 #include <asm/io.h>
28 #include <asm/opal.h>
29 #include <asm/smp.h>
30 
31 #define KVM_CMA_CHUNK_ORDER	18
32 
33 #include "book3s_xics.h"
34 #include "book3s_xive.h"
35 #include "book3s_hv.h"
36 
37 /*
38  * Hash page table alignment on newer cpus(CPU_FTR_ARCH_206)
39  * should be power of 2.
40  */
41 #define HPT_ALIGN_PAGES		((1 << 18) >> PAGE_SHIFT) /* 256k */
42 /*
43  * By default we reserve 5% of memory for hash pagetable allocation.
44  */
45 static unsigned long kvm_cma_resv_ratio = 5;
46 
47 static struct cma *kvm_cma;
48 
49 static int __init early_parse_kvm_cma_resv(char *p)
50 {
51 	pr_debug("%s(%s)\n", __func__, p);
52 	if (!p)
53 		return -EINVAL;
54 	return kstrtoul(p, 0, &kvm_cma_resv_ratio);
55 }
56 early_param("kvm_cma_resv_ratio", early_parse_kvm_cma_resv);
57 
58 struct page *kvm_alloc_hpt_cma(unsigned long nr_pages)
59 {
60 	VM_BUG_ON(order_base_2(nr_pages) < KVM_CMA_CHUNK_ORDER - PAGE_SHIFT);
61 
62 	return cma_alloc(kvm_cma, nr_pages, order_base_2(HPT_ALIGN_PAGES),
63 			 false);
64 }
65 EXPORT_SYMBOL_GPL(kvm_alloc_hpt_cma);
66 
67 void kvm_free_hpt_cma(struct page *page, unsigned long nr_pages)
68 {
69 	cma_release(kvm_cma, page, nr_pages);
70 }
71 EXPORT_SYMBOL_GPL(kvm_free_hpt_cma);
72 
73 /**
74  * kvm_cma_reserve() - reserve area for kvm hash pagetable
75  *
76  * This function reserves memory from early allocator. It should be
77  * called by arch specific code once the memblock allocator
78  * has been activated and all other subsystems have already allocated/reserved
79  * memory.
80  */
81 void __init kvm_cma_reserve(void)
82 {
83 	unsigned long align_size;
84 	phys_addr_t selected_size;
85 
86 	/*
87 	 * We need CMA reservation only when we are in HV mode
88 	 */
89 	if (!cpu_has_feature(CPU_FTR_HVMODE))
90 		return;
91 
92 	selected_size = PAGE_ALIGN(memblock_phys_mem_size() * kvm_cma_resv_ratio / 100);
93 	if (selected_size) {
94 		pr_info("%s: reserving %ld MiB for global area\n", __func__,
95 			 (unsigned long)selected_size / SZ_1M);
96 		align_size = HPT_ALIGN_PAGES << PAGE_SHIFT;
97 		cma_declare_contiguous(0, selected_size, 0, align_size,
98 			KVM_CMA_CHUNK_ORDER - PAGE_SHIFT, false, "kvm_cma",
99 			&kvm_cma);
100 	}
101 }
102 
103 /*
104  * Real-mode H_CONFER implementation.
105  * We check if we are the only vcpu out of this virtual core
106  * still running in the guest and not ceded.  If so, we pop up
107  * to the virtual-mode implementation; if not, just return to
108  * the guest.
109  */
110 long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target,
111 			    unsigned int yield_count)
112 {
113 	struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
114 	int ptid = local_paca->kvm_hstate.ptid;
115 	int threads_running;
116 	int threads_ceded;
117 	int threads_conferring;
118 	u64 stop = get_tb() + 10 * tb_ticks_per_usec;
119 	int rv = H_SUCCESS; /* => don't yield */
120 
121 	set_bit(ptid, &vc->conferring_threads);
122 	while ((get_tb() < stop) && !VCORE_IS_EXITING(vc)) {
123 		threads_running = VCORE_ENTRY_MAP(vc);
124 		threads_ceded = vc->napping_threads;
125 		threads_conferring = vc->conferring_threads;
126 		if ((threads_ceded | threads_conferring) == threads_running) {
127 			rv = H_TOO_HARD; /* => do yield */
128 			break;
129 		}
130 	}
131 	clear_bit(ptid, &vc->conferring_threads);
132 	return rv;
133 }
134 
135 /*
136  * When running HV mode KVM we need to block certain operations while KVM VMs
137  * exist in the system. We use a counter of VMs to track this.
138  *
139  * One of the operations we need to block is onlining of secondaries, so we
140  * protect hv_vm_count with cpus_read_lock/unlock().
141  */
142 static atomic_t hv_vm_count;
143 
144 void kvm_hv_vm_activated(void)
145 {
146 	cpus_read_lock();
147 	atomic_inc(&hv_vm_count);
148 	cpus_read_unlock();
149 }
150 EXPORT_SYMBOL_GPL(kvm_hv_vm_activated);
151 
152 void kvm_hv_vm_deactivated(void)
153 {
154 	cpus_read_lock();
155 	atomic_dec(&hv_vm_count);
156 	cpus_read_unlock();
157 }
158 EXPORT_SYMBOL_GPL(kvm_hv_vm_deactivated);
159 
160 bool kvm_hv_mode_active(void)
161 {
162 	return atomic_read(&hv_vm_count) != 0;
163 }
164 
165 extern int hcall_real_table[], hcall_real_table_end[];
166 
167 int kvmppc_hcall_impl_hv_realmode(unsigned long cmd)
168 {
169 	cmd /= 4;
170 	if (cmd < hcall_real_table_end - hcall_real_table &&
171 	    hcall_real_table[cmd])
172 		return 1;
173 
174 	return 0;
175 }
176 EXPORT_SYMBOL_GPL(kvmppc_hcall_impl_hv_realmode);
177 
178 int kvmppc_hwrng_present(void)
179 {
180 	return ppc_md.get_random_seed != NULL;
181 }
182 EXPORT_SYMBOL_GPL(kvmppc_hwrng_present);
183 
184 long kvmppc_rm_h_random(struct kvm_vcpu *vcpu)
185 {
186 	if (ppc_md.get_random_seed &&
187 	    ppc_md.get_random_seed(&vcpu->arch.regs.gpr[4]))
188 		return H_SUCCESS;
189 
190 	return H_HARDWARE;
191 }
192 
193 /*
194  * Send an interrupt or message to another CPU.
195  * The caller needs to include any barrier needed to order writes
196  * to memory vs. the IPI/message.
197  */
198 void kvmhv_rm_send_ipi(int cpu)
199 {
200 	void __iomem *xics_phys;
201 	unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
202 
203 	/* On POWER9 we can use msgsnd for any destination cpu. */
204 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
205 		msg |= get_hard_smp_processor_id(cpu);
206 		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
207 		return;
208 	}
209 
210 	/* On POWER8 for IPIs to threads in the same core, use msgsnd. */
211 	if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
212 	    cpu_first_thread_sibling(cpu) ==
213 	    cpu_first_thread_sibling(raw_smp_processor_id())) {
214 		msg |= cpu_thread_in_core(cpu);
215 		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
216 		return;
217 	}
218 
219 	/* We should never reach this */
220 	if (WARN_ON_ONCE(xics_on_xive()))
221 	    return;
222 
223 	/* Else poke the target with an IPI */
224 	xics_phys = paca_ptrs[cpu]->kvm_hstate.xics_phys;
225 	if (xics_phys)
226 		__raw_rm_writeb(IPI_PRIORITY, xics_phys + XICS_MFRR);
227 	else
228 		opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
229 }
230 
231 /*
232  * The following functions are called from the assembly code
233  * in book3s_hv_rmhandlers.S.
234  */
235 static void kvmhv_interrupt_vcore(struct kvmppc_vcore *vc, int active)
236 {
237 	int cpu = vc->pcpu;
238 
239 	/* Order setting of exit map vs. msgsnd/IPI */
240 	smp_mb();
241 	for (; active; active >>= 1, ++cpu)
242 		if (active & 1)
243 			kvmhv_rm_send_ipi(cpu);
244 }
245 
246 void kvmhv_commence_exit(int trap)
247 {
248 	struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
249 	int ptid = local_paca->kvm_hstate.ptid;
250 	struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode;
251 	int me, ee, i;
252 
253 	/* Set our bit in the threads-exiting-guest map in the 0xff00
254 	   bits of vcore->entry_exit_map */
255 	me = 0x100 << ptid;
256 	do {
257 		ee = vc->entry_exit_map;
258 	} while (cmpxchg(&vc->entry_exit_map, ee, ee | me) != ee);
259 
260 	/* Are we the first here? */
261 	if ((ee >> 8) != 0)
262 		return;
263 
264 	/*
265 	 * Trigger the other threads in this vcore to exit the guest.
266 	 * If this is a hypervisor decrementer interrupt then they
267 	 * will be already on their way out of the guest.
268 	 */
269 	if (trap != BOOK3S_INTERRUPT_HV_DECREMENTER)
270 		kvmhv_interrupt_vcore(vc, ee & ~(1 << ptid));
271 
272 	/*
273 	 * If we are doing dynamic micro-threading, interrupt the other
274 	 * subcores to pull them out of their guests too.
275 	 */
276 	if (!sip)
277 		return;
278 
279 	for (i = 0; i < MAX_SUBCORES; ++i) {
280 		vc = sip->vc[i];
281 		if (!vc)
282 			break;
283 		do {
284 			ee = vc->entry_exit_map;
285 			/* Already asked to exit? */
286 			if ((ee >> 8) != 0)
287 				break;
288 		} while (cmpxchg(&vc->entry_exit_map, ee,
289 				 ee | VCORE_EXIT_REQ) != ee);
290 		if ((ee >> 8) == 0)
291 			kvmhv_interrupt_vcore(vc, ee);
292 	}
293 }
294 
295 struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv;
296 EXPORT_SYMBOL_GPL(kvmppc_host_rm_ops_hv);
297 
298 #ifdef CONFIG_KVM_XICS
299 static struct kvmppc_irq_map *get_irqmap(struct kvmppc_passthru_irqmap *pimap,
300 					 u32 xisr)
301 {
302 	int i;
303 
304 	/*
305 	 * We access the mapped array here without a lock.  That
306 	 * is safe because we never reduce the number of entries
307 	 * in the array and we never change the v_hwirq field of
308 	 * an entry once it is set.
309 	 *
310 	 * We have also carefully ordered the stores in the writer
311 	 * and the loads here in the reader, so that if we find a matching
312 	 * hwirq here, the associated GSI and irq_desc fields are valid.
313 	 */
314 	for (i = 0; i < pimap->n_mapped; i++)  {
315 		if (xisr == pimap->mapped[i].r_hwirq) {
316 			/*
317 			 * Order subsequent reads in the caller to serialize
318 			 * with the writer.
319 			 */
320 			smp_rmb();
321 			return &pimap->mapped[i];
322 		}
323 	}
324 	return NULL;
325 }
326 
327 /*
328  * If we have an interrupt that's not an IPI, check if we have a
329  * passthrough adapter and if so, check if this external interrupt
330  * is for the adapter.
331  * We will attempt to deliver the IRQ directly to the target VCPU's
332  * ICP, the virtual ICP (based on affinity - the xive value in ICS).
333  *
334  * If the delivery fails or if this is not for a passthrough adapter,
335  * return to the host to handle this interrupt. We earlier
336  * saved a copy of the XIRR in the PACA, it will be picked up by
337  * the host ICP driver.
338  */
339 static int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
340 {
341 	struct kvmppc_passthru_irqmap *pimap;
342 	struct kvmppc_irq_map *irq_map;
343 	struct kvm_vcpu *vcpu;
344 
345 	vcpu = local_paca->kvm_hstate.kvm_vcpu;
346 	if (!vcpu)
347 		return 1;
348 	pimap = kvmppc_get_passthru_irqmap(vcpu->kvm);
349 	if (!pimap)
350 		return 1;
351 	irq_map = get_irqmap(pimap, xisr);
352 	if (!irq_map)
353 		return 1;
354 
355 	/* We're handling this interrupt, generic code doesn't need to */
356 	local_paca->kvm_hstate.saved_xirr = 0;
357 
358 	return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap, again);
359 }
360 
361 #else
362 static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
363 {
364 	return 1;
365 }
366 #endif
367 
368 /*
369  * Determine what sort of external interrupt is pending (if any).
370  * Returns:
371  *	0 if no interrupt is pending
372  *	1 if an interrupt is pending that needs to be handled by the host
373  *	2 Passthrough that needs completion in the host
374  *	-1 if there was a guest wakeup IPI (which has now been cleared)
375  *	-2 if there is PCI passthrough external interrupt that was handled
376  */
377 static long kvmppc_read_one_intr(bool *again);
378 
379 long kvmppc_read_intr(void)
380 {
381 	long ret = 0;
382 	long rc;
383 	bool again;
384 
385 	if (xive_enabled())
386 		return 1;
387 
388 	do {
389 		again = false;
390 		rc = kvmppc_read_one_intr(&again);
391 		if (rc && (ret == 0 || rc > ret))
392 			ret = rc;
393 	} while (again);
394 	return ret;
395 }
396 
397 static long kvmppc_read_one_intr(bool *again)
398 {
399 	void __iomem *xics_phys;
400 	u32 h_xirr;
401 	__be32 xirr;
402 	u32 xisr;
403 	u8 host_ipi;
404 	int64_t rc;
405 
406 	if (xive_enabled())
407 		return 1;
408 
409 	/* see if a host IPI is pending */
410 	host_ipi = READ_ONCE(local_paca->kvm_hstate.host_ipi);
411 	if (host_ipi)
412 		return 1;
413 
414 	/* Now read the interrupt from the ICP */
415 	xics_phys = local_paca->kvm_hstate.xics_phys;
416 	rc = 0;
417 	if (!xics_phys)
418 		rc = opal_int_get_xirr(&xirr, false);
419 	else
420 		xirr = __raw_rm_readl(xics_phys + XICS_XIRR);
421 	if (rc < 0)
422 		return 1;
423 
424 	/*
425 	 * Save XIRR for later. Since we get control in reverse endian
426 	 * on LE systems, save it byte reversed and fetch it back in
427 	 * host endian. Note that xirr is the value read from the
428 	 * XIRR register, while h_xirr is the host endian version.
429 	 */
430 	h_xirr = be32_to_cpu(xirr);
431 	local_paca->kvm_hstate.saved_xirr = h_xirr;
432 	xisr = h_xirr & 0xffffff;
433 	/*
434 	 * Ensure that the store/load complete to guarantee all side
435 	 * effects of loading from XIRR has completed
436 	 */
437 	smp_mb();
438 
439 	/* if nothing pending in the ICP */
440 	if (!xisr)
441 		return 0;
442 
443 	/* We found something in the ICP...
444 	 *
445 	 * If it is an IPI, clear the MFRR and EOI it.
446 	 */
447 	if (xisr == XICS_IPI) {
448 		rc = 0;
449 		if (xics_phys) {
450 			__raw_rm_writeb(0xff, xics_phys + XICS_MFRR);
451 			__raw_rm_writel(xirr, xics_phys + XICS_XIRR);
452 		} else {
453 			opal_int_set_mfrr(hard_smp_processor_id(), 0xff);
454 			rc = opal_int_eoi(h_xirr);
455 		}
456 		/* If rc > 0, there is another interrupt pending */
457 		*again = rc > 0;
458 
459 		/*
460 		 * Need to ensure side effects of above stores
461 		 * complete before proceeding.
462 		 */
463 		smp_mb();
464 
465 		/*
466 		 * We need to re-check host IPI now in case it got set in the
467 		 * meantime. If it's clear, we bounce the interrupt to the
468 		 * guest
469 		 */
470 		host_ipi = READ_ONCE(local_paca->kvm_hstate.host_ipi);
471 		if (unlikely(host_ipi != 0)) {
472 			/* We raced with the host,
473 			 * we need to resend that IPI, bummer
474 			 */
475 			if (xics_phys)
476 				__raw_rm_writeb(IPI_PRIORITY,
477 						xics_phys + XICS_MFRR);
478 			else
479 				opal_int_set_mfrr(hard_smp_processor_id(),
480 						  IPI_PRIORITY);
481 			/* Let side effects complete */
482 			smp_mb();
483 			return 1;
484 		}
485 
486 		/* OK, it's an IPI for us */
487 		local_paca->kvm_hstate.saved_xirr = 0;
488 		return -1;
489 	}
490 
491 	return kvmppc_check_passthru(xisr, xirr, again);
492 }
493 
494 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
495 {
496 	vcpu->arch.ceded = 0;
497 	if (vcpu->arch.timer_running) {
498 		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
499 		vcpu->arch.timer_running = 0;
500 	}
501 }
502 
503 void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
504 {
505 	/* Guest must always run with ME enabled, HV disabled. */
506 	msr = (msr | MSR_ME) & ~MSR_HV;
507 
508 	/*
509 	 * Check for illegal transactional state bit combination
510 	 * and if we find it, force the TS field to a safe state.
511 	 */
512 	if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
513 		msr &= ~MSR_TS_MASK;
514 	__kvmppc_set_msr_hv(vcpu, msr);
515 	kvmppc_end_cede(vcpu);
516 }
517 EXPORT_SYMBOL_GPL(kvmppc_set_msr_hv);
518 
519 static void inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags)
520 {
521 	unsigned long msr, pc, new_msr, new_pc;
522 
523 	msr = kvmppc_get_msr(vcpu);
524 	pc = kvmppc_get_pc(vcpu);
525 	new_msr = vcpu->arch.intr_msr;
526 	new_pc = vec;
527 
528 	/* If transactional, change to suspend mode on IRQ delivery */
529 	if (MSR_TM_TRANSACTIONAL(msr))
530 		new_msr |= MSR_TS_S;
531 	else
532 		new_msr |= msr & MSR_TS_MASK;
533 
534 	/*
535 	 * Perform MSR and PC adjustment for LPCR[AIL]=3 if it is set and
536 	 * applicable. AIL=2 is not supported.
537 	 *
538 	 * AIL does not apply to SRESET, MCE, or HMI (which is never
539 	 * delivered to the guest), and does not apply if IR=0 or DR=0.
540 	 */
541 	if (vec != BOOK3S_INTERRUPT_SYSTEM_RESET &&
542 	    vec != BOOK3S_INTERRUPT_MACHINE_CHECK &&
543 	    (vcpu->arch.vcore->lpcr & LPCR_AIL) == LPCR_AIL_3 &&
544 	    (msr & (MSR_IR|MSR_DR)) == (MSR_IR|MSR_DR) ) {
545 		new_msr |= MSR_IR | MSR_DR;
546 		new_pc += 0xC000000000004000ULL;
547 	}
548 
549 	kvmppc_set_srr0(vcpu, pc);
550 	kvmppc_set_srr1(vcpu, (msr & SRR1_MSR_BITS) | srr1_flags);
551 	kvmppc_set_pc(vcpu, new_pc);
552 	__kvmppc_set_msr_hv(vcpu, new_msr);
553 }
554 
555 void kvmppc_inject_interrupt_hv(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags)
556 {
557 	inject_interrupt(vcpu, vec, srr1_flags);
558 	kvmppc_end_cede(vcpu);
559 }
560 EXPORT_SYMBOL_GPL(kvmppc_inject_interrupt_hv);
561 
562 /*
563  * Is there a PRIV_DOORBELL pending for the guest (on POWER9)?
564  * Can we inject a Decrementer or a External interrupt?
565  */
566 void kvmppc_guest_entry_inject_int(struct kvm_vcpu *vcpu)
567 {
568 	int ext;
569 	unsigned long lpcr;
570 
571 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
572 
573 	/* Insert EXTERNAL bit into LPCR at the MER bit position */
574 	ext = (vcpu->arch.pending_exceptions >> BOOK3S_IRQPRIO_EXTERNAL) & 1;
575 	lpcr = mfspr(SPRN_LPCR);
576 	lpcr |= ext << LPCR_MER_SH;
577 	mtspr(SPRN_LPCR, lpcr);
578 	isync();
579 
580 	if (vcpu->arch.shregs.msr & MSR_EE) {
581 		if (ext) {
582 			inject_interrupt(vcpu, BOOK3S_INTERRUPT_EXTERNAL, 0);
583 		} else {
584 			long int dec = mfspr(SPRN_DEC);
585 			if (!(lpcr & LPCR_LD))
586 				dec = (int) dec;
587 			if (dec < 0)
588 				inject_interrupt(vcpu,
589 					BOOK3S_INTERRUPT_DECREMENTER, 0);
590 		}
591 	}
592 
593 	if (vcpu->arch.doorbell_request) {
594 		mtspr(SPRN_DPDES, 1);
595 		vcpu->arch.vcore->dpdes = 1;
596 		smp_wmb();
597 		vcpu->arch.doorbell_request = 0;
598 	}
599 }
600 
601 static void flush_guest_tlb(struct kvm *kvm)
602 {
603 	unsigned long rb, set;
604 
605 	rb = PPC_BIT(52);	/* IS = 2 */
606 	for (set = 0; set < kvm->arch.tlb_sets; ++set) {
607 		/* R=0 PRS=0 RIC=0 */
608 		asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
609 			     : : "r" (rb), "i" (0), "i" (0), "i" (0),
610 			       "r" (0) : "memory");
611 		rb += PPC_BIT(51);	/* increment set number */
612 	}
613 	asm volatile("ptesync": : :"memory");
614 }
615 
616 void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu)
617 {
618 	if (cpumask_test_cpu(pcpu, &kvm->arch.need_tlb_flush)) {
619 		flush_guest_tlb(kvm);
620 
621 		/* Clear the bit after the TLB flush */
622 		cpumask_clear_cpu(pcpu, &kvm->arch.need_tlb_flush);
623 	}
624 }
625 EXPORT_SYMBOL_GPL(kvmppc_check_need_tlb_flush);
626