xref: /openbmc/linux/arch/powerpc/kvm/powerpc.c (revision 4f79a18a)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *
4  * Copyright IBM Corp. 2007
5  *
6  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
7  *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
8  */
9 
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/vmalloc.h>
14 #include <linux/hrtimer.h>
15 #include <linux/sched/signal.h>
16 #include <linux/fs.h>
17 #include <linux/slab.h>
18 #include <linux/file.h>
19 #include <linux/module.h>
20 #include <linux/irqbypass.h>
21 #include <linux/kvm_irqfd.h>
22 #include <linux/of.h>
23 #include <asm/cputable.h>
24 #include <linux/uaccess.h>
25 #include <asm/kvm_ppc.h>
26 #include <asm/cputhreads.h>
27 #include <asm/irqflags.h>
28 #include <asm/iommu.h>
29 #include <asm/switch_to.h>
30 #include <asm/xive.h>
31 #ifdef CONFIG_PPC_PSERIES
32 #include <asm/hvcall.h>
33 #include <asm/plpar_wrappers.h>
34 #endif
35 #include <asm/ultravisor.h>
36 #include <asm/setup.h>
37 
38 #include "timing.h"
39 #include "../mm/mmu_decl.h"
40 
41 #define CREATE_TRACE_POINTS
42 #include "trace.h"
43 
44 struct kvmppc_ops *kvmppc_hv_ops;
45 EXPORT_SYMBOL_GPL(kvmppc_hv_ops);
46 struct kvmppc_ops *kvmppc_pr_ops;
47 EXPORT_SYMBOL_GPL(kvmppc_pr_ops);
48 
49 
kvm_arch_vcpu_runnable(struct kvm_vcpu * v)50 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
51 {
52 	return !!(v->arch.pending_exceptions) || kvm_request_pending(v);
53 }
54 
kvm_arch_dy_runnable(struct kvm_vcpu * vcpu)55 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
56 {
57 	return kvm_arch_vcpu_runnable(vcpu);
58 }
59 
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)60 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
61 {
62 	return false;
63 }
64 
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)65 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
66 {
67 	return 1;
68 }
69 
70 /*
71  * Common checks before entering the guest world.  Call with interrupts
72  * disabled.
73  *
74  * returns:
75  *
76  * == 1 if we're ready to go into guest state
77  * <= 0 if we need to go back to the host with return value
78  */
kvmppc_prepare_to_enter(struct kvm_vcpu * vcpu)79 int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
80 {
81 	int r;
82 
83 	WARN_ON(irqs_disabled());
84 	hard_irq_disable();
85 
86 	while (true) {
87 		if (need_resched()) {
88 			local_irq_enable();
89 			cond_resched();
90 			hard_irq_disable();
91 			continue;
92 		}
93 
94 		if (signal_pending(current)) {
95 			kvmppc_account_exit(vcpu, SIGNAL_EXITS);
96 			vcpu->run->exit_reason = KVM_EXIT_INTR;
97 			r = -EINTR;
98 			break;
99 		}
100 
101 		vcpu->mode = IN_GUEST_MODE;
102 
103 		/*
104 		 * Reading vcpu->requests must happen after setting vcpu->mode,
105 		 * so we don't miss a request because the requester sees
106 		 * OUTSIDE_GUEST_MODE and assumes we'll be checking requests
107 		 * before next entering the guest (and thus doesn't IPI).
108 		 * This also orders the write to mode from any reads
109 		 * to the page tables done while the VCPU is running.
110 		 * Please see the comment in kvm_flush_remote_tlbs.
111 		 */
112 		smp_mb();
113 
114 		if (kvm_request_pending(vcpu)) {
115 			/* Make sure we process requests preemptable */
116 			local_irq_enable();
117 			trace_kvm_check_requests(vcpu);
118 			r = kvmppc_core_check_requests(vcpu);
119 			hard_irq_disable();
120 			if (r > 0)
121 				continue;
122 			break;
123 		}
124 
125 		if (kvmppc_core_prepare_to_enter(vcpu)) {
126 			/* interrupts got enabled in between, so we
127 			   are back at square 1 */
128 			continue;
129 		}
130 
131 		guest_enter_irqoff();
132 		return 1;
133 	}
134 
135 	/* return to host */
136 	local_irq_enable();
137 	return r;
138 }
139 EXPORT_SYMBOL_GPL(kvmppc_prepare_to_enter);
140 
141 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
kvmppc_swab_shared(struct kvm_vcpu * vcpu)142 static void kvmppc_swab_shared(struct kvm_vcpu *vcpu)
143 {
144 	struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared;
145 	int i;
146 
147 	shared->sprg0 = swab64(shared->sprg0);
148 	shared->sprg1 = swab64(shared->sprg1);
149 	shared->sprg2 = swab64(shared->sprg2);
150 	shared->sprg3 = swab64(shared->sprg3);
151 	shared->srr0 = swab64(shared->srr0);
152 	shared->srr1 = swab64(shared->srr1);
153 	shared->dar = swab64(shared->dar);
154 	shared->msr = swab64(shared->msr);
155 	shared->dsisr = swab32(shared->dsisr);
156 	shared->int_pending = swab32(shared->int_pending);
157 	for (i = 0; i < ARRAY_SIZE(shared->sr); i++)
158 		shared->sr[i] = swab32(shared->sr[i]);
159 }
160 #endif
161 
kvmppc_kvm_pv(struct kvm_vcpu * vcpu)162 int kvmppc_kvm_pv(struct kvm_vcpu *vcpu)
163 {
164 	int nr = kvmppc_get_gpr(vcpu, 11);
165 	int r;
166 	unsigned long __maybe_unused param1 = kvmppc_get_gpr(vcpu, 3);
167 	unsigned long __maybe_unused param2 = kvmppc_get_gpr(vcpu, 4);
168 	unsigned long __maybe_unused param3 = kvmppc_get_gpr(vcpu, 5);
169 	unsigned long __maybe_unused param4 = kvmppc_get_gpr(vcpu, 6);
170 	unsigned long r2 = 0;
171 
172 	if (!(kvmppc_get_msr(vcpu) & MSR_SF)) {
173 		/* 32 bit mode */
174 		param1 &= 0xffffffff;
175 		param2 &= 0xffffffff;
176 		param3 &= 0xffffffff;
177 		param4 &= 0xffffffff;
178 	}
179 
180 	switch (nr) {
181 	case KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE):
182 	{
183 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
184 		/* Book3S can be little endian, find it out here */
185 		int shared_big_endian = true;
186 		if (vcpu->arch.intr_msr & MSR_LE)
187 			shared_big_endian = false;
188 		if (shared_big_endian != vcpu->arch.shared_big_endian)
189 			kvmppc_swab_shared(vcpu);
190 		vcpu->arch.shared_big_endian = shared_big_endian;
191 #endif
192 
193 		if (!(param2 & MAGIC_PAGE_FLAG_NOT_MAPPED_NX)) {
194 			/*
195 			 * Older versions of the Linux magic page code had
196 			 * a bug where they would map their trampoline code
197 			 * NX. If that's the case, remove !PR NX capability.
198 			 */
199 			vcpu->arch.disable_kernel_nx = true;
200 			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
201 		}
202 
203 		vcpu->arch.magic_page_pa = param1 & ~0xfffULL;
204 		vcpu->arch.magic_page_ea = param2 & ~0xfffULL;
205 
206 #ifdef CONFIG_PPC_64K_PAGES
207 		/*
208 		 * Make sure our 4k magic page is in the same window of a 64k
209 		 * page within the guest and within the host's page.
210 		 */
211 		if ((vcpu->arch.magic_page_pa & 0xf000) !=
212 		    ((ulong)vcpu->arch.shared & 0xf000)) {
213 			void *old_shared = vcpu->arch.shared;
214 			ulong shared = (ulong)vcpu->arch.shared;
215 			void *new_shared;
216 
217 			shared &= PAGE_MASK;
218 			shared |= vcpu->arch.magic_page_pa & 0xf000;
219 			new_shared = (void*)shared;
220 			memcpy(new_shared, old_shared, 0x1000);
221 			vcpu->arch.shared = new_shared;
222 		}
223 #endif
224 
225 		r2 = KVM_MAGIC_FEAT_SR | KVM_MAGIC_FEAT_MAS0_TO_SPRG7;
226 
227 		r = EV_SUCCESS;
228 		break;
229 	}
230 	case KVM_HCALL_TOKEN(KVM_HC_FEATURES):
231 		r = EV_SUCCESS;
232 #if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500V2)
233 		r2 |= (1 << KVM_FEATURE_MAGIC_PAGE);
234 #endif
235 
236 		/* Second return value is in r4 */
237 		break;
238 	case EV_HCALL_TOKEN(EV_IDLE):
239 		r = EV_SUCCESS;
240 		kvm_vcpu_halt(vcpu);
241 		break;
242 	default:
243 		r = EV_UNIMPLEMENTED;
244 		break;
245 	}
246 
247 	kvmppc_set_gpr(vcpu, 4, r2);
248 
249 	return r;
250 }
251 EXPORT_SYMBOL_GPL(kvmppc_kvm_pv);
252 
kvmppc_sanity_check(struct kvm_vcpu * vcpu)253 int kvmppc_sanity_check(struct kvm_vcpu *vcpu)
254 {
255 	int r = false;
256 
257 	/* We have to know what CPU to virtualize */
258 	if (!vcpu->arch.pvr)
259 		goto out;
260 
261 	/* PAPR only works with book3s_64 */
262 	if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled)
263 		goto out;
264 
265 	/* HV KVM can only do PAPR mode for now */
266 	if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm))
267 		goto out;
268 
269 #ifdef CONFIG_KVM_BOOKE_HV
270 	if (!cpu_has_feature(CPU_FTR_EMB_HV))
271 		goto out;
272 #endif
273 
274 	r = true;
275 
276 out:
277 	vcpu->arch.sane = r;
278 	return r ? 0 : -EINVAL;
279 }
280 EXPORT_SYMBOL_GPL(kvmppc_sanity_check);
281 
kvmppc_emulate_mmio(struct kvm_vcpu * vcpu)282 int kvmppc_emulate_mmio(struct kvm_vcpu *vcpu)
283 {
284 	enum emulation_result er;
285 	int r;
286 
287 	er = kvmppc_emulate_loadstore(vcpu);
288 	switch (er) {
289 	case EMULATE_DONE:
290 		/* Future optimization: only reload non-volatiles if they were
291 		 * actually modified. */
292 		r = RESUME_GUEST_NV;
293 		break;
294 	case EMULATE_AGAIN:
295 		r = RESUME_GUEST;
296 		break;
297 	case EMULATE_DO_MMIO:
298 		vcpu->run->exit_reason = KVM_EXIT_MMIO;
299 		/* We must reload nonvolatiles because "update" load/store
300 		 * instructions modify register state. */
301 		/* Future optimization: only reload non-volatiles if they were
302 		 * actually modified. */
303 		r = RESUME_HOST_NV;
304 		break;
305 	case EMULATE_FAIL:
306 	{
307 		ppc_inst_t last_inst;
308 
309 		kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
310 		kvm_debug_ratelimited("Guest access to device memory using unsupported instruction (opcode: %#08x)\n",
311 				      ppc_inst_val(last_inst));
312 
313 		/*
314 		 * Injecting a Data Storage here is a bit more
315 		 * accurate since the instruction that caused the
316 		 * access could still be a valid one.
317 		 */
318 		if (!IS_ENABLED(CONFIG_BOOKE)) {
319 			ulong dsisr = DSISR_BADACCESS;
320 
321 			if (vcpu->mmio_is_write)
322 				dsisr |= DSISR_ISSTORE;
323 
324 			kvmppc_core_queue_data_storage(vcpu,
325 					kvmppc_get_msr(vcpu) & SRR1_PREFIXED,
326 					vcpu->arch.vaddr_accessed, dsisr);
327 		} else {
328 			/*
329 			 * BookE does not send a SIGBUS on a bad
330 			 * fault, so use a Program interrupt instead
331 			 * to avoid a fault loop.
332 			 */
333 			kvmppc_core_queue_program(vcpu, 0);
334 		}
335 
336 		r = RESUME_GUEST;
337 		break;
338 	}
339 	default:
340 		WARN_ON(1);
341 		r = RESUME_GUEST;
342 	}
343 
344 	return r;
345 }
346 EXPORT_SYMBOL_GPL(kvmppc_emulate_mmio);
347 
kvmppc_st(struct kvm_vcpu * vcpu,ulong * eaddr,int size,void * ptr,bool data)348 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
349 	      bool data)
350 {
351 	ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
352 	struct kvmppc_pte pte;
353 	int r = -EINVAL;
354 
355 	vcpu->stat.st++;
356 
357 	if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->store_to_eaddr)
358 		r = vcpu->kvm->arch.kvm_ops->store_to_eaddr(vcpu, eaddr, ptr,
359 							    size);
360 
361 	if ((!r) || (r == -EAGAIN))
362 		return r;
363 
364 	r = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
365 			 XLATE_WRITE, &pte);
366 	if (r < 0)
367 		return r;
368 
369 	*eaddr = pte.raddr;
370 
371 	if (!pte.may_write)
372 		return -EPERM;
373 
374 	/* Magic page override */
375 	if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
376 	    ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
377 	    !(kvmppc_get_msr(vcpu) & MSR_PR)) {
378 		void *magic = vcpu->arch.shared;
379 		magic += pte.eaddr & 0xfff;
380 		memcpy(magic, ptr, size);
381 		return EMULATE_DONE;
382 	}
383 
384 	if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size))
385 		return EMULATE_DO_MMIO;
386 
387 	return EMULATE_DONE;
388 }
389 EXPORT_SYMBOL_GPL(kvmppc_st);
390 
kvmppc_ld(struct kvm_vcpu * vcpu,ulong * eaddr,int size,void * ptr,bool data)391 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
392 		      bool data)
393 {
394 	ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
395 	struct kvmppc_pte pte;
396 	int rc = -EINVAL;
397 
398 	vcpu->stat.ld++;
399 
400 	if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->load_from_eaddr)
401 		rc = vcpu->kvm->arch.kvm_ops->load_from_eaddr(vcpu, eaddr, ptr,
402 							      size);
403 
404 	if ((!rc) || (rc == -EAGAIN))
405 		return rc;
406 
407 	rc = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
408 			  XLATE_READ, &pte);
409 	if (rc)
410 		return rc;
411 
412 	*eaddr = pte.raddr;
413 
414 	if (!pte.may_read)
415 		return -EPERM;
416 
417 	if (!data && !pte.may_execute)
418 		return -ENOEXEC;
419 
420 	/* Magic page override */
421 	if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
422 	    ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
423 	    !(kvmppc_get_msr(vcpu) & MSR_PR)) {
424 		void *magic = vcpu->arch.shared;
425 		magic += pte.eaddr & 0xfff;
426 		memcpy(ptr, magic, size);
427 		return EMULATE_DONE;
428 	}
429 
430 	kvm_vcpu_srcu_read_lock(vcpu);
431 	rc = kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size);
432 	kvm_vcpu_srcu_read_unlock(vcpu);
433 	if (rc)
434 		return EMULATE_DO_MMIO;
435 
436 	return EMULATE_DONE;
437 }
438 EXPORT_SYMBOL_GPL(kvmppc_ld);
439 
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)440 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
441 {
442 	struct kvmppc_ops *kvm_ops = NULL;
443 	int r;
444 
445 	/*
446 	 * if we have both HV and PR enabled, default is HV
447 	 */
448 	if (type == 0) {
449 		if (kvmppc_hv_ops)
450 			kvm_ops = kvmppc_hv_ops;
451 		else
452 			kvm_ops = kvmppc_pr_ops;
453 		if (!kvm_ops)
454 			goto err_out;
455 	} else	if (type == KVM_VM_PPC_HV) {
456 		if (!kvmppc_hv_ops)
457 			goto err_out;
458 		kvm_ops = kvmppc_hv_ops;
459 	} else if (type == KVM_VM_PPC_PR) {
460 		if (!kvmppc_pr_ops)
461 			goto err_out;
462 		kvm_ops = kvmppc_pr_ops;
463 	} else
464 		goto err_out;
465 
466 	if (!try_module_get(kvm_ops->owner))
467 		return -ENOENT;
468 
469 	kvm->arch.kvm_ops = kvm_ops;
470 	r = kvmppc_core_init_vm(kvm);
471 	if (r)
472 		module_put(kvm_ops->owner);
473 	return r;
474 err_out:
475 	return -EINVAL;
476 }
477 
kvm_arch_destroy_vm(struct kvm * kvm)478 void kvm_arch_destroy_vm(struct kvm *kvm)
479 {
480 #ifdef CONFIG_KVM_XICS
481 	/*
482 	 * We call kick_all_cpus_sync() to ensure that all
483 	 * CPUs have executed any pending IPIs before we
484 	 * continue and free VCPUs structures below.
485 	 */
486 	if (is_kvmppc_hv_enabled(kvm))
487 		kick_all_cpus_sync();
488 #endif
489 
490 	kvm_destroy_vcpus(kvm);
491 
492 	mutex_lock(&kvm->lock);
493 
494 	kvmppc_core_destroy_vm(kvm);
495 
496 	mutex_unlock(&kvm->lock);
497 
498 	/* drop the module reference */
499 	module_put(kvm->arch.kvm_ops->owner);
500 }
501 
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)502 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
503 {
504 	int r;
505 	/* Assume we're using HV mode when the HV module is loaded */
506 	int hv_enabled = kvmppc_hv_ops ? 1 : 0;
507 
508 	if (kvm) {
509 		/*
510 		 * Hooray - we know which VM type we're running on. Depend on
511 		 * that rather than the guess above.
512 		 */
513 		hv_enabled = is_kvmppc_hv_enabled(kvm);
514 	}
515 
516 	switch (ext) {
517 #ifdef CONFIG_BOOKE
518 	case KVM_CAP_PPC_BOOKE_SREGS:
519 	case KVM_CAP_PPC_BOOKE_WATCHDOG:
520 	case KVM_CAP_PPC_EPR:
521 #else
522 	case KVM_CAP_PPC_SEGSTATE:
523 	case KVM_CAP_PPC_HIOR:
524 	case KVM_CAP_PPC_PAPR:
525 #endif
526 	case KVM_CAP_PPC_UNSET_IRQ:
527 	case KVM_CAP_PPC_IRQ_LEVEL:
528 	case KVM_CAP_ENABLE_CAP:
529 	case KVM_CAP_ONE_REG:
530 	case KVM_CAP_IOEVENTFD:
531 	case KVM_CAP_DEVICE_CTRL:
532 	case KVM_CAP_IMMEDIATE_EXIT:
533 	case KVM_CAP_SET_GUEST_DEBUG:
534 		r = 1;
535 		break;
536 	case KVM_CAP_PPC_GUEST_DEBUG_SSTEP:
537 	case KVM_CAP_PPC_PAIRED_SINGLES:
538 	case KVM_CAP_PPC_OSI:
539 	case KVM_CAP_PPC_GET_PVINFO:
540 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
541 	case KVM_CAP_SW_TLB:
542 #endif
543 		/* We support this only for PR */
544 		r = !hv_enabled;
545 		break;
546 #ifdef CONFIG_KVM_MPIC
547 	case KVM_CAP_IRQ_MPIC:
548 		r = 1;
549 		break;
550 #endif
551 
552 #ifdef CONFIG_PPC_BOOK3S_64
553 	case KVM_CAP_SPAPR_TCE:
554 	case KVM_CAP_SPAPR_TCE_64:
555 		r = 1;
556 		break;
557 	case KVM_CAP_SPAPR_TCE_VFIO:
558 		r = !!cpu_has_feature(CPU_FTR_HVMODE);
559 		break;
560 	case KVM_CAP_PPC_RTAS:
561 	case KVM_CAP_PPC_FIXUP_HCALL:
562 	case KVM_CAP_PPC_ENABLE_HCALL:
563 #ifdef CONFIG_KVM_XICS
564 	case KVM_CAP_IRQ_XICS:
565 #endif
566 	case KVM_CAP_PPC_GET_CPU_CHAR:
567 		r = 1;
568 		break;
569 #ifdef CONFIG_KVM_XIVE
570 	case KVM_CAP_PPC_IRQ_XIVE:
571 		/*
572 		 * We need XIVE to be enabled on the platform (implies
573 		 * a POWER9 processor) and the PowerNV platform, as
574 		 * nested is not yet supported.
575 		 */
576 		r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE) &&
577 			kvmppc_xive_native_supported();
578 		break;
579 #endif
580 
581 #ifdef CONFIG_HAVE_KVM_IRQFD
582 	case KVM_CAP_IRQFD_RESAMPLE:
583 		r = !xive_enabled();
584 		break;
585 #endif
586 
587 	case KVM_CAP_PPC_ALLOC_HTAB:
588 		r = hv_enabled;
589 		break;
590 #endif /* CONFIG_PPC_BOOK3S_64 */
591 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
592 	case KVM_CAP_PPC_SMT:
593 		r = 0;
594 		if (kvm) {
595 			if (kvm->arch.emul_smt_mode > 1)
596 				r = kvm->arch.emul_smt_mode;
597 			else
598 				r = kvm->arch.smt_mode;
599 		} else if (hv_enabled) {
600 			if (cpu_has_feature(CPU_FTR_ARCH_300))
601 				r = 1;
602 			else
603 				r = threads_per_subcore;
604 		}
605 		break;
606 	case KVM_CAP_PPC_SMT_POSSIBLE:
607 		r = 1;
608 		if (hv_enabled) {
609 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
610 				r = ((threads_per_subcore << 1) - 1);
611 			else
612 				/* P9 can emulate dbells, so allow any mode */
613 				r = 8 | 4 | 2 | 1;
614 		}
615 		break;
616 	case KVM_CAP_PPC_RMA:
617 		r = 0;
618 		break;
619 	case KVM_CAP_PPC_HWRNG:
620 		r = kvmppc_hwrng_present();
621 		break;
622 	case KVM_CAP_PPC_MMU_RADIX:
623 		r = !!(hv_enabled && radix_enabled());
624 		break;
625 	case KVM_CAP_PPC_MMU_HASH_V3:
626 		r = !!(hv_enabled && kvmppc_hv_ops->hash_v3_possible &&
627 		       kvmppc_hv_ops->hash_v3_possible());
628 		break;
629 	case KVM_CAP_PPC_NESTED_HV:
630 		r = !!(hv_enabled && kvmppc_hv_ops->enable_nested &&
631 		       !kvmppc_hv_ops->enable_nested(NULL));
632 		break;
633 #endif
634 	case KVM_CAP_SYNC_MMU:
635 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
636 		r = hv_enabled;
637 #elif defined(KVM_ARCH_WANT_MMU_NOTIFIER)
638 		r = 1;
639 #else
640 		r = 0;
641 #endif
642 		break;
643 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
644 	case KVM_CAP_PPC_HTAB_FD:
645 		r = hv_enabled;
646 		break;
647 #endif
648 	case KVM_CAP_NR_VCPUS:
649 		/*
650 		 * Recommending a number of CPUs is somewhat arbitrary; we
651 		 * return the number of present CPUs for -HV (since a host
652 		 * will have secondary threads "offline"), and for other KVM
653 		 * implementations just count online CPUs.
654 		 */
655 		if (hv_enabled)
656 			r = min_t(unsigned int, num_present_cpus(), KVM_MAX_VCPUS);
657 		else
658 			r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
659 		break;
660 	case KVM_CAP_MAX_VCPUS:
661 		r = KVM_MAX_VCPUS;
662 		break;
663 	case KVM_CAP_MAX_VCPU_ID:
664 		r = KVM_MAX_VCPU_IDS;
665 		break;
666 #ifdef CONFIG_PPC_BOOK3S_64
667 	case KVM_CAP_PPC_GET_SMMU_INFO:
668 		r = 1;
669 		break;
670 	case KVM_CAP_SPAPR_MULTITCE:
671 		r = 1;
672 		break;
673 	case KVM_CAP_SPAPR_RESIZE_HPT:
674 		r = !!hv_enabled;
675 		break;
676 #endif
677 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
678 	case KVM_CAP_PPC_FWNMI:
679 		r = hv_enabled;
680 		break;
681 #endif
682 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
683 	case KVM_CAP_PPC_HTM:
684 		r = !!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_HTM) ||
685 		     (hv_enabled && cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST));
686 		break;
687 #endif
688 #if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
689 	case KVM_CAP_PPC_SECURE_GUEST:
690 		r = hv_enabled && kvmppc_hv_ops->enable_svm &&
691 			!kvmppc_hv_ops->enable_svm(NULL);
692 		break;
693 	case KVM_CAP_PPC_DAWR1:
694 		r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 &&
695 		       !kvmppc_hv_ops->enable_dawr1(NULL));
696 		break;
697 	case KVM_CAP_PPC_RPT_INVALIDATE:
698 		r = 1;
699 		break;
700 #endif
701 	case KVM_CAP_PPC_AIL_MODE_3:
702 		r = 0;
703 		/*
704 		 * KVM PR, POWER7, and some POWER9s don't support AIL=3 mode.
705 		 * The POWER9s can support it if the guest runs in hash mode,
706 		 * but QEMU doesn't necessarily query the capability in time.
707 		 */
708 		if (hv_enabled) {
709 			if (kvmhv_on_pseries()) {
710 				if (pseries_reloc_on_exception())
711 					r = 1;
712 			} else if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
713 				  !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
714 				r = 1;
715 			}
716 		}
717 		break;
718 	default:
719 		r = 0;
720 		break;
721 	}
722 	return r;
723 
724 }
725 
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)726 long kvm_arch_dev_ioctl(struct file *filp,
727                         unsigned int ioctl, unsigned long arg)
728 {
729 	return -EINVAL;
730 }
731 
kvm_arch_free_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)732 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
733 {
734 	kvmppc_core_free_memslot(kvm, slot);
735 }
736 
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)737 int kvm_arch_prepare_memory_region(struct kvm *kvm,
738 				   const struct kvm_memory_slot *old,
739 				   struct kvm_memory_slot *new,
740 				   enum kvm_mr_change change)
741 {
742 	return kvmppc_core_prepare_memory_region(kvm, old, new, change);
743 }
744 
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)745 void kvm_arch_commit_memory_region(struct kvm *kvm,
746 				   struct kvm_memory_slot *old,
747 				   const struct kvm_memory_slot *new,
748 				   enum kvm_mr_change change)
749 {
750 	kvmppc_core_commit_memory_region(kvm, old, new, change);
751 }
752 
kvm_arch_flush_shadow_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)753 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
754 				   struct kvm_memory_slot *slot)
755 {
756 	kvmppc_core_flush_memslot(kvm, slot);
757 }
758 
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)759 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
760 {
761 	return 0;
762 }
763 
kvmppc_decrementer_wakeup(struct hrtimer * timer)764 static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer)
765 {
766 	struct kvm_vcpu *vcpu;
767 
768 	vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer);
769 	kvmppc_decrementer_func(vcpu);
770 
771 	return HRTIMER_NORESTART;
772 }
773 
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)774 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
775 {
776 	int err;
777 
778 	hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
779 	vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
780 
781 #ifdef CONFIG_KVM_EXIT_TIMING
782 	mutex_init(&vcpu->arch.exit_timing_lock);
783 #endif
784 	err = kvmppc_subarch_vcpu_init(vcpu);
785 	if (err)
786 		return err;
787 
788 	err = kvmppc_core_vcpu_create(vcpu);
789 	if (err)
790 		goto out_vcpu_uninit;
791 
792 	rcuwait_init(&vcpu->arch.wait);
793 	vcpu->arch.waitp = &vcpu->arch.wait;
794 	return 0;
795 
796 out_vcpu_uninit:
797 	kvmppc_subarch_vcpu_uninit(vcpu);
798 	return err;
799 }
800 
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)801 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
802 {
803 }
804 
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)805 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
806 {
807 	/* Make sure we're not using the vcpu anymore */
808 	hrtimer_cancel(&vcpu->arch.dec_timer);
809 
810 	switch (vcpu->arch.irq_type) {
811 	case KVMPPC_IRQ_MPIC:
812 		kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu);
813 		break;
814 	case KVMPPC_IRQ_XICS:
815 		if (xics_on_xive())
816 			kvmppc_xive_cleanup_vcpu(vcpu);
817 		else
818 			kvmppc_xics_free_icp(vcpu);
819 		break;
820 	case KVMPPC_IRQ_XIVE:
821 		kvmppc_xive_native_cleanup_vcpu(vcpu);
822 		break;
823 	}
824 
825 	kvmppc_core_vcpu_free(vcpu);
826 
827 	kvmppc_subarch_vcpu_uninit(vcpu);
828 }
829 
kvm_cpu_has_pending_timer(struct kvm_vcpu * vcpu)830 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
831 {
832 	return kvmppc_core_pending_dec(vcpu);
833 }
834 
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)835 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
836 {
837 #ifdef CONFIG_BOOKE
838 	/*
839 	 * vrsave (formerly usprg0) isn't used by Linux, but may
840 	 * be used by the guest.
841 	 *
842 	 * On non-booke this is associated with Altivec and
843 	 * is handled by code in book3s.c.
844 	 */
845 	mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
846 #endif
847 	kvmppc_core_vcpu_load(vcpu, cpu);
848 }
849 
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)850 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
851 {
852 	kvmppc_core_vcpu_put(vcpu);
853 #ifdef CONFIG_BOOKE
854 	vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
855 #endif
856 }
857 
858 /*
859  * irq_bypass_add_producer and irq_bypass_del_producer are only
860  * useful if the architecture supports PCI passthrough.
861  * irq_bypass_stop and irq_bypass_start are not needed and so
862  * kvm_ops are not defined for them.
863  */
kvm_arch_has_irq_bypass(void)864 bool kvm_arch_has_irq_bypass(void)
865 {
866 	return ((kvmppc_hv_ops && kvmppc_hv_ops->irq_bypass_add_producer) ||
867 		(kvmppc_pr_ops && kvmppc_pr_ops->irq_bypass_add_producer));
868 }
869 
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)870 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
871 				     struct irq_bypass_producer *prod)
872 {
873 	struct kvm_kernel_irqfd *irqfd =
874 		container_of(cons, struct kvm_kernel_irqfd, consumer);
875 	struct kvm *kvm = irqfd->kvm;
876 
877 	if (kvm->arch.kvm_ops->irq_bypass_add_producer)
878 		return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod);
879 
880 	return 0;
881 }
882 
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)883 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
884 				      struct irq_bypass_producer *prod)
885 {
886 	struct kvm_kernel_irqfd *irqfd =
887 		container_of(cons, struct kvm_kernel_irqfd, consumer);
888 	struct kvm *kvm = irqfd->kvm;
889 
890 	if (kvm->arch.kvm_ops->irq_bypass_del_producer)
891 		kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod);
892 }
893 
894 #ifdef CONFIG_VSX
kvmppc_get_vsr_dword_offset(int index)895 static inline int kvmppc_get_vsr_dword_offset(int index)
896 {
897 	int offset;
898 
899 	if ((index != 0) && (index != 1))
900 		return -1;
901 
902 #ifdef __BIG_ENDIAN
903 	offset =  index;
904 #else
905 	offset = 1 - index;
906 #endif
907 
908 	return offset;
909 }
910 
kvmppc_get_vsr_word_offset(int index)911 static inline int kvmppc_get_vsr_word_offset(int index)
912 {
913 	int offset;
914 
915 	if ((index > 3) || (index < 0))
916 		return -1;
917 
918 #ifdef __BIG_ENDIAN
919 	offset = index;
920 #else
921 	offset = 3 - index;
922 #endif
923 	return offset;
924 }
925 
kvmppc_set_vsr_dword(struct kvm_vcpu * vcpu,u64 gpr)926 static inline void kvmppc_set_vsr_dword(struct kvm_vcpu *vcpu,
927 	u64 gpr)
928 {
929 	union kvmppc_one_reg val;
930 	int offset = kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset);
931 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
932 
933 	if (offset == -1)
934 		return;
935 
936 	if (index >= 32) {
937 		val.vval = VCPU_VSX_VR(vcpu, index - 32);
938 		val.vsxval[offset] = gpr;
939 		VCPU_VSX_VR(vcpu, index - 32) = val.vval;
940 	} else {
941 		VCPU_VSX_FPR(vcpu, index, offset) = gpr;
942 	}
943 }
944 
kvmppc_set_vsr_dword_dump(struct kvm_vcpu * vcpu,u64 gpr)945 static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu,
946 	u64 gpr)
947 {
948 	union kvmppc_one_reg val;
949 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
950 
951 	if (index >= 32) {
952 		val.vval = VCPU_VSX_VR(vcpu, index - 32);
953 		val.vsxval[0] = gpr;
954 		val.vsxval[1] = gpr;
955 		VCPU_VSX_VR(vcpu, index - 32) = val.vval;
956 	} else {
957 		VCPU_VSX_FPR(vcpu, index, 0) = gpr;
958 		VCPU_VSX_FPR(vcpu, index, 1) = gpr;
959 	}
960 }
961 
kvmppc_set_vsr_word_dump(struct kvm_vcpu * vcpu,u32 gpr)962 static inline void kvmppc_set_vsr_word_dump(struct kvm_vcpu *vcpu,
963 	u32 gpr)
964 {
965 	union kvmppc_one_reg val;
966 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
967 
968 	if (index >= 32) {
969 		val.vsx32val[0] = gpr;
970 		val.vsx32val[1] = gpr;
971 		val.vsx32val[2] = gpr;
972 		val.vsx32val[3] = gpr;
973 		VCPU_VSX_VR(vcpu, index - 32) = val.vval;
974 	} else {
975 		val.vsx32val[0] = gpr;
976 		val.vsx32val[1] = gpr;
977 		VCPU_VSX_FPR(vcpu, index, 0) = val.vsxval[0];
978 		VCPU_VSX_FPR(vcpu, index, 1) = val.vsxval[0];
979 	}
980 }
981 
kvmppc_set_vsr_word(struct kvm_vcpu * vcpu,u32 gpr32)982 static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu,
983 	u32 gpr32)
984 {
985 	union kvmppc_one_reg val;
986 	int offset = kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset);
987 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
988 	int dword_offset, word_offset;
989 
990 	if (offset == -1)
991 		return;
992 
993 	if (index >= 32) {
994 		val.vval = VCPU_VSX_VR(vcpu, index - 32);
995 		val.vsx32val[offset] = gpr32;
996 		VCPU_VSX_VR(vcpu, index - 32) = val.vval;
997 	} else {
998 		dword_offset = offset / 2;
999 		word_offset = offset % 2;
1000 		val.vsxval[0] = VCPU_VSX_FPR(vcpu, index, dword_offset);
1001 		val.vsx32val[word_offset] = gpr32;
1002 		VCPU_VSX_FPR(vcpu, index, dword_offset) = val.vsxval[0];
1003 	}
1004 }
1005 #endif /* CONFIG_VSX */
1006 
1007 #ifdef CONFIG_ALTIVEC
kvmppc_get_vmx_offset_generic(struct kvm_vcpu * vcpu,int index,int element_size)1008 static inline int kvmppc_get_vmx_offset_generic(struct kvm_vcpu *vcpu,
1009 		int index, int element_size)
1010 {
1011 	int offset;
1012 	int elts = sizeof(vector128)/element_size;
1013 
1014 	if ((index < 0) || (index >= elts))
1015 		return -1;
1016 
1017 	if (kvmppc_need_byteswap(vcpu))
1018 		offset = elts - index - 1;
1019 	else
1020 		offset = index;
1021 
1022 	return offset;
1023 }
1024 
kvmppc_get_vmx_dword_offset(struct kvm_vcpu * vcpu,int index)1025 static inline int kvmppc_get_vmx_dword_offset(struct kvm_vcpu *vcpu,
1026 		int index)
1027 {
1028 	return kvmppc_get_vmx_offset_generic(vcpu, index, 8);
1029 }
1030 
kvmppc_get_vmx_word_offset(struct kvm_vcpu * vcpu,int index)1031 static inline int kvmppc_get_vmx_word_offset(struct kvm_vcpu *vcpu,
1032 		int index)
1033 {
1034 	return kvmppc_get_vmx_offset_generic(vcpu, index, 4);
1035 }
1036 
kvmppc_get_vmx_hword_offset(struct kvm_vcpu * vcpu,int index)1037 static inline int kvmppc_get_vmx_hword_offset(struct kvm_vcpu *vcpu,
1038 		int index)
1039 {
1040 	return kvmppc_get_vmx_offset_generic(vcpu, index, 2);
1041 }
1042 
kvmppc_get_vmx_byte_offset(struct kvm_vcpu * vcpu,int index)1043 static inline int kvmppc_get_vmx_byte_offset(struct kvm_vcpu *vcpu,
1044 		int index)
1045 {
1046 	return kvmppc_get_vmx_offset_generic(vcpu, index, 1);
1047 }
1048 
1049 
kvmppc_set_vmx_dword(struct kvm_vcpu * vcpu,u64 gpr)1050 static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu,
1051 	u64 gpr)
1052 {
1053 	union kvmppc_one_reg val;
1054 	int offset = kvmppc_get_vmx_dword_offset(vcpu,
1055 			vcpu->arch.mmio_vmx_offset);
1056 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1057 
1058 	if (offset == -1)
1059 		return;
1060 
1061 	val.vval = VCPU_VSX_VR(vcpu, index);
1062 	val.vsxval[offset] = gpr;
1063 	VCPU_VSX_VR(vcpu, index) = val.vval;
1064 }
1065 
kvmppc_set_vmx_word(struct kvm_vcpu * vcpu,u32 gpr32)1066 static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu,
1067 	u32 gpr32)
1068 {
1069 	union kvmppc_one_reg val;
1070 	int offset = kvmppc_get_vmx_word_offset(vcpu,
1071 			vcpu->arch.mmio_vmx_offset);
1072 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1073 
1074 	if (offset == -1)
1075 		return;
1076 
1077 	val.vval = VCPU_VSX_VR(vcpu, index);
1078 	val.vsx32val[offset] = gpr32;
1079 	VCPU_VSX_VR(vcpu, index) = val.vval;
1080 }
1081 
kvmppc_set_vmx_hword(struct kvm_vcpu * vcpu,u16 gpr16)1082 static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu,
1083 	u16 gpr16)
1084 {
1085 	union kvmppc_one_reg val;
1086 	int offset = kvmppc_get_vmx_hword_offset(vcpu,
1087 			vcpu->arch.mmio_vmx_offset);
1088 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1089 
1090 	if (offset == -1)
1091 		return;
1092 
1093 	val.vval = VCPU_VSX_VR(vcpu, index);
1094 	val.vsx16val[offset] = gpr16;
1095 	VCPU_VSX_VR(vcpu, index) = val.vval;
1096 }
1097 
kvmppc_set_vmx_byte(struct kvm_vcpu * vcpu,u8 gpr8)1098 static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu,
1099 	u8 gpr8)
1100 {
1101 	union kvmppc_one_reg val;
1102 	int offset = kvmppc_get_vmx_byte_offset(vcpu,
1103 			vcpu->arch.mmio_vmx_offset);
1104 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1105 
1106 	if (offset == -1)
1107 		return;
1108 
1109 	val.vval = VCPU_VSX_VR(vcpu, index);
1110 	val.vsx8val[offset] = gpr8;
1111 	VCPU_VSX_VR(vcpu, index) = val.vval;
1112 }
1113 #endif /* CONFIG_ALTIVEC */
1114 
1115 #ifdef CONFIG_PPC_FPU
sp_to_dp(u32 fprs)1116 static inline u64 sp_to_dp(u32 fprs)
1117 {
1118 	u64 fprd;
1119 
1120 	preempt_disable();
1121 	enable_kernel_fp();
1122 	asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m<>" (fprd) : "m<>" (fprs)
1123 	     : "fr0");
1124 	preempt_enable();
1125 	return fprd;
1126 }
1127 
dp_to_sp(u64 fprd)1128 static inline u32 dp_to_sp(u64 fprd)
1129 {
1130 	u32 fprs;
1131 
1132 	preempt_disable();
1133 	enable_kernel_fp();
1134 	asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m<>" (fprs) : "m<>" (fprd)
1135 	     : "fr0");
1136 	preempt_enable();
1137 	return fprs;
1138 }
1139 
1140 #else
1141 #define sp_to_dp(x)	(x)
1142 #define dp_to_sp(x)	(x)
1143 #endif /* CONFIG_PPC_FPU */
1144 
kvmppc_complete_mmio_load(struct kvm_vcpu * vcpu)1145 static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu)
1146 {
1147 	struct kvm_run *run = vcpu->run;
1148 	u64 gpr;
1149 
1150 	if (run->mmio.len > sizeof(gpr))
1151 		return;
1152 
1153 	if (!vcpu->arch.mmio_host_swabbed) {
1154 		switch (run->mmio.len) {
1155 		case 8: gpr = *(u64 *)run->mmio.data; break;
1156 		case 4: gpr = *(u32 *)run->mmio.data; break;
1157 		case 2: gpr = *(u16 *)run->mmio.data; break;
1158 		case 1: gpr = *(u8 *)run->mmio.data; break;
1159 		}
1160 	} else {
1161 		switch (run->mmio.len) {
1162 		case 8: gpr = swab64(*(u64 *)run->mmio.data); break;
1163 		case 4: gpr = swab32(*(u32 *)run->mmio.data); break;
1164 		case 2: gpr = swab16(*(u16 *)run->mmio.data); break;
1165 		case 1: gpr = *(u8 *)run->mmio.data; break;
1166 		}
1167 	}
1168 
1169 	/* conversion between single and double precision */
1170 	if ((vcpu->arch.mmio_sp64_extend) && (run->mmio.len == 4))
1171 		gpr = sp_to_dp(gpr);
1172 
1173 	if (vcpu->arch.mmio_sign_extend) {
1174 		switch (run->mmio.len) {
1175 #ifdef CONFIG_PPC64
1176 		case 4:
1177 			gpr = (s64)(s32)gpr;
1178 			break;
1179 #endif
1180 		case 2:
1181 			gpr = (s64)(s16)gpr;
1182 			break;
1183 		case 1:
1184 			gpr = (s64)(s8)gpr;
1185 			break;
1186 		}
1187 	}
1188 
1189 	switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) {
1190 	case KVM_MMIO_REG_GPR:
1191 		kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
1192 		break;
1193 	case KVM_MMIO_REG_FPR:
1194 		if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1195 			vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP);
1196 
1197 		VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
1198 		break;
1199 #ifdef CONFIG_PPC_BOOK3S
1200 	case KVM_MMIO_REG_QPR:
1201 		vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
1202 		break;
1203 	case KVM_MMIO_REG_FQPR:
1204 		VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
1205 		vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
1206 		break;
1207 #endif
1208 #ifdef CONFIG_VSX
1209 	case KVM_MMIO_REG_VSX:
1210 		if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1211 			vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX);
1212 
1213 		if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_DWORD)
1214 			kvmppc_set_vsr_dword(vcpu, gpr);
1215 		else if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_WORD)
1216 			kvmppc_set_vsr_word(vcpu, gpr);
1217 		else if (vcpu->arch.mmio_copy_type ==
1218 				KVMPPC_VSX_COPY_DWORD_LOAD_DUMP)
1219 			kvmppc_set_vsr_dword_dump(vcpu, gpr);
1220 		else if (vcpu->arch.mmio_copy_type ==
1221 				KVMPPC_VSX_COPY_WORD_LOAD_DUMP)
1222 			kvmppc_set_vsr_word_dump(vcpu, gpr);
1223 		break;
1224 #endif
1225 #ifdef CONFIG_ALTIVEC
1226 	case KVM_MMIO_REG_VMX:
1227 		if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1228 			vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC);
1229 
1230 		if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_DWORD)
1231 			kvmppc_set_vmx_dword(vcpu, gpr);
1232 		else if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_WORD)
1233 			kvmppc_set_vmx_word(vcpu, gpr);
1234 		else if (vcpu->arch.mmio_copy_type ==
1235 				KVMPPC_VMX_COPY_HWORD)
1236 			kvmppc_set_vmx_hword(vcpu, gpr);
1237 		else if (vcpu->arch.mmio_copy_type ==
1238 				KVMPPC_VMX_COPY_BYTE)
1239 			kvmppc_set_vmx_byte(vcpu, gpr);
1240 		break;
1241 #endif
1242 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1243 	case KVM_MMIO_REG_NESTED_GPR:
1244 		if (kvmppc_need_byteswap(vcpu))
1245 			gpr = swab64(gpr);
1246 		kvm_vcpu_write_guest(vcpu, vcpu->arch.nested_io_gpr, &gpr,
1247 				     sizeof(gpr));
1248 		break;
1249 #endif
1250 	default:
1251 		BUG();
1252 	}
1253 }
1254 
__kvmppc_handle_load(struct kvm_vcpu * vcpu,unsigned int rt,unsigned int bytes,int is_default_endian,int sign_extend)1255 static int __kvmppc_handle_load(struct kvm_vcpu *vcpu,
1256 				unsigned int rt, unsigned int bytes,
1257 				int is_default_endian, int sign_extend)
1258 {
1259 	struct kvm_run *run = vcpu->run;
1260 	int idx, ret;
1261 	bool host_swabbed;
1262 
1263 	/* Pity C doesn't have a logical XOR operator */
1264 	if (kvmppc_need_byteswap(vcpu)) {
1265 		host_swabbed = is_default_endian;
1266 	} else {
1267 		host_swabbed = !is_default_endian;
1268 	}
1269 
1270 	if (bytes > sizeof(run->mmio.data))
1271 		return EMULATE_FAIL;
1272 
1273 	run->mmio.phys_addr = vcpu->arch.paddr_accessed;
1274 	run->mmio.len = bytes;
1275 	run->mmio.is_write = 0;
1276 
1277 	vcpu->arch.io_gpr = rt;
1278 	vcpu->arch.mmio_host_swabbed = host_swabbed;
1279 	vcpu->mmio_needed = 1;
1280 	vcpu->mmio_is_write = 0;
1281 	vcpu->arch.mmio_sign_extend = sign_extend;
1282 
1283 	idx = srcu_read_lock(&vcpu->kvm->srcu);
1284 
1285 	ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
1286 			      bytes, &run->mmio.data);
1287 
1288 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
1289 
1290 	if (!ret) {
1291 		kvmppc_complete_mmio_load(vcpu);
1292 		vcpu->mmio_needed = 0;
1293 		return EMULATE_DONE;
1294 	}
1295 
1296 	return EMULATE_DO_MMIO;
1297 }
1298 
kvmppc_handle_load(struct kvm_vcpu * vcpu,unsigned int rt,unsigned int bytes,int is_default_endian)1299 int kvmppc_handle_load(struct kvm_vcpu *vcpu,
1300 		       unsigned int rt, unsigned int bytes,
1301 		       int is_default_endian)
1302 {
1303 	return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 0);
1304 }
1305 EXPORT_SYMBOL_GPL(kvmppc_handle_load);
1306 
1307 /* Same as above, but sign extends */
kvmppc_handle_loads(struct kvm_vcpu * vcpu,unsigned int rt,unsigned int bytes,int is_default_endian)1308 int kvmppc_handle_loads(struct kvm_vcpu *vcpu,
1309 			unsigned int rt, unsigned int bytes,
1310 			int is_default_endian)
1311 {
1312 	return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 1);
1313 }
1314 
1315 #ifdef CONFIG_VSX
kvmppc_handle_vsx_load(struct kvm_vcpu * vcpu,unsigned int rt,unsigned int bytes,int is_default_endian,int mmio_sign_extend)1316 int kvmppc_handle_vsx_load(struct kvm_vcpu *vcpu,
1317 			unsigned int rt, unsigned int bytes,
1318 			int is_default_endian, int mmio_sign_extend)
1319 {
1320 	enum emulation_result emulated = EMULATE_DONE;
1321 
1322 	/* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
1323 	if (vcpu->arch.mmio_vsx_copy_nums > 4)
1324 		return EMULATE_FAIL;
1325 
1326 	while (vcpu->arch.mmio_vsx_copy_nums) {
1327 		emulated = __kvmppc_handle_load(vcpu, rt, bytes,
1328 			is_default_endian, mmio_sign_extend);
1329 
1330 		if (emulated != EMULATE_DONE)
1331 			break;
1332 
1333 		vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1334 
1335 		vcpu->arch.mmio_vsx_copy_nums--;
1336 		vcpu->arch.mmio_vsx_offset++;
1337 	}
1338 	return emulated;
1339 }
1340 #endif /* CONFIG_VSX */
1341 
kvmppc_handle_store(struct kvm_vcpu * vcpu,u64 val,unsigned int bytes,int is_default_endian)1342 int kvmppc_handle_store(struct kvm_vcpu *vcpu,
1343 			u64 val, unsigned int bytes, int is_default_endian)
1344 {
1345 	struct kvm_run *run = vcpu->run;
1346 	void *data = run->mmio.data;
1347 	int idx, ret;
1348 	bool host_swabbed;
1349 
1350 	/* Pity C doesn't have a logical XOR operator */
1351 	if (kvmppc_need_byteswap(vcpu)) {
1352 		host_swabbed = is_default_endian;
1353 	} else {
1354 		host_swabbed = !is_default_endian;
1355 	}
1356 
1357 	if (bytes > sizeof(run->mmio.data))
1358 		return EMULATE_FAIL;
1359 
1360 	run->mmio.phys_addr = vcpu->arch.paddr_accessed;
1361 	run->mmio.len = bytes;
1362 	run->mmio.is_write = 1;
1363 	vcpu->mmio_needed = 1;
1364 	vcpu->mmio_is_write = 1;
1365 
1366 	if ((vcpu->arch.mmio_sp64_extend) && (bytes == 4))
1367 		val = dp_to_sp(val);
1368 
1369 	/* Store the value at the lowest bytes in 'data'. */
1370 	if (!host_swabbed) {
1371 		switch (bytes) {
1372 		case 8: *(u64 *)data = val; break;
1373 		case 4: *(u32 *)data = val; break;
1374 		case 2: *(u16 *)data = val; break;
1375 		case 1: *(u8  *)data = val; break;
1376 		}
1377 	} else {
1378 		switch (bytes) {
1379 		case 8: *(u64 *)data = swab64(val); break;
1380 		case 4: *(u32 *)data = swab32(val); break;
1381 		case 2: *(u16 *)data = swab16(val); break;
1382 		case 1: *(u8  *)data = val; break;
1383 		}
1384 	}
1385 
1386 	idx = srcu_read_lock(&vcpu->kvm->srcu);
1387 
1388 	ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
1389 			       bytes, &run->mmio.data);
1390 
1391 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
1392 
1393 	if (!ret) {
1394 		vcpu->mmio_needed = 0;
1395 		return EMULATE_DONE;
1396 	}
1397 
1398 	return EMULATE_DO_MMIO;
1399 }
1400 EXPORT_SYMBOL_GPL(kvmppc_handle_store);
1401 
1402 #ifdef CONFIG_VSX
kvmppc_get_vsr_data(struct kvm_vcpu * vcpu,int rs,u64 * val)1403 static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val)
1404 {
1405 	u32 dword_offset, word_offset;
1406 	union kvmppc_one_reg reg;
1407 	int vsx_offset = 0;
1408 	int copy_type = vcpu->arch.mmio_copy_type;
1409 	int result = 0;
1410 
1411 	switch (copy_type) {
1412 	case KVMPPC_VSX_COPY_DWORD:
1413 		vsx_offset =
1414 			kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset);
1415 
1416 		if (vsx_offset == -1) {
1417 			result = -1;
1418 			break;
1419 		}
1420 
1421 		if (rs < 32) {
1422 			*val = VCPU_VSX_FPR(vcpu, rs, vsx_offset);
1423 		} else {
1424 			reg.vval = VCPU_VSX_VR(vcpu, rs - 32);
1425 			*val = reg.vsxval[vsx_offset];
1426 		}
1427 		break;
1428 
1429 	case KVMPPC_VSX_COPY_WORD:
1430 		vsx_offset =
1431 			kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset);
1432 
1433 		if (vsx_offset == -1) {
1434 			result = -1;
1435 			break;
1436 		}
1437 
1438 		if (rs < 32) {
1439 			dword_offset = vsx_offset / 2;
1440 			word_offset = vsx_offset % 2;
1441 			reg.vsxval[0] = VCPU_VSX_FPR(vcpu, rs, dword_offset);
1442 			*val = reg.vsx32val[word_offset];
1443 		} else {
1444 			reg.vval = VCPU_VSX_VR(vcpu, rs - 32);
1445 			*val = reg.vsx32val[vsx_offset];
1446 		}
1447 		break;
1448 
1449 	default:
1450 		result = -1;
1451 		break;
1452 	}
1453 
1454 	return result;
1455 }
1456 
kvmppc_handle_vsx_store(struct kvm_vcpu * vcpu,int rs,unsigned int bytes,int is_default_endian)1457 int kvmppc_handle_vsx_store(struct kvm_vcpu *vcpu,
1458 			int rs, unsigned int bytes, int is_default_endian)
1459 {
1460 	u64 val;
1461 	enum emulation_result emulated = EMULATE_DONE;
1462 
1463 	vcpu->arch.io_gpr = rs;
1464 
1465 	/* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
1466 	if (vcpu->arch.mmio_vsx_copy_nums > 4)
1467 		return EMULATE_FAIL;
1468 
1469 	while (vcpu->arch.mmio_vsx_copy_nums) {
1470 		if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1)
1471 			return EMULATE_FAIL;
1472 
1473 		emulated = kvmppc_handle_store(vcpu,
1474 			 val, bytes, is_default_endian);
1475 
1476 		if (emulated != EMULATE_DONE)
1477 			break;
1478 
1479 		vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1480 
1481 		vcpu->arch.mmio_vsx_copy_nums--;
1482 		vcpu->arch.mmio_vsx_offset++;
1483 	}
1484 
1485 	return emulated;
1486 }
1487 
kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu * vcpu)1488 static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu)
1489 {
1490 	struct kvm_run *run = vcpu->run;
1491 	enum emulation_result emulated = EMULATE_FAIL;
1492 	int r;
1493 
1494 	vcpu->arch.paddr_accessed += run->mmio.len;
1495 
1496 	if (!vcpu->mmio_is_write) {
1497 		emulated = kvmppc_handle_vsx_load(vcpu, vcpu->arch.io_gpr,
1498 			 run->mmio.len, 1, vcpu->arch.mmio_sign_extend);
1499 	} else {
1500 		emulated = kvmppc_handle_vsx_store(vcpu,
1501 			 vcpu->arch.io_gpr, run->mmio.len, 1);
1502 	}
1503 
1504 	switch (emulated) {
1505 	case EMULATE_DO_MMIO:
1506 		run->exit_reason = KVM_EXIT_MMIO;
1507 		r = RESUME_HOST;
1508 		break;
1509 	case EMULATE_FAIL:
1510 		pr_info("KVM: MMIO emulation failed (VSX repeat)\n");
1511 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1512 		run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1513 		r = RESUME_HOST;
1514 		break;
1515 	default:
1516 		r = RESUME_GUEST;
1517 		break;
1518 	}
1519 	return r;
1520 }
1521 #endif /* CONFIG_VSX */
1522 
1523 #ifdef CONFIG_ALTIVEC
kvmppc_handle_vmx_load(struct kvm_vcpu * vcpu,unsigned int rt,unsigned int bytes,int is_default_endian)1524 int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu,
1525 		unsigned int rt, unsigned int bytes, int is_default_endian)
1526 {
1527 	enum emulation_result emulated = EMULATE_DONE;
1528 
1529 	if (vcpu->arch.mmio_vmx_copy_nums > 2)
1530 		return EMULATE_FAIL;
1531 
1532 	while (vcpu->arch.mmio_vmx_copy_nums) {
1533 		emulated = __kvmppc_handle_load(vcpu, rt, bytes,
1534 				is_default_endian, 0);
1535 
1536 		if (emulated != EMULATE_DONE)
1537 			break;
1538 
1539 		vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1540 		vcpu->arch.mmio_vmx_copy_nums--;
1541 		vcpu->arch.mmio_vmx_offset++;
1542 	}
1543 
1544 	return emulated;
1545 }
1546 
kvmppc_get_vmx_dword(struct kvm_vcpu * vcpu,int index,u64 * val)1547 static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val)
1548 {
1549 	union kvmppc_one_reg reg;
1550 	int vmx_offset = 0;
1551 	int result = 0;
1552 
1553 	vmx_offset =
1554 		kvmppc_get_vmx_dword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1555 
1556 	if (vmx_offset == -1)
1557 		return -1;
1558 
1559 	reg.vval = VCPU_VSX_VR(vcpu, index);
1560 	*val = reg.vsxval[vmx_offset];
1561 
1562 	return result;
1563 }
1564 
kvmppc_get_vmx_word(struct kvm_vcpu * vcpu,int index,u64 * val)1565 static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val)
1566 {
1567 	union kvmppc_one_reg reg;
1568 	int vmx_offset = 0;
1569 	int result = 0;
1570 
1571 	vmx_offset =
1572 		kvmppc_get_vmx_word_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1573 
1574 	if (vmx_offset == -1)
1575 		return -1;
1576 
1577 	reg.vval = VCPU_VSX_VR(vcpu, index);
1578 	*val = reg.vsx32val[vmx_offset];
1579 
1580 	return result;
1581 }
1582 
kvmppc_get_vmx_hword(struct kvm_vcpu * vcpu,int index,u64 * val)1583 static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val)
1584 {
1585 	union kvmppc_one_reg reg;
1586 	int vmx_offset = 0;
1587 	int result = 0;
1588 
1589 	vmx_offset =
1590 		kvmppc_get_vmx_hword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1591 
1592 	if (vmx_offset == -1)
1593 		return -1;
1594 
1595 	reg.vval = VCPU_VSX_VR(vcpu, index);
1596 	*val = reg.vsx16val[vmx_offset];
1597 
1598 	return result;
1599 }
1600 
kvmppc_get_vmx_byte(struct kvm_vcpu * vcpu,int index,u64 * val)1601 static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val)
1602 {
1603 	union kvmppc_one_reg reg;
1604 	int vmx_offset = 0;
1605 	int result = 0;
1606 
1607 	vmx_offset =
1608 		kvmppc_get_vmx_byte_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1609 
1610 	if (vmx_offset == -1)
1611 		return -1;
1612 
1613 	reg.vval = VCPU_VSX_VR(vcpu, index);
1614 	*val = reg.vsx8val[vmx_offset];
1615 
1616 	return result;
1617 }
1618 
kvmppc_handle_vmx_store(struct kvm_vcpu * vcpu,unsigned int rs,unsigned int bytes,int is_default_endian)1619 int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu,
1620 		unsigned int rs, unsigned int bytes, int is_default_endian)
1621 {
1622 	u64 val = 0;
1623 	unsigned int index = rs & KVM_MMIO_REG_MASK;
1624 	enum emulation_result emulated = EMULATE_DONE;
1625 
1626 	if (vcpu->arch.mmio_vmx_copy_nums > 2)
1627 		return EMULATE_FAIL;
1628 
1629 	vcpu->arch.io_gpr = rs;
1630 
1631 	while (vcpu->arch.mmio_vmx_copy_nums) {
1632 		switch (vcpu->arch.mmio_copy_type) {
1633 		case KVMPPC_VMX_COPY_DWORD:
1634 			if (kvmppc_get_vmx_dword(vcpu, index, &val) == -1)
1635 				return EMULATE_FAIL;
1636 
1637 			break;
1638 		case KVMPPC_VMX_COPY_WORD:
1639 			if (kvmppc_get_vmx_word(vcpu, index, &val) == -1)
1640 				return EMULATE_FAIL;
1641 			break;
1642 		case KVMPPC_VMX_COPY_HWORD:
1643 			if (kvmppc_get_vmx_hword(vcpu, index, &val) == -1)
1644 				return EMULATE_FAIL;
1645 			break;
1646 		case KVMPPC_VMX_COPY_BYTE:
1647 			if (kvmppc_get_vmx_byte(vcpu, index, &val) == -1)
1648 				return EMULATE_FAIL;
1649 			break;
1650 		default:
1651 			return EMULATE_FAIL;
1652 		}
1653 
1654 		emulated = kvmppc_handle_store(vcpu, val, bytes,
1655 				is_default_endian);
1656 		if (emulated != EMULATE_DONE)
1657 			break;
1658 
1659 		vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1660 		vcpu->arch.mmio_vmx_copy_nums--;
1661 		vcpu->arch.mmio_vmx_offset++;
1662 	}
1663 
1664 	return emulated;
1665 }
1666 
kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu * vcpu)1667 static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu)
1668 {
1669 	struct kvm_run *run = vcpu->run;
1670 	enum emulation_result emulated = EMULATE_FAIL;
1671 	int r;
1672 
1673 	vcpu->arch.paddr_accessed += run->mmio.len;
1674 
1675 	if (!vcpu->mmio_is_write) {
1676 		emulated = kvmppc_handle_vmx_load(vcpu,
1677 				vcpu->arch.io_gpr, run->mmio.len, 1);
1678 	} else {
1679 		emulated = kvmppc_handle_vmx_store(vcpu,
1680 				vcpu->arch.io_gpr, run->mmio.len, 1);
1681 	}
1682 
1683 	switch (emulated) {
1684 	case EMULATE_DO_MMIO:
1685 		run->exit_reason = KVM_EXIT_MMIO;
1686 		r = RESUME_HOST;
1687 		break;
1688 	case EMULATE_FAIL:
1689 		pr_info("KVM: MMIO emulation failed (VMX repeat)\n");
1690 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1691 		run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1692 		r = RESUME_HOST;
1693 		break;
1694 	default:
1695 		r = RESUME_GUEST;
1696 		break;
1697 	}
1698 	return r;
1699 }
1700 #endif /* CONFIG_ALTIVEC */
1701 
kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)1702 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1703 {
1704 	int r = 0;
1705 	union kvmppc_one_reg val;
1706 	int size;
1707 
1708 	size = one_reg_size(reg->id);
1709 	if (size > sizeof(val))
1710 		return -EINVAL;
1711 
1712 	r = kvmppc_get_one_reg(vcpu, reg->id, &val);
1713 	if (r == -EINVAL) {
1714 		r = 0;
1715 		switch (reg->id) {
1716 #ifdef CONFIG_ALTIVEC
1717 		case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
1718 			if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1719 				r = -ENXIO;
1720 				break;
1721 			}
1722 			val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0];
1723 			break;
1724 		case KVM_REG_PPC_VSCR:
1725 			if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1726 				r = -ENXIO;
1727 				break;
1728 			}
1729 			val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]);
1730 			break;
1731 		case KVM_REG_PPC_VRSAVE:
1732 			val = get_reg_val(reg->id, vcpu->arch.vrsave);
1733 			break;
1734 #endif /* CONFIG_ALTIVEC */
1735 		default:
1736 			r = -EINVAL;
1737 			break;
1738 		}
1739 	}
1740 
1741 	if (r)
1742 		return r;
1743 
1744 	if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size))
1745 		r = -EFAULT;
1746 
1747 	return r;
1748 }
1749 
kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)1750 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1751 {
1752 	int r;
1753 	union kvmppc_one_reg val;
1754 	int size;
1755 
1756 	size = one_reg_size(reg->id);
1757 	if (size > sizeof(val))
1758 		return -EINVAL;
1759 
1760 	if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size))
1761 		return -EFAULT;
1762 
1763 	r = kvmppc_set_one_reg(vcpu, reg->id, &val);
1764 	if (r == -EINVAL) {
1765 		r = 0;
1766 		switch (reg->id) {
1767 #ifdef CONFIG_ALTIVEC
1768 		case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
1769 			if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1770 				r = -ENXIO;
1771 				break;
1772 			}
1773 			vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval;
1774 			break;
1775 		case KVM_REG_PPC_VSCR:
1776 			if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1777 				r = -ENXIO;
1778 				break;
1779 			}
1780 			vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val);
1781 			break;
1782 		case KVM_REG_PPC_VRSAVE:
1783 			if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1784 				r = -ENXIO;
1785 				break;
1786 			}
1787 			vcpu->arch.vrsave = set_reg_val(reg->id, val);
1788 			break;
1789 #endif /* CONFIG_ALTIVEC */
1790 		default:
1791 			r = -EINVAL;
1792 			break;
1793 		}
1794 	}
1795 
1796 	return r;
1797 }
1798 
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)1799 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1800 {
1801 	struct kvm_run *run = vcpu->run;
1802 	int r;
1803 
1804 	vcpu_load(vcpu);
1805 
1806 	if (vcpu->mmio_needed) {
1807 		vcpu->mmio_needed = 0;
1808 		if (!vcpu->mmio_is_write)
1809 			kvmppc_complete_mmio_load(vcpu);
1810 #ifdef CONFIG_VSX
1811 		if (vcpu->arch.mmio_vsx_copy_nums > 0) {
1812 			vcpu->arch.mmio_vsx_copy_nums--;
1813 			vcpu->arch.mmio_vsx_offset++;
1814 		}
1815 
1816 		if (vcpu->arch.mmio_vsx_copy_nums > 0) {
1817 			r = kvmppc_emulate_mmio_vsx_loadstore(vcpu);
1818 			if (r == RESUME_HOST) {
1819 				vcpu->mmio_needed = 1;
1820 				goto out;
1821 			}
1822 		}
1823 #endif
1824 #ifdef CONFIG_ALTIVEC
1825 		if (vcpu->arch.mmio_vmx_copy_nums > 0) {
1826 			vcpu->arch.mmio_vmx_copy_nums--;
1827 			vcpu->arch.mmio_vmx_offset++;
1828 		}
1829 
1830 		if (vcpu->arch.mmio_vmx_copy_nums > 0) {
1831 			r = kvmppc_emulate_mmio_vmx_loadstore(vcpu);
1832 			if (r == RESUME_HOST) {
1833 				vcpu->mmio_needed = 1;
1834 				goto out;
1835 			}
1836 		}
1837 #endif
1838 	} else if (vcpu->arch.osi_needed) {
1839 		u64 *gprs = run->osi.gprs;
1840 		int i;
1841 
1842 		for (i = 0; i < 32; i++)
1843 			kvmppc_set_gpr(vcpu, i, gprs[i]);
1844 		vcpu->arch.osi_needed = 0;
1845 	} else if (vcpu->arch.hcall_needed) {
1846 		int i;
1847 
1848 		kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret);
1849 		for (i = 0; i < 9; ++i)
1850 			kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]);
1851 		vcpu->arch.hcall_needed = 0;
1852 #ifdef CONFIG_BOOKE
1853 	} else if (vcpu->arch.epr_needed) {
1854 		kvmppc_set_epr(vcpu, run->epr.epr);
1855 		vcpu->arch.epr_needed = 0;
1856 #endif
1857 	}
1858 
1859 	kvm_sigset_activate(vcpu);
1860 
1861 	if (run->immediate_exit)
1862 		r = -EINTR;
1863 	else
1864 		r = kvmppc_vcpu_run(vcpu);
1865 
1866 	kvm_sigset_deactivate(vcpu);
1867 
1868 #ifdef CONFIG_ALTIVEC
1869 out:
1870 #endif
1871 
1872 	/*
1873 	 * We're already returning to userspace, don't pass the
1874 	 * RESUME_HOST flags along.
1875 	 */
1876 	if (r > 0)
1877 		r = 0;
1878 
1879 	vcpu_put(vcpu);
1880 	return r;
1881 }
1882 
kvm_vcpu_ioctl_interrupt(struct kvm_vcpu * vcpu,struct kvm_interrupt * irq)1883 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
1884 {
1885 	if (irq->irq == KVM_INTERRUPT_UNSET) {
1886 		kvmppc_core_dequeue_external(vcpu);
1887 		return 0;
1888 	}
1889 
1890 	kvmppc_core_queue_external(vcpu, irq);
1891 
1892 	kvm_vcpu_kick(vcpu);
1893 
1894 	return 0;
1895 }
1896 
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)1897 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1898 				     struct kvm_enable_cap *cap)
1899 {
1900 	int r;
1901 
1902 	if (cap->flags)
1903 		return -EINVAL;
1904 
1905 	switch (cap->cap) {
1906 	case KVM_CAP_PPC_OSI:
1907 		r = 0;
1908 		vcpu->arch.osi_enabled = true;
1909 		break;
1910 	case KVM_CAP_PPC_PAPR:
1911 		r = 0;
1912 		vcpu->arch.papr_enabled = true;
1913 		break;
1914 	case KVM_CAP_PPC_EPR:
1915 		r = 0;
1916 		if (cap->args[0])
1917 			vcpu->arch.epr_flags |= KVMPPC_EPR_USER;
1918 		else
1919 			vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER;
1920 		break;
1921 #ifdef CONFIG_BOOKE
1922 	case KVM_CAP_PPC_BOOKE_WATCHDOG:
1923 		r = 0;
1924 		vcpu->arch.watchdog_enabled = true;
1925 		break;
1926 #endif
1927 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
1928 	case KVM_CAP_SW_TLB: {
1929 		struct kvm_config_tlb cfg;
1930 		void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0];
1931 
1932 		r = -EFAULT;
1933 		if (copy_from_user(&cfg, user_ptr, sizeof(cfg)))
1934 			break;
1935 
1936 		r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg);
1937 		break;
1938 	}
1939 #endif
1940 #ifdef CONFIG_KVM_MPIC
1941 	case KVM_CAP_IRQ_MPIC: {
1942 		struct fd f;
1943 		struct kvm_device *dev;
1944 
1945 		r = -EBADF;
1946 		f = fdget(cap->args[0]);
1947 		if (!f.file)
1948 			break;
1949 
1950 		r = -EPERM;
1951 		dev = kvm_device_from_filp(f.file);
1952 		if (dev)
1953 			r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]);
1954 
1955 		fdput(f);
1956 		break;
1957 	}
1958 #endif
1959 #ifdef CONFIG_KVM_XICS
1960 	case KVM_CAP_IRQ_XICS: {
1961 		struct fd f;
1962 		struct kvm_device *dev;
1963 
1964 		r = -EBADF;
1965 		f = fdget(cap->args[0]);
1966 		if (!f.file)
1967 			break;
1968 
1969 		r = -EPERM;
1970 		dev = kvm_device_from_filp(f.file);
1971 		if (dev) {
1972 			if (xics_on_xive())
1973 				r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]);
1974 			else
1975 				r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]);
1976 		}
1977 
1978 		fdput(f);
1979 		break;
1980 	}
1981 #endif /* CONFIG_KVM_XICS */
1982 #ifdef CONFIG_KVM_XIVE
1983 	case KVM_CAP_PPC_IRQ_XIVE: {
1984 		struct fd f;
1985 		struct kvm_device *dev;
1986 
1987 		r = -EBADF;
1988 		f = fdget(cap->args[0]);
1989 		if (!f.file)
1990 			break;
1991 
1992 		r = -ENXIO;
1993 		if (!xive_enabled()) {
1994 			fdput(f);
1995 			break;
1996 		}
1997 
1998 		r = -EPERM;
1999 		dev = kvm_device_from_filp(f.file);
2000 		if (dev)
2001 			r = kvmppc_xive_native_connect_vcpu(dev, vcpu,
2002 							    cap->args[1]);
2003 
2004 		fdput(f);
2005 		break;
2006 	}
2007 #endif /* CONFIG_KVM_XIVE */
2008 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
2009 	case KVM_CAP_PPC_FWNMI:
2010 		r = -EINVAL;
2011 		if (!is_kvmppc_hv_enabled(vcpu->kvm))
2012 			break;
2013 		r = 0;
2014 		vcpu->kvm->arch.fwnmi_enabled = true;
2015 		break;
2016 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
2017 	default:
2018 		r = -EINVAL;
2019 		break;
2020 	}
2021 
2022 	if (!r)
2023 		r = kvmppc_sanity_check(vcpu);
2024 
2025 	return r;
2026 }
2027 
kvm_arch_intc_initialized(struct kvm * kvm)2028 bool kvm_arch_intc_initialized(struct kvm *kvm)
2029 {
2030 #ifdef CONFIG_KVM_MPIC
2031 	if (kvm->arch.mpic)
2032 		return true;
2033 #endif
2034 #ifdef CONFIG_KVM_XICS
2035 	if (kvm->arch.xics || kvm->arch.xive)
2036 		return true;
2037 #endif
2038 	return false;
2039 }
2040 
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)2041 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2042                                     struct kvm_mp_state *mp_state)
2043 {
2044 	return -EINVAL;
2045 }
2046 
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)2047 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2048                                     struct kvm_mp_state *mp_state)
2049 {
2050 	return -EINVAL;
2051 }
2052 
kvm_arch_vcpu_async_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2053 long kvm_arch_vcpu_async_ioctl(struct file *filp,
2054 			       unsigned int ioctl, unsigned long arg)
2055 {
2056 	struct kvm_vcpu *vcpu = filp->private_data;
2057 	void __user *argp = (void __user *)arg;
2058 
2059 	if (ioctl == KVM_INTERRUPT) {
2060 		struct kvm_interrupt irq;
2061 		if (copy_from_user(&irq, argp, sizeof(irq)))
2062 			return -EFAULT;
2063 		return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
2064 	}
2065 	return -ENOIOCTLCMD;
2066 }
2067 
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2068 long kvm_arch_vcpu_ioctl(struct file *filp,
2069                          unsigned int ioctl, unsigned long arg)
2070 {
2071 	struct kvm_vcpu *vcpu = filp->private_data;
2072 	void __user *argp = (void __user *)arg;
2073 	long r;
2074 
2075 	switch (ioctl) {
2076 	case KVM_ENABLE_CAP:
2077 	{
2078 		struct kvm_enable_cap cap;
2079 		r = -EFAULT;
2080 		if (copy_from_user(&cap, argp, sizeof(cap)))
2081 			goto out;
2082 		vcpu_load(vcpu);
2083 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2084 		vcpu_put(vcpu);
2085 		break;
2086 	}
2087 
2088 	case KVM_SET_ONE_REG:
2089 	case KVM_GET_ONE_REG:
2090 	{
2091 		struct kvm_one_reg reg;
2092 		r = -EFAULT;
2093 		if (copy_from_user(&reg, argp, sizeof(reg)))
2094 			goto out;
2095 		if (ioctl == KVM_SET_ONE_REG)
2096 			r = kvm_vcpu_ioctl_set_one_reg(vcpu, &reg);
2097 		else
2098 			r = kvm_vcpu_ioctl_get_one_reg(vcpu, &reg);
2099 		break;
2100 	}
2101 
2102 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
2103 	case KVM_DIRTY_TLB: {
2104 		struct kvm_dirty_tlb dirty;
2105 		r = -EFAULT;
2106 		if (copy_from_user(&dirty, argp, sizeof(dirty)))
2107 			goto out;
2108 		vcpu_load(vcpu);
2109 		r = kvm_vcpu_ioctl_dirty_tlb(vcpu, &dirty);
2110 		vcpu_put(vcpu);
2111 		break;
2112 	}
2113 #endif
2114 	default:
2115 		r = -EINVAL;
2116 	}
2117 
2118 out:
2119 	return r;
2120 }
2121 
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)2122 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2123 {
2124 	return VM_FAULT_SIGBUS;
2125 }
2126 
kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo * pvinfo)2127 static int kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo *pvinfo)
2128 {
2129 	u32 inst_nop = 0x60000000;
2130 #ifdef CONFIG_KVM_BOOKE_HV
2131 	u32 inst_sc1 = 0x44000022;
2132 	pvinfo->hcall[0] = cpu_to_be32(inst_sc1);
2133 	pvinfo->hcall[1] = cpu_to_be32(inst_nop);
2134 	pvinfo->hcall[2] = cpu_to_be32(inst_nop);
2135 	pvinfo->hcall[3] = cpu_to_be32(inst_nop);
2136 #else
2137 	u32 inst_lis = 0x3c000000;
2138 	u32 inst_ori = 0x60000000;
2139 	u32 inst_sc = 0x44000002;
2140 	u32 inst_imm_mask = 0xffff;
2141 
2142 	/*
2143 	 * The hypercall to get into KVM from within guest context is as
2144 	 * follows:
2145 	 *
2146 	 *    lis r0, r0, KVM_SC_MAGIC_R0@h
2147 	 *    ori r0, KVM_SC_MAGIC_R0@l
2148 	 *    sc
2149 	 *    nop
2150 	 */
2151 	pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask));
2152 	pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask));
2153 	pvinfo->hcall[2] = cpu_to_be32(inst_sc);
2154 	pvinfo->hcall[3] = cpu_to_be32(inst_nop);
2155 #endif
2156 
2157 	pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE;
2158 
2159 	return 0;
2160 }
2161 
kvm_arch_irqchip_in_kernel(struct kvm * kvm)2162 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2163 {
2164 	int ret = 0;
2165 
2166 #ifdef CONFIG_KVM_MPIC
2167 	ret = ret || (kvm->arch.mpic != NULL);
2168 #endif
2169 #ifdef CONFIG_KVM_XICS
2170 	ret = ret || (kvm->arch.xics != NULL);
2171 	ret = ret || (kvm->arch.xive != NULL);
2172 #endif
2173 	smp_rmb();
2174 	return ret;
2175 }
2176 
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_event,bool line_status)2177 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
2178 			  bool line_status)
2179 {
2180 	if (!kvm_arch_irqchip_in_kernel(kvm))
2181 		return -ENXIO;
2182 
2183 	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
2184 					irq_event->irq, irq_event->level,
2185 					line_status);
2186 	return 0;
2187 }
2188 
2189 
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)2190 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
2191 			    struct kvm_enable_cap *cap)
2192 {
2193 	int r;
2194 
2195 	if (cap->flags)
2196 		return -EINVAL;
2197 
2198 	switch (cap->cap) {
2199 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2200 	case KVM_CAP_PPC_ENABLE_HCALL: {
2201 		unsigned long hcall = cap->args[0];
2202 
2203 		r = -EINVAL;
2204 		if (hcall > MAX_HCALL_OPCODE || (hcall & 3) ||
2205 		    cap->args[1] > 1)
2206 			break;
2207 		if (!kvmppc_book3s_hcall_implemented(kvm, hcall))
2208 			break;
2209 		if (cap->args[1])
2210 			set_bit(hcall / 4, kvm->arch.enabled_hcalls);
2211 		else
2212 			clear_bit(hcall / 4, kvm->arch.enabled_hcalls);
2213 		r = 0;
2214 		break;
2215 	}
2216 	case KVM_CAP_PPC_SMT: {
2217 		unsigned long mode = cap->args[0];
2218 		unsigned long flags = cap->args[1];
2219 
2220 		r = -EINVAL;
2221 		if (kvm->arch.kvm_ops->set_smt_mode)
2222 			r = kvm->arch.kvm_ops->set_smt_mode(kvm, mode, flags);
2223 		break;
2224 	}
2225 
2226 	case KVM_CAP_PPC_NESTED_HV:
2227 		r = -EINVAL;
2228 		if (!is_kvmppc_hv_enabled(kvm) ||
2229 		    !kvm->arch.kvm_ops->enable_nested)
2230 			break;
2231 		r = kvm->arch.kvm_ops->enable_nested(kvm);
2232 		break;
2233 #endif
2234 #if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
2235 	case KVM_CAP_PPC_SECURE_GUEST:
2236 		r = -EINVAL;
2237 		if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_svm)
2238 			break;
2239 		r = kvm->arch.kvm_ops->enable_svm(kvm);
2240 		break;
2241 	case KVM_CAP_PPC_DAWR1:
2242 		r = -EINVAL;
2243 		if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_dawr1)
2244 			break;
2245 		r = kvm->arch.kvm_ops->enable_dawr1(kvm);
2246 		break;
2247 #endif
2248 	default:
2249 		r = -EINVAL;
2250 		break;
2251 	}
2252 
2253 	return r;
2254 }
2255 
2256 #ifdef CONFIG_PPC_BOOK3S_64
2257 /*
2258  * These functions check whether the underlying hardware is safe
2259  * against attacks based on observing the effects of speculatively
2260  * executed instructions, and whether it supplies instructions for
2261  * use in workarounds.  The information comes from firmware, either
2262  * via the device tree on powernv platforms or from an hcall on
2263  * pseries platforms.
2264  */
2265 #ifdef CONFIG_PPC_PSERIES
pseries_get_cpu_char(struct kvm_ppc_cpu_char * cp)2266 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2267 {
2268 	struct h_cpu_char_result c;
2269 	unsigned long rc;
2270 
2271 	if (!machine_is(pseries))
2272 		return -ENOTTY;
2273 
2274 	rc = plpar_get_cpu_characteristics(&c);
2275 	if (rc == H_SUCCESS) {
2276 		cp->character = c.character;
2277 		cp->behaviour = c.behaviour;
2278 		cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
2279 			KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
2280 			KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
2281 			KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
2282 			KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
2283 			KVM_PPC_CPU_CHAR_BR_HINT_HONOURED |
2284 			KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF |
2285 			KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS |
2286 			KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
2287 		cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
2288 			KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
2289 			KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR |
2290 			KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE;
2291 	}
2292 	return 0;
2293 }
2294 #else
pseries_get_cpu_char(struct kvm_ppc_cpu_char * cp)2295 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2296 {
2297 	return -ENOTTY;
2298 }
2299 #endif
2300 
have_fw_feat(struct device_node * fw_features,const char * state,const char * name)2301 static inline bool have_fw_feat(struct device_node *fw_features,
2302 				const char *state, const char *name)
2303 {
2304 	struct device_node *np;
2305 	bool r = false;
2306 
2307 	np = of_get_child_by_name(fw_features, name);
2308 	if (np) {
2309 		r = of_property_read_bool(np, state);
2310 		of_node_put(np);
2311 	}
2312 	return r;
2313 }
2314 
kvmppc_get_cpu_char(struct kvm_ppc_cpu_char * cp)2315 static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2316 {
2317 	struct device_node *np, *fw_features;
2318 	int r;
2319 
2320 	memset(cp, 0, sizeof(*cp));
2321 	r = pseries_get_cpu_char(cp);
2322 	if (r != -ENOTTY)
2323 		return r;
2324 
2325 	np = of_find_node_by_name(NULL, "ibm,opal");
2326 	if (np) {
2327 		fw_features = of_get_child_by_name(np, "fw-features");
2328 		of_node_put(np);
2329 		if (!fw_features)
2330 			return 0;
2331 		if (have_fw_feat(fw_features, "enabled",
2332 				 "inst-spec-barrier-ori31,31,0"))
2333 			cp->character |= KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31;
2334 		if (have_fw_feat(fw_features, "enabled",
2335 				 "fw-bcctrl-serialized"))
2336 			cp->character |= KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED;
2337 		if (have_fw_feat(fw_features, "enabled",
2338 				 "inst-l1d-flush-ori30,30,0"))
2339 			cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30;
2340 		if (have_fw_feat(fw_features, "enabled",
2341 				 "inst-l1d-flush-trig2"))
2342 			cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2;
2343 		if (have_fw_feat(fw_features, "enabled",
2344 				 "fw-l1d-thread-split"))
2345 			cp->character |= KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV;
2346 		if (have_fw_feat(fw_features, "enabled",
2347 				 "fw-count-cache-disabled"))
2348 			cp->character |= KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS;
2349 		if (have_fw_feat(fw_features, "enabled",
2350 				 "fw-count-cache-flush-bcctr2,0,0"))
2351 			cp->character |= KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
2352 		cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
2353 			KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
2354 			KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
2355 			KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
2356 			KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
2357 			KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS |
2358 			KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
2359 
2360 		if (have_fw_feat(fw_features, "enabled",
2361 				 "speculation-policy-favor-security"))
2362 			cp->behaviour |= KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY;
2363 		if (!have_fw_feat(fw_features, "disabled",
2364 				  "needs-l1d-flush-msr-pr-0-to-1"))
2365 			cp->behaviour |= KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR;
2366 		if (!have_fw_feat(fw_features, "disabled",
2367 				  "needs-spec-barrier-for-bound-checks"))
2368 			cp->behaviour |= KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
2369 		if (have_fw_feat(fw_features, "enabled",
2370 				 "needs-count-cache-flush-on-context-switch"))
2371 			cp->behaviour |= KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE;
2372 		cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
2373 			KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
2374 			KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR |
2375 			KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE;
2376 
2377 		of_node_put(fw_features);
2378 	}
2379 
2380 	return 0;
2381 }
2382 #endif
2383 
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2384 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
2385 {
2386 	struct kvm *kvm __maybe_unused = filp->private_data;
2387 	void __user *argp = (void __user *)arg;
2388 	int r;
2389 
2390 	switch (ioctl) {
2391 	case KVM_PPC_GET_PVINFO: {
2392 		struct kvm_ppc_pvinfo pvinfo;
2393 		memset(&pvinfo, 0, sizeof(pvinfo));
2394 		r = kvm_vm_ioctl_get_pvinfo(&pvinfo);
2395 		if (copy_to_user(argp, &pvinfo, sizeof(pvinfo))) {
2396 			r = -EFAULT;
2397 			goto out;
2398 		}
2399 
2400 		break;
2401 	}
2402 #ifdef CONFIG_SPAPR_TCE_IOMMU
2403 	case KVM_CREATE_SPAPR_TCE_64: {
2404 		struct kvm_create_spapr_tce_64 create_tce_64;
2405 
2406 		r = -EFAULT;
2407 		if (copy_from_user(&create_tce_64, argp, sizeof(create_tce_64)))
2408 			goto out;
2409 		if (create_tce_64.flags) {
2410 			r = -EINVAL;
2411 			goto out;
2412 		}
2413 		r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
2414 		goto out;
2415 	}
2416 	case KVM_CREATE_SPAPR_TCE: {
2417 		struct kvm_create_spapr_tce create_tce;
2418 		struct kvm_create_spapr_tce_64 create_tce_64;
2419 
2420 		r = -EFAULT;
2421 		if (copy_from_user(&create_tce, argp, sizeof(create_tce)))
2422 			goto out;
2423 
2424 		create_tce_64.liobn = create_tce.liobn;
2425 		create_tce_64.page_shift = IOMMU_PAGE_SHIFT_4K;
2426 		create_tce_64.offset = 0;
2427 		create_tce_64.size = create_tce.window_size >>
2428 				IOMMU_PAGE_SHIFT_4K;
2429 		create_tce_64.flags = 0;
2430 		r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
2431 		goto out;
2432 	}
2433 #endif
2434 #ifdef CONFIG_PPC_BOOK3S_64
2435 	case KVM_PPC_GET_SMMU_INFO: {
2436 		struct kvm_ppc_smmu_info info;
2437 		struct kvm *kvm = filp->private_data;
2438 
2439 		memset(&info, 0, sizeof(info));
2440 		r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info);
2441 		if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
2442 			r = -EFAULT;
2443 		break;
2444 	}
2445 	case KVM_PPC_RTAS_DEFINE_TOKEN: {
2446 		struct kvm *kvm = filp->private_data;
2447 
2448 		r = kvm_vm_ioctl_rtas_define_token(kvm, argp);
2449 		break;
2450 	}
2451 	case KVM_PPC_CONFIGURE_V3_MMU: {
2452 		struct kvm *kvm = filp->private_data;
2453 		struct kvm_ppc_mmuv3_cfg cfg;
2454 
2455 		r = -EINVAL;
2456 		if (!kvm->arch.kvm_ops->configure_mmu)
2457 			goto out;
2458 		r = -EFAULT;
2459 		if (copy_from_user(&cfg, argp, sizeof(cfg)))
2460 			goto out;
2461 		r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg);
2462 		break;
2463 	}
2464 	case KVM_PPC_GET_RMMU_INFO: {
2465 		struct kvm *kvm = filp->private_data;
2466 		struct kvm_ppc_rmmu_info info;
2467 
2468 		r = -EINVAL;
2469 		if (!kvm->arch.kvm_ops->get_rmmu_info)
2470 			goto out;
2471 		r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info);
2472 		if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
2473 			r = -EFAULT;
2474 		break;
2475 	}
2476 	case KVM_PPC_GET_CPU_CHAR: {
2477 		struct kvm_ppc_cpu_char cpuchar;
2478 
2479 		r = kvmppc_get_cpu_char(&cpuchar);
2480 		if (r >= 0 && copy_to_user(argp, &cpuchar, sizeof(cpuchar)))
2481 			r = -EFAULT;
2482 		break;
2483 	}
2484 	case KVM_PPC_SVM_OFF: {
2485 		struct kvm *kvm = filp->private_data;
2486 
2487 		r = 0;
2488 		if (!kvm->arch.kvm_ops->svm_off)
2489 			goto out;
2490 
2491 		r = kvm->arch.kvm_ops->svm_off(kvm);
2492 		break;
2493 	}
2494 	default: {
2495 		struct kvm *kvm = filp->private_data;
2496 		r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg);
2497 	}
2498 #else /* CONFIG_PPC_BOOK3S_64 */
2499 	default:
2500 		r = -ENOTTY;
2501 #endif
2502 	}
2503 out:
2504 	return r;
2505 }
2506 
2507 static DEFINE_IDA(lpid_inuse);
2508 static unsigned long nr_lpids;
2509 
kvmppc_alloc_lpid(void)2510 long kvmppc_alloc_lpid(void)
2511 {
2512 	int lpid;
2513 
2514 	/* The host LPID must always be 0 (allocation starts at 1) */
2515 	lpid = ida_alloc_range(&lpid_inuse, 1, nr_lpids - 1, GFP_KERNEL);
2516 	if (lpid < 0) {
2517 		if (lpid == -ENOMEM)
2518 			pr_err("%s: Out of memory\n", __func__);
2519 		else
2520 			pr_err("%s: No LPIDs free\n", __func__);
2521 		return -ENOMEM;
2522 	}
2523 
2524 	return lpid;
2525 }
2526 EXPORT_SYMBOL_GPL(kvmppc_alloc_lpid);
2527 
kvmppc_free_lpid(long lpid)2528 void kvmppc_free_lpid(long lpid)
2529 {
2530 	ida_free(&lpid_inuse, lpid);
2531 }
2532 EXPORT_SYMBOL_GPL(kvmppc_free_lpid);
2533 
2534 /* nr_lpids_param includes the host LPID */
kvmppc_init_lpid(unsigned long nr_lpids_param)2535 void kvmppc_init_lpid(unsigned long nr_lpids_param)
2536 {
2537 	nr_lpids = nr_lpids_param;
2538 }
2539 EXPORT_SYMBOL_GPL(kvmppc_init_lpid);
2540 
2541 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ppc_instr);
2542 
kvm_arch_create_vcpu_debugfs(struct kvm_vcpu * vcpu,struct dentry * debugfs_dentry)2543 void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry)
2544 {
2545 	if (vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs)
2546 		vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs(vcpu, debugfs_dentry);
2547 }
2548 
kvm_arch_create_vm_debugfs(struct kvm * kvm)2549 int kvm_arch_create_vm_debugfs(struct kvm *kvm)
2550 {
2551 	if (kvm->arch.kvm_ops->create_vm_debugfs)
2552 		kvm->arch.kvm_ops->create_vm_debugfs(kvm);
2553 	return 0;
2554 }
2555