xref: /openbmc/linux/arch/powerpc/kvm/book3s_hv.c (revision 6aa7de05)
1 /*
2  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4  *
5  * Authors:
6  *    Paul Mackerras <paulus@au1.ibm.com>
7  *    Alexander Graf <agraf@suse.de>
8  *    Kevin Wolf <mail@kevin-wolf.de>
9  *
10  * Description: KVM functions specific to running on Book 3S
11  * processors in hypervisor mode (specifically POWER7 and later).
12  *
13  * This file is derived from arch/powerpc/kvm/book3s.c,
14  * by Alexander Graf <agraf@suse.de>.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License, version 2, as
18  * published by the Free Software Foundation.
19  */
20 
21 #include <linux/kvm_host.h>
22 #include <linux/err.h>
23 #include <linux/slab.h>
24 #include <linux/preempt.h>
25 #include <linux/sched/signal.h>
26 #include <linux/sched/stat.h>
27 #include <linux/delay.h>
28 #include <linux/export.h>
29 #include <linux/fs.h>
30 #include <linux/anon_inodes.h>
31 #include <linux/cpu.h>
32 #include <linux/cpumask.h>
33 #include <linux/spinlock.h>
34 #include <linux/page-flags.h>
35 #include <linux/srcu.h>
36 #include <linux/miscdevice.h>
37 #include <linux/debugfs.h>
38 #include <linux/gfp.h>
39 #include <linux/vmalloc.h>
40 #include <linux/highmem.h>
41 #include <linux/hugetlb.h>
42 #include <linux/kvm_irqfd.h>
43 #include <linux/irqbypass.h>
44 #include <linux/module.h>
45 #include <linux/compiler.h>
46 #include <linux/of.h>
47 
48 #include <asm/reg.h>
49 #include <asm/ppc-opcode.h>
50 #include <asm/disassemble.h>
51 #include <asm/cputable.h>
52 #include <asm/cacheflush.h>
53 #include <asm/tlbflush.h>
54 #include <linux/uaccess.h>
55 #include <asm/io.h>
56 #include <asm/kvm_ppc.h>
57 #include <asm/kvm_book3s.h>
58 #include <asm/mmu_context.h>
59 #include <asm/lppaca.h>
60 #include <asm/processor.h>
61 #include <asm/cputhreads.h>
62 #include <asm/page.h>
63 #include <asm/hvcall.h>
64 #include <asm/switch_to.h>
65 #include <asm/smp.h>
66 #include <asm/dbell.h>
67 #include <asm/hmi.h>
68 #include <asm/pnv-pci.h>
69 #include <asm/mmu.h>
70 #include <asm/opal.h>
71 #include <asm/xics.h>
72 #include <asm/xive.h>
73 
74 #include "book3s.h"
75 
76 #define CREATE_TRACE_POINTS
77 #include "trace_hv.h"
78 
79 /* #define EXIT_DEBUG */
80 /* #define EXIT_DEBUG_SIMPLE */
81 /* #define EXIT_DEBUG_INT */
82 
83 /* Used to indicate that a guest page fault needs to be handled */
84 #define RESUME_PAGE_FAULT	(RESUME_GUEST | RESUME_FLAG_ARCH1)
85 /* Used to indicate that a guest passthrough interrupt needs to be handled */
86 #define RESUME_PASSTHROUGH	(RESUME_GUEST | RESUME_FLAG_ARCH2)
87 
88 /* Used as a "null" value for timebase values */
89 #define TB_NIL	(~(u64)0)
90 
91 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
92 
93 static int dynamic_mt_modes = 6;
94 module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR);
95 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
96 static int target_smt_mode;
97 module_param(target_smt_mode, int, S_IRUGO | S_IWUSR);
98 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
99 
100 #ifdef CONFIG_KVM_XICS
101 static struct kernel_param_ops module_param_ops = {
102 	.set = param_set_int,
103 	.get = param_get_int,
104 };
105 
106 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass,
107 							S_IRUGO | S_IWUSR);
108 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
109 
110 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect,
111 							S_IRUGO | S_IWUSR);
112 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
113 #endif
114 
115 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
116 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
117 
118 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
119 		int *ip)
120 {
121 	int i = *ip;
122 	struct kvm_vcpu *vcpu;
123 
124 	while (++i < MAX_SMT_THREADS) {
125 		vcpu = READ_ONCE(vc->runnable_threads[i]);
126 		if (vcpu) {
127 			*ip = i;
128 			return vcpu;
129 		}
130 	}
131 	return NULL;
132 }
133 
134 /* Used to traverse the list of runnable threads for a given vcore */
135 #define for_each_runnable_thread(i, vcpu, vc) \
136 	for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
137 
138 static bool kvmppc_ipi_thread(int cpu)
139 {
140 	unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
141 
142 	/* On POWER9 we can use msgsnd to IPI any cpu */
143 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
144 		msg |= get_hard_smp_processor_id(cpu);
145 		smp_mb();
146 		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
147 		return true;
148 	}
149 
150 	/* On POWER8 for IPIs to threads in the same core, use msgsnd */
151 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
152 		preempt_disable();
153 		if (cpu_first_thread_sibling(cpu) ==
154 		    cpu_first_thread_sibling(smp_processor_id())) {
155 			msg |= cpu_thread_in_core(cpu);
156 			smp_mb();
157 			__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
158 			preempt_enable();
159 			return true;
160 		}
161 		preempt_enable();
162 	}
163 
164 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
165 	if (cpu >= 0 && cpu < nr_cpu_ids) {
166 		if (paca[cpu].kvm_hstate.xics_phys) {
167 			xics_wake_cpu(cpu);
168 			return true;
169 		}
170 		opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
171 		return true;
172 	}
173 #endif
174 
175 	return false;
176 }
177 
178 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
179 {
180 	int cpu;
181 	struct swait_queue_head *wqp;
182 
183 	wqp = kvm_arch_vcpu_wq(vcpu);
184 	if (swq_has_sleeper(wqp)) {
185 		swake_up(wqp);
186 		++vcpu->stat.halt_wakeup;
187 	}
188 
189 	cpu = READ_ONCE(vcpu->arch.thread_cpu);
190 	if (cpu >= 0 && kvmppc_ipi_thread(cpu))
191 		return;
192 
193 	/* CPU points to the first thread of the core */
194 	cpu = vcpu->cpu;
195 	if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
196 		smp_send_reschedule(cpu);
197 }
198 
199 /*
200  * We use the vcpu_load/put functions to measure stolen time.
201  * Stolen time is counted as time when either the vcpu is able to
202  * run as part of a virtual core, but the task running the vcore
203  * is preempted or sleeping, or when the vcpu needs something done
204  * in the kernel by the task running the vcpu, but that task is
205  * preempted or sleeping.  Those two things have to be counted
206  * separately, since one of the vcpu tasks will take on the job
207  * of running the core, and the other vcpu tasks in the vcore will
208  * sleep waiting for it to do that, but that sleep shouldn't count
209  * as stolen time.
210  *
211  * Hence we accumulate stolen time when the vcpu can run as part of
212  * a vcore using vc->stolen_tb, and the stolen time when the vcpu
213  * needs its task to do other things in the kernel (for example,
214  * service a page fault) in busy_stolen.  We don't accumulate
215  * stolen time for a vcore when it is inactive, or for a vcpu
216  * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
217  * a misnomer; it means that the vcpu task is not executing in
218  * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
219  * the kernel.  We don't have any way of dividing up that time
220  * between time that the vcpu is genuinely stopped, time that
221  * the task is actively working on behalf of the vcpu, and time
222  * that the task is preempted, so we don't count any of it as
223  * stolen.
224  *
225  * Updates to busy_stolen are protected by arch.tbacct_lock;
226  * updates to vc->stolen_tb are protected by the vcore->stoltb_lock
227  * lock.  The stolen times are measured in units of timebase ticks.
228  * (Note that the != TB_NIL checks below are purely defensive;
229  * they should never fail.)
230  */
231 
232 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
233 {
234 	unsigned long flags;
235 
236 	spin_lock_irqsave(&vc->stoltb_lock, flags);
237 	vc->preempt_tb = mftb();
238 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
239 }
240 
241 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
242 {
243 	unsigned long flags;
244 
245 	spin_lock_irqsave(&vc->stoltb_lock, flags);
246 	if (vc->preempt_tb != TB_NIL) {
247 		vc->stolen_tb += mftb() - vc->preempt_tb;
248 		vc->preempt_tb = TB_NIL;
249 	}
250 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
251 }
252 
253 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
254 {
255 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
256 	unsigned long flags;
257 
258 	/*
259 	 * We can test vc->runner without taking the vcore lock,
260 	 * because only this task ever sets vc->runner to this
261 	 * vcpu, and once it is set to this vcpu, only this task
262 	 * ever sets it to NULL.
263 	 */
264 	if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
265 		kvmppc_core_end_stolen(vc);
266 
267 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
268 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
269 	    vcpu->arch.busy_preempt != TB_NIL) {
270 		vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
271 		vcpu->arch.busy_preempt = TB_NIL;
272 	}
273 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
274 }
275 
276 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
277 {
278 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
279 	unsigned long flags;
280 
281 	if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
282 		kvmppc_core_start_stolen(vc);
283 
284 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
285 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
286 		vcpu->arch.busy_preempt = mftb();
287 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
288 }
289 
290 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
291 {
292 	/*
293 	 * Check for illegal transactional state bit combination
294 	 * and if we find it, force the TS field to a safe state.
295 	 */
296 	if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
297 		msr &= ~MSR_TS_MASK;
298 	vcpu->arch.shregs.msr = msr;
299 	kvmppc_end_cede(vcpu);
300 }
301 
302 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
303 {
304 	vcpu->arch.pvr = pvr;
305 }
306 
307 /* Dummy value used in computing PCR value below */
308 #define PCR_ARCH_300	(PCR_ARCH_207 << 1)
309 
310 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
311 {
312 	unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
313 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
314 
315 	/* We can (emulate) our own architecture version and anything older */
316 	if (cpu_has_feature(CPU_FTR_ARCH_300))
317 		host_pcr_bit = PCR_ARCH_300;
318 	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
319 		host_pcr_bit = PCR_ARCH_207;
320 	else if (cpu_has_feature(CPU_FTR_ARCH_206))
321 		host_pcr_bit = PCR_ARCH_206;
322 	else
323 		host_pcr_bit = PCR_ARCH_205;
324 
325 	/* Determine lowest PCR bit needed to run guest in given PVR level */
326 	guest_pcr_bit = host_pcr_bit;
327 	if (arch_compat) {
328 		switch (arch_compat) {
329 		case PVR_ARCH_205:
330 			guest_pcr_bit = PCR_ARCH_205;
331 			break;
332 		case PVR_ARCH_206:
333 		case PVR_ARCH_206p:
334 			guest_pcr_bit = PCR_ARCH_206;
335 			break;
336 		case PVR_ARCH_207:
337 			guest_pcr_bit = PCR_ARCH_207;
338 			break;
339 		case PVR_ARCH_300:
340 			guest_pcr_bit = PCR_ARCH_300;
341 			break;
342 		default:
343 			return -EINVAL;
344 		}
345 	}
346 
347 	/* Check requested PCR bits don't exceed our capabilities */
348 	if (guest_pcr_bit > host_pcr_bit)
349 		return -EINVAL;
350 
351 	spin_lock(&vc->lock);
352 	vc->arch_compat = arch_compat;
353 	/* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */
354 	vc->pcr = host_pcr_bit - guest_pcr_bit;
355 	spin_unlock(&vc->lock);
356 
357 	return 0;
358 }
359 
360 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
361 {
362 	int r;
363 
364 	pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
365 	pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
366 	       vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
367 	for (r = 0; r < 16; ++r)
368 		pr_err("r%2d = %.16lx  r%d = %.16lx\n",
369 		       r, kvmppc_get_gpr(vcpu, r),
370 		       r+16, kvmppc_get_gpr(vcpu, r+16));
371 	pr_err("ctr = %.16lx  lr  = %.16lx\n",
372 	       vcpu->arch.ctr, vcpu->arch.lr);
373 	pr_err("srr0 = %.16llx srr1 = %.16llx\n",
374 	       vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
375 	pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
376 	       vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
377 	pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
378 	       vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
379 	pr_err("cr = %.8x  xer = %.16lx  dsisr = %.8x\n",
380 	       vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
381 	pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
382 	pr_err("fault dar = %.16lx dsisr = %.8x\n",
383 	       vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
384 	pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
385 	for (r = 0; r < vcpu->arch.slb_max; ++r)
386 		pr_err("  ESID = %.16llx VSID = %.16llx\n",
387 		       vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
388 	pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
389 	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
390 	       vcpu->arch.last_inst);
391 }
392 
393 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
394 {
395 	struct kvm_vcpu *ret;
396 
397 	mutex_lock(&kvm->lock);
398 	ret = kvm_get_vcpu_by_id(kvm, id);
399 	mutex_unlock(&kvm->lock);
400 	return ret;
401 }
402 
403 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
404 {
405 	vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
406 	vpa->yield_count = cpu_to_be32(1);
407 }
408 
409 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
410 		   unsigned long addr, unsigned long len)
411 {
412 	/* check address is cacheline aligned */
413 	if (addr & (L1_CACHE_BYTES - 1))
414 		return -EINVAL;
415 	spin_lock(&vcpu->arch.vpa_update_lock);
416 	if (v->next_gpa != addr || v->len != len) {
417 		v->next_gpa = addr;
418 		v->len = addr ? len : 0;
419 		v->update_pending = 1;
420 	}
421 	spin_unlock(&vcpu->arch.vpa_update_lock);
422 	return 0;
423 }
424 
425 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
426 struct reg_vpa {
427 	u32 dummy;
428 	union {
429 		__be16 hword;
430 		__be32 word;
431 	} length;
432 };
433 
434 static int vpa_is_registered(struct kvmppc_vpa *vpap)
435 {
436 	if (vpap->update_pending)
437 		return vpap->next_gpa != 0;
438 	return vpap->pinned_addr != NULL;
439 }
440 
441 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
442 				       unsigned long flags,
443 				       unsigned long vcpuid, unsigned long vpa)
444 {
445 	struct kvm *kvm = vcpu->kvm;
446 	unsigned long len, nb;
447 	void *va;
448 	struct kvm_vcpu *tvcpu;
449 	int err;
450 	int subfunc;
451 	struct kvmppc_vpa *vpap;
452 
453 	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
454 	if (!tvcpu)
455 		return H_PARAMETER;
456 
457 	subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
458 	if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
459 	    subfunc == H_VPA_REG_SLB) {
460 		/* Registering new area - address must be cache-line aligned */
461 		if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
462 			return H_PARAMETER;
463 
464 		/* convert logical addr to kernel addr and read length */
465 		va = kvmppc_pin_guest_page(kvm, vpa, &nb);
466 		if (va == NULL)
467 			return H_PARAMETER;
468 		if (subfunc == H_VPA_REG_VPA)
469 			len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
470 		else
471 			len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
472 		kvmppc_unpin_guest_page(kvm, va, vpa, false);
473 
474 		/* Check length */
475 		if (len > nb || len < sizeof(struct reg_vpa))
476 			return H_PARAMETER;
477 	} else {
478 		vpa = 0;
479 		len = 0;
480 	}
481 
482 	err = H_PARAMETER;
483 	vpap = NULL;
484 	spin_lock(&tvcpu->arch.vpa_update_lock);
485 
486 	switch (subfunc) {
487 	case H_VPA_REG_VPA:		/* register VPA */
488 		/*
489 		 * The size of our lppaca is 1kB because of the way we align
490 		 * it for the guest to avoid crossing a 4kB boundary. We only
491 		 * use 640 bytes of the structure though, so we should accept
492 		 * clients that set a size of 640.
493 		 */
494 		if (len < 640)
495 			break;
496 		vpap = &tvcpu->arch.vpa;
497 		err = 0;
498 		break;
499 
500 	case H_VPA_REG_DTL:		/* register DTL */
501 		if (len < sizeof(struct dtl_entry))
502 			break;
503 		len -= len % sizeof(struct dtl_entry);
504 
505 		/* Check that they have previously registered a VPA */
506 		err = H_RESOURCE;
507 		if (!vpa_is_registered(&tvcpu->arch.vpa))
508 			break;
509 
510 		vpap = &tvcpu->arch.dtl;
511 		err = 0;
512 		break;
513 
514 	case H_VPA_REG_SLB:		/* register SLB shadow buffer */
515 		/* Check that they have previously registered a VPA */
516 		err = H_RESOURCE;
517 		if (!vpa_is_registered(&tvcpu->arch.vpa))
518 			break;
519 
520 		vpap = &tvcpu->arch.slb_shadow;
521 		err = 0;
522 		break;
523 
524 	case H_VPA_DEREG_VPA:		/* deregister VPA */
525 		/* Check they don't still have a DTL or SLB buf registered */
526 		err = H_RESOURCE;
527 		if (vpa_is_registered(&tvcpu->arch.dtl) ||
528 		    vpa_is_registered(&tvcpu->arch.slb_shadow))
529 			break;
530 
531 		vpap = &tvcpu->arch.vpa;
532 		err = 0;
533 		break;
534 
535 	case H_VPA_DEREG_DTL:		/* deregister DTL */
536 		vpap = &tvcpu->arch.dtl;
537 		err = 0;
538 		break;
539 
540 	case H_VPA_DEREG_SLB:		/* deregister SLB shadow buffer */
541 		vpap = &tvcpu->arch.slb_shadow;
542 		err = 0;
543 		break;
544 	}
545 
546 	if (vpap) {
547 		vpap->next_gpa = vpa;
548 		vpap->len = len;
549 		vpap->update_pending = 1;
550 	}
551 
552 	spin_unlock(&tvcpu->arch.vpa_update_lock);
553 
554 	return err;
555 }
556 
557 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
558 {
559 	struct kvm *kvm = vcpu->kvm;
560 	void *va;
561 	unsigned long nb;
562 	unsigned long gpa;
563 
564 	/*
565 	 * We need to pin the page pointed to by vpap->next_gpa,
566 	 * but we can't call kvmppc_pin_guest_page under the lock
567 	 * as it does get_user_pages() and down_read().  So we
568 	 * have to drop the lock, pin the page, then get the lock
569 	 * again and check that a new area didn't get registered
570 	 * in the meantime.
571 	 */
572 	for (;;) {
573 		gpa = vpap->next_gpa;
574 		spin_unlock(&vcpu->arch.vpa_update_lock);
575 		va = NULL;
576 		nb = 0;
577 		if (gpa)
578 			va = kvmppc_pin_guest_page(kvm, gpa, &nb);
579 		spin_lock(&vcpu->arch.vpa_update_lock);
580 		if (gpa == vpap->next_gpa)
581 			break;
582 		/* sigh... unpin that one and try again */
583 		if (va)
584 			kvmppc_unpin_guest_page(kvm, va, gpa, false);
585 	}
586 
587 	vpap->update_pending = 0;
588 	if (va && nb < vpap->len) {
589 		/*
590 		 * If it's now too short, it must be that userspace
591 		 * has changed the mappings underlying guest memory,
592 		 * so unregister the region.
593 		 */
594 		kvmppc_unpin_guest_page(kvm, va, gpa, false);
595 		va = NULL;
596 	}
597 	if (vpap->pinned_addr)
598 		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
599 					vpap->dirty);
600 	vpap->gpa = gpa;
601 	vpap->pinned_addr = va;
602 	vpap->dirty = false;
603 	if (va)
604 		vpap->pinned_end = va + vpap->len;
605 }
606 
607 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
608 {
609 	if (!(vcpu->arch.vpa.update_pending ||
610 	      vcpu->arch.slb_shadow.update_pending ||
611 	      vcpu->arch.dtl.update_pending))
612 		return;
613 
614 	spin_lock(&vcpu->arch.vpa_update_lock);
615 	if (vcpu->arch.vpa.update_pending) {
616 		kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
617 		if (vcpu->arch.vpa.pinned_addr)
618 			init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
619 	}
620 	if (vcpu->arch.dtl.update_pending) {
621 		kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
622 		vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
623 		vcpu->arch.dtl_index = 0;
624 	}
625 	if (vcpu->arch.slb_shadow.update_pending)
626 		kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
627 	spin_unlock(&vcpu->arch.vpa_update_lock);
628 }
629 
630 /*
631  * Return the accumulated stolen time for the vcore up until `now'.
632  * The caller should hold the vcore lock.
633  */
634 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
635 {
636 	u64 p;
637 	unsigned long flags;
638 
639 	spin_lock_irqsave(&vc->stoltb_lock, flags);
640 	p = vc->stolen_tb;
641 	if (vc->vcore_state != VCORE_INACTIVE &&
642 	    vc->preempt_tb != TB_NIL)
643 		p += now - vc->preempt_tb;
644 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
645 	return p;
646 }
647 
648 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
649 				    struct kvmppc_vcore *vc)
650 {
651 	struct dtl_entry *dt;
652 	struct lppaca *vpa;
653 	unsigned long stolen;
654 	unsigned long core_stolen;
655 	u64 now;
656 	unsigned long flags;
657 
658 	dt = vcpu->arch.dtl_ptr;
659 	vpa = vcpu->arch.vpa.pinned_addr;
660 	now = mftb();
661 	core_stolen = vcore_stolen_time(vc, now);
662 	stolen = core_stolen - vcpu->arch.stolen_logged;
663 	vcpu->arch.stolen_logged = core_stolen;
664 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
665 	stolen += vcpu->arch.busy_stolen;
666 	vcpu->arch.busy_stolen = 0;
667 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
668 	if (!dt || !vpa)
669 		return;
670 	memset(dt, 0, sizeof(struct dtl_entry));
671 	dt->dispatch_reason = 7;
672 	dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
673 	dt->timebase = cpu_to_be64(now + vc->tb_offset);
674 	dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
675 	dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
676 	dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
677 	++dt;
678 	if (dt == vcpu->arch.dtl.pinned_end)
679 		dt = vcpu->arch.dtl.pinned_addr;
680 	vcpu->arch.dtl_ptr = dt;
681 	/* order writing *dt vs. writing vpa->dtl_idx */
682 	smp_wmb();
683 	vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
684 	vcpu->arch.dtl.dirty = true;
685 }
686 
687 /* See if there is a doorbell interrupt pending for a vcpu */
688 static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
689 {
690 	int thr;
691 	struct kvmppc_vcore *vc;
692 
693 	if (vcpu->arch.doorbell_request)
694 		return true;
695 	/*
696 	 * Ensure that the read of vcore->dpdes comes after the read
697 	 * of vcpu->doorbell_request.  This barrier matches the
698 	 * lwsync in book3s_hv_rmhandlers.S just before the
699 	 * fast_guest_return label.
700 	 */
701 	smp_rmb();
702 	vc = vcpu->arch.vcore;
703 	thr = vcpu->vcpu_id - vc->first_vcpuid;
704 	return !!(vc->dpdes & (1 << thr));
705 }
706 
707 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
708 {
709 	if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
710 		return true;
711 	if ((!vcpu->arch.vcore->arch_compat) &&
712 	    cpu_has_feature(CPU_FTR_ARCH_207S))
713 		return true;
714 	return false;
715 }
716 
717 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
718 			     unsigned long resource, unsigned long value1,
719 			     unsigned long value2)
720 {
721 	switch (resource) {
722 	case H_SET_MODE_RESOURCE_SET_CIABR:
723 		if (!kvmppc_power8_compatible(vcpu))
724 			return H_P2;
725 		if (value2)
726 			return H_P4;
727 		if (mflags)
728 			return H_UNSUPPORTED_FLAG_START;
729 		/* Guests can't breakpoint the hypervisor */
730 		if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
731 			return H_P3;
732 		vcpu->arch.ciabr  = value1;
733 		return H_SUCCESS;
734 	case H_SET_MODE_RESOURCE_SET_DAWR:
735 		if (!kvmppc_power8_compatible(vcpu))
736 			return H_P2;
737 		if (mflags)
738 			return H_UNSUPPORTED_FLAG_START;
739 		if (value2 & DABRX_HYP)
740 			return H_P4;
741 		vcpu->arch.dawr  = value1;
742 		vcpu->arch.dawrx = value2;
743 		return H_SUCCESS;
744 	default:
745 		return H_TOO_HARD;
746 	}
747 }
748 
749 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
750 {
751 	struct kvmppc_vcore *vcore = target->arch.vcore;
752 
753 	/*
754 	 * We expect to have been called by the real mode handler
755 	 * (kvmppc_rm_h_confer()) which would have directly returned
756 	 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may
757 	 * have useful work to do and should not confer) so we don't
758 	 * recheck that here.
759 	 */
760 
761 	spin_lock(&vcore->lock);
762 	if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
763 	    vcore->vcore_state != VCORE_INACTIVE &&
764 	    vcore->runner)
765 		target = vcore->runner;
766 	spin_unlock(&vcore->lock);
767 
768 	return kvm_vcpu_yield_to(target);
769 }
770 
771 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
772 {
773 	int yield_count = 0;
774 	struct lppaca *lppaca;
775 
776 	spin_lock(&vcpu->arch.vpa_update_lock);
777 	lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
778 	if (lppaca)
779 		yield_count = be32_to_cpu(lppaca->yield_count);
780 	spin_unlock(&vcpu->arch.vpa_update_lock);
781 	return yield_count;
782 }
783 
784 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
785 {
786 	unsigned long req = kvmppc_get_gpr(vcpu, 3);
787 	unsigned long target, ret = H_SUCCESS;
788 	int yield_count;
789 	struct kvm_vcpu *tvcpu;
790 	int idx, rc;
791 
792 	if (req <= MAX_HCALL_OPCODE &&
793 	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
794 		return RESUME_HOST;
795 
796 	switch (req) {
797 	case H_CEDE:
798 		break;
799 	case H_PROD:
800 		target = kvmppc_get_gpr(vcpu, 4);
801 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
802 		if (!tvcpu) {
803 			ret = H_PARAMETER;
804 			break;
805 		}
806 		tvcpu->arch.prodded = 1;
807 		smp_mb();
808 		if (tvcpu->arch.ceded)
809 			kvmppc_fast_vcpu_kick_hv(tvcpu);
810 		break;
811 	case H_CONFER:
812 		target = kvmppc_get_gpr(vcpu, 4);
813 		if (target == -1)
814 			break;
815 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
816 		if (!tvcpu) {
817 			ret = H_PARAMETER;
818 			break;
819 		}
820 		yield_count = kvmppc_get_gpr(vcpu, 5);
821 		if (kvmppc_get_yield_count(tvcpu) != yield_count)
822 			break;
823 		kvm_arch_vcpu_yield_to(tvcpu);
824 		break;
825 	case H_REGISTER_VPA:
826 		ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
827 					kvmppc_get_gpr(vcpu, 5),
828 					kvmppc_get_gpr(vcpu, 6));
829 		break;
830 	case H_RTAS:
831 		if (list_empty(&vcpu->kvm->arch.rtas_tokens))
832 			return RESUME_HOST;
833 
834 		idx = srcu_read_lock(&vcpu->kvm->srcu);
835 		rc = kvmppc_rtas_hcall(vcpu);
836 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
837 
838 		if (rc == -ENOENT)
839 			return RESUME_HOST;
840 		else if (rc == 0)
841 			break;
842 
843 		/* Send the error out to userspace via KVM_RUN */
844 		return rc;
845 	case H_LOGICAL_CI_LOAD:
846 		ret = kvmppc_h_logical_ci_load(vcpu);
847 		if (ret == H_TOO_HARD)
848 			return RESUME_HOST;
849 		break;
850 	case H_LOGICAL_CI_STORE:
851 		ret = kvmppc_h_logical_ci_store(vcpu);
852 		if (ret == H_TOO_HARD)
853 			return RESUME_HOST;
854 		break;
855 	case H_SET_MODE:
856 		ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
857 					kvmppc_get_gpr(vcpu, 5),
858 					kvmppc_get_gpr(vcpu, 6),
859 					kvmppc_get_gpr(vcpu, 7));
860 		if (ret == H_TOO_HARD)
861 			return RESUME_HOST;
862 		break;
863 	case H_XIRR:
864 	case H_CPPR:
865 	case H_EOI:
866 	case H_IPI:
867 	case H_IPOLL:
868 	case H_XIRR_X:
869 		if (kvmppc_xics_enabled(vcpu)) {
870 			if (xive_enabled()) {
871 				ret = H_NOT_AVAILABLE;
872 				return RESUME_GUEST;
873 			}
874 			ret = kvmppc_xics_hcall(vcpu, req);
875 			break;
876 		}
877 		return RESUME_HOST;
878 	case H_PUT_TCE:
879 		ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
880 						kvmppc_get_gpr(vcpu, 5),
881 						kvmppc_get_gpr(vcpu, 6));
882 		if (ret == H_TOO_HARD)
883 			return RESUME_HOST;
884 		break;
885 	case H_PUT_TCE_INDIRECT:
886 		ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
887 						kvmppc_get_gpr(vcpu, 5),
888 						kvmppc_get_gpr(vcpu, 6),
889 						kvmppc_get_gpr(vcpu, 7));
890 		if (ret == H_TOO_HARD)
891 			return RESUME_HOST;
892 		break;
893 	case H_STUFF_TCE:
894 		ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
895 						kvmppc_get_gpr(vcpu, 5),
896 						kvmppc_get_gpr(vcpu, 6),
897 						kvmppc_get_gpr(vcpu, 7));
898 		if (ret == H_TOO_HARD)
899 			return RESUME_HOST;
900 		break;
901 	default:
902 		return RESUME_HOST;
903 	}
904 	kvmppc_set_gpr(vcpu, 3, ret);
905 	vcpu->arch.hcall_needed = 0;
906 	return RESUME_GUEST;
907 }
908 
909 static int kvmppc_hcall_impl_hv(unsigned long cmd)
910 {
911 	switch (cmd) {
912 	case H_CEDE:
913 	case H_PROD:
914 	case H_CONFER:
915 	case H_REGISTER_VPA:
916 	case H_SET_MODE:
917 	case H_LOGICAL_CI_LOAD:
918 	case H_LOGICAL_CI_STORE:
919 #ifdef CONFIG_KVM_XICS
920 	case H_XIRR:
921 	case H_CPPR:
922 	case H_EOI:
923 	case H_IPI:
924 	case H_IPOLL:
925 	case H_XIRR_X:
926 #endif
927 		return 1;
928 	}
929 
930 	/* See if it's in the real-mode table */
931 	return kvmppc_hcall_impl_hv_realmode(cmd);
932 }
933 
934 static int kvmppc_emulate_debug_inst(struct kvm_run *run,
935 					struct kvm_vcpu *vcpu)
936 {
937 	u32 last_inst;
938 
939 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
940 					EMULATE_DONE) {
941 		/*
942 		 * Fetch failed, so return to guest and
943 		 * try executing it again.
944 		 */
945 		return RESUME_GUEST;
946 	}
947 
948 	if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
949 		run->exit_reason = KVM_EXIT_DEBUG;
950 		run->debug.arch.address = kvmppc_get_pc(vcpu);
951 		return RESUME_HOST;
952 	} else {
953 		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
954 		return RESUME_GUEST;
955 	}
956 }
957 
958 static void do_nothing(void *x)
959 {
960 }
961 
962 static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
963 {
964 	int thr, cpu, pcpu, nthreads;
965 	struct kvm_vcpu *v;
966 	unsigned long dpdes;
967 
968 	nthreads = vcpu->kvm->arch.emul_smt_mode;
969 	dpdes = 0;
970 	cpu = vcpu->vcpu_id & ~(nthreads - 1);
971 	for (thr = 0; thr < nthreads; ++thr, ++cpu) {
972 		v = kvmppc_find_vcpu(vcpu->kvm, cpu);
973 		if (!v)
974 			continue;
975 		/*
976 		 * If the vcpu is currently running on a physical cpu thread,
977 		 * interrupt it in order to pull it out of the guest briefly,
978 		 * which will update its vcore->dpdes value.
979 		 */
980 		pcpu = READ_ONCE(v->cpu);
981 		if (pcpu >= 0)
982 			smp_call_function_single(pcpu, do_nothing, NULL, 1);
983 		if (kvmppc_doorbell_pending(v))
984 			dpdes |= 1 << thr;
985 	}
986 	return dpdes;
987 }
988 
989 /*
990  * On POWER9, emulate doorbell-related instructions in order to
991  * give the guest the illusion of running on a multi-threaded core.
992  * The instructions emulated are msgsndp, msgclrp, mfspr TIR,
993  * and mfspr DPDES.
994  */
995 static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
996 {
997 	u32 inst, rb, thr;
998 	unsigned long arg;
999 	struct kvm *kvm = vcpu->kvm;
1000 	struct kvm_vcpu *tvcpu;
1001 
1002 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
1003 		return EMULATE_FAIL;
1004 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1005 		return RESUME_GUEST;
1006 	if (get_op(inst) != 31)
1007 		return EMULATE_FAIL;
1008 	rb = get_rb(inst);
1009 	thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1010 	switch (get_xop(inst)) {
1011 	case OP_31_XOP_MSGSNDP:
1012 		arg = kvmppc_get_gpr(vcpu, rb);
1013 		if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1014 			break;
1015 		arg &= 0x3f;
1016 		if (arg >= kvm->arch.emul_smt_mode)
1017 			break;
1018 		tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1019 		if (!tvcpu)
1020 			break;
1021 		if (!tvcpu->arch.doorbell_request) {
1022 			tvcpu->arch.doorbell_request = 1;
1023 			kvmppc_fast_vcpu_kick_hv(tvcpu);
1024 		}
1025 		break;
1026 	case OP_31_XOP_MSGCLRP:
1027 		arg = kvmppc_get_gpr(vcpu, rb);
1028 		if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1029 			break;
1030 		vcpu->arch.vcore->dpdes = 0;
1031 		vcpu->arch.doorbell_request = 0;
1032 		break;
1033 	case OP_31_XOP_MFSPR:
1034 		switch (get_sprn(inst)) {
1035 		case SPRN_TIR:
1036 			arg = thr;
1037 			break;
1038 		case SPRN_DPDES:
1039 			arg = kvmppc_read_dpdes(vcpu);
1040 			break;
1041 		default:
1042 			return EMULATE_FAIL;
1043 		}
1044 		kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1045 		break;
1046 	default:
1047 		return EMULATE_FAIL;
1048 	}
1049 	kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1050 	return RESUME_GUEST;
1051 }
1052 
1053 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
1054 				 struct task_struct *tsk)
1055 {
1056 	int r = RESUME_HOST;
1057 
1058 	vcpu->stat.sum_exits++;
1059 
1060 	/*
1061 	 * This can happen if an interrupt occurs in the last stages
1062 	 * of guest entry or the first stages of guest exit (i.e. after
1063 	 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1064 	 * and before setting it to KVM_GUEST_MODE_HOST_HV).
1065 	 * That can happen due to a bug, or due to a machine check
1066 	 * occurring at just the wrong time.
1067 	 */
1068 	if (vcpu->arch.shregs.msr & MSR_HV) {
1069 		printk(KERN_EMERG "KVM trap in HV mode!\n");
1070 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1071 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
1072 			vcpu->arch.shregs.msr);
1073 		kvmppc_dump_regs(vcpu);
1074 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1075 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1076 		return RESUME_HOST;
1077 	}
1078 	run->exit_reason = KVM_EXIT_UNKNOWN;
1079 	run->ready_for_interrupt_injection = 1;
1080 	switch (vcpu->arch.trap) {
1081 	/* We're good on these - the host merely wanted to get our attention */
1082 	case BOOK3S_INTERRUPT_HV_DECREMENTER:
1083 		vcpu->stat.dec_exits++;
1084 		r = RESUME_GUEST;
1085 		break;
1086 	case BOOK3S_INTERRUPT_EXTERNAL:
1087 	case BOOK3S_INTERRUPT_H_DOORBELL:
1088 	case BOOK3S_INTERRUPT_H_VIRT:
1089 		vcpu->stat.ext_intr_exits++;
1090 		r = RESUME_GUEST;
1091 		break;
1092 	/* HMI is hypervisor interrupt and host has handled it. Resume guest.*/
1093 	case BOOK3S_INTERRUPT_HMI:
1094 	case BOOK3S_INTERRUPT_PERFMON:
1095 		r = RESUME_GUEST;
1096 		break;
1097 	case BOOK3S_INTERRUPT_MACHINE_CHECK:
1098 		/* Exit to guest with KVM_EXIT_NMI as exit reason */
1099 		run->exit_reason = KVM_EXIT_NMI;
1100 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1101 		/* Clear out the old NMI status from run->flags */
1102 		run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1103 		/* Now set the NMI status */
1104 		if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1105 			run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1106 		else
1107 			run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1108 
1109 		r = RESUME_HOST;
1110 		/* Print the MCE event to host console. */
1111 		machine_check_print_event_info(&vcpu->arch.mce_evt, false);
1112 		break;
1113 	case BOOK3S_INTERRUPT_PROGRAM:
1114 	{
1115 		ulong flags;
1116 		/*
1117 		 * Normally program interrupts are delivered directly
1118 		 * to the guest by the hardware, but we can get here
1119 		 * as a result of a hypervisor emulation interrupt
1120 		 * (e40) getting turned into a 700 by BML RTAS.
1121 		 */
1122 		flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1123 		kvmppc_core_queue_program(vcpu, flags);
1124 		r = RESUME_GUEST;
1125 		break;
1126 	}
1127 	case BOOK3S_INTERRUPT_SYSCALL:
1128 	{
1129 		/* hcall - punt to userspace */
1130 		int i;
1131 
1132 		/* hypercall with MSR_PR has already been handled in rmode,
1133 		 * and never reaches here.
1134 		 */
1135 
1136 		run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1137 		for (i = 0; i < 9; ++i)
1138 			run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1139 		run->exit_reason = KVM_EXIT_PAPR_HCALL;
1140 		vcpu->arch.hcall_needed = 1;
1141 		r = RESUME_HOST;
1142 		break;
1143 	}
1144 	/*
1145 	 * We get these next two if the guest accesses a page which it thinks
1146 	 * it has mapped but which is not actually present, either because
1147 	 * it is for an emulated I/O device or because the corresonding
1148 	 * host page has been paged out.  Any other HDSI/HISI interrupts
1149 	 * have been handled already.
1150 	 */
1151 	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1152 		r = RESUME_PAGE_FAULT;
1153 		break;
1154 	case BOOK3S_INTERRUPT_H_INST_STORAGE:
1155 		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1156 		vcpu->arch.fault_dsisr = 0;
1157 		r = RESUME_PAGE_FAULT;
1158 		break;
1159 	/*
1160 	 * This occurs if the guest executes an illegal instruction.
1161 	 * If the guest debug is disabled, generate a program interrupt
1162 	 * to the guest. If guest debug is enabled, we need to check
1163 	 * whether the instruction is a software breakpoint instruction.
1164 	 * Accordingly return to Guest or Host.
1165 	 */
1166 	case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1167 		if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1168 			vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1169 				swab32(vcpu->arch.emul_inst) :
1170 				vcpu->arch.emul_inst;
1171 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1172 			r = kvmppc_emulate_debug_inst(run, vcpu);
1173 		} else {
1174 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1175 			r = RESUME_GUEST;
1176 		}
1177 		break;
1178 	/*
1179 	 * This occurs if the guest (kernel or userspace), does something that
1180 	 * is prohibited by HFSCR.
1181 	 * On POWER9, this could be a doorbell instruction that we need
1182 	 * to emulate.
1183 	 * Otherwise, we just generate a program interrupt to the guest.
1184 	 */
1185 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1186 		r = EMULATE_FAIL;
1187 		if ((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG)
1188 			r = kvmppc_emulate_doorbell_instr(vcpu);
1189 		if (r == EMULATE_FAIL) {
1190 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1191 			r = RESUME_GUEST;
1192 		}
1193 		break;
1194 	case BOOK3S_INTERRUPT_HV_RM_HARD:
1195 		r = RESUME_PASSTHROUGH;
1196 		break;
1197 	default:
1198 		kvmppc_dump_regs(vcpu);
1199 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1200 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
1201 			vcpu->arch.shregs.msr);
1202 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1203 		r = RESUME_HOST;
1204 		break;
1205 	}
1206 
1207 	return r;
1208 }
1209 
1210 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1211 					    struct kvm_sregs *sregs)
1212 {
1213 	int i;
1214 
1215 	memset(sregs, 0, sizeof(struct kvm_sregs));
1216 	sregs->pvr = vcpu->arch.pvr;
1217 	for (i = 0; i < vcpu->arch.slb_max; i++) {
1218 		sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1219 		sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1220 	}
1221 
1222 	return 0;
1223 }
1224 
1225 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1226 					    struct kvm_sregs *sregs)
1227 {
1228 	int i, j;
1229 
1230 	/* Only accept the same PVR as the host's, since we can't spoof it */
1231 	if (sregs->pvr != vcpu->arch.pvr)
1232 		return -EINVAL;
1233 
1234 	j = 0;
1235 	for (i = 0; i < vcpu->arch.slb_nr; i++) {
1236 		if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1237 			vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1238 			vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1239 			++j;
1240 		}
1241 	}
1242 	vcpu->arch.slb_max = j;
1243 
1244 	return 0;
1245 }
1246 
1247 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1248 		bool preserve_top32)
1249 {
1250 	struct kvm *kvm = vcpu->kvm;
1251 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
1252 	u64 mask;
1253 
1254 	mutex_lock(&kvm->lock);
1255 	spin_lock(&vc->lock);
1256 	/*
1257 	 * If ILE (interrupt little-endian) has changed, update the
1258 	 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
1259 	 */
1260 	if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1261 		struct kvm_vcpu *vcpu;
1262 		int i;
1263 
1264 		kvm_for_each_vcpu(i, vcpu, kvm) {
1265 			if (vcpu->arch.vcore != vc)
1266 				continue;
1267 			if (new_lpcr & LPCR_ILE)
1268 				vcpu->arch.intr_msr |= MSR_LE;
1269 			else
1270 				vcpu->arch.intr_msr &= ~MSR_LE;
1271 		}
1272 	}
1273 
1274 	/*
1275 	 * Userspace can only modify DPFD (default prefetch depth),
1276 	 * ILE (interrupt little-endian) and TC (translation control).
1277 	 * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.).
1278 	 */
1279 	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
1280 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
1281 		mask |= LPCR_AIL;
1282 	/*
1283 	 * On POWER9, allow userspace to enable large decrementer for the
1284 	 * guest, whether or not the host has it enabled.
1285 	 */
1286 	if (cpu_has_feature(CPU_FTR_ARCH_300))
1287 		mask |= LPCR_LD;
1288 
1289 	/* Broken 32-bit version of LPCR must not clear top bits */
1290 	if (preserve_top32)
1291 		mask &= 0xFFFFFFFF;
1292 	vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
1293 	spin_unlock(&vc->lock);
1294 	mutex_unlock(&kvm->lock);
1295 }
1296 
1297 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1298 				 union kvmppc_one_reg *val)
1299 {
1300 	int r = 0;
1301 	long int i;
1302 
1303 	switch (id) {
1304 	case KVM_REG_PPC_DEBUG_INST:
1305 		*val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1306 		break;
1307 	case KVM_REG_PPC_HIOR:
1308 		*val = get_reg_val(id, 0);
1309 		break;
1310 	case KVM_REG_PPC_DABR:
1311 		*val = get_reg_val(id, vcpu->arch.dabr);
1312 		break;
1313 	case KVM_REG_PPC_DABRX:
1314 		*val = get_reg_val(id, vcpu->arch.dabrx);
1315 		break;
1316 	case KVM_REG_PPC_DSCR:
1317 		*val = get_reg_val(id, vcpu->arch.dscr);
1318 		break;
1319 	case KVM_REG_PPC_PURR:
1320 		*val = get_reg_val(id, vcpu->arch.purr);
1321 		break;
1322 	case KVM_REG_PPC_SPURR:
1323 		*val = get_reg_val(id, vcpu->arch.spurr);
1324 		break;
1325 	case KVM_REG_PPC_AMR:
1326 		*val = get_reg_val(id, vcpu->arch.amr);
1327 		break;
1328 	case KVM_REG_PPC_UAMOR:
1329 		*val = get_reg_val(id, vcpu->arch.uamor);
1330 		break;
1331 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1332 		i = id - KVM_REG_PPC_MMCR0;
1333 		*val = get_reg_val(id, vcpu->arch.mmcr[i]);
1334 		break;
1335 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1336 		i = id - KVM_REG_PPC_PMC1;
1337 		*val = get_reg_val(id, vcpu->arch.pmc[i]);
1338 		break;
1339 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1340 		i = id - KVM_REG_PPC_SPMC1;
1341 		*val = get_reg_val(id, vcpu->arch.spmc[i]);
1342 		break;
1343 	case KVM_REG_PPC_SIAR:
1344 		*val = get_reg_val(id, vcpu->arch.siar);
1345 		break;
1346 	case KVM_REG_PPC_SDAR:
1347 		*val = get_reg_val(id, vcpu->arch.sdar);
1348 		break;
1349 	case KVM_REG_PPC_SIER:
1350 		*val = get_reg_val(id, vcpu->arch.sier);
1351 		break;
1352 	case KVM_REG_PPC_IAMR:
1353 		*val = get_reg_val(id, vcpu->arch.iamr);
1354 		break;
1355 	case KVM_REG_PPC_PSPB:
1356 		*val = get_reg_val(id, vcpu->arch.pspb);
1357 		break;
1358 	case KVM_REG_PPC_DPDES:
1359 		*val = get_reg_val(id, vcpu->arch.vcore->dpdes);
1360 		break;
1361 	case KVM_REG_PPC_VTB:
1362 		*val = get_reg_val(id, vcpu->arch.vcore->vtb);
1363 		break;
1364 	case KVM_REG_PPC_DAWR:
1365 		*val = get_reg_val(id, vcpu->arch.dawr);
1366 		break;
1367 	case KVM_REG_PPC_DAWRX:
1368 		*val = get_reg_val(id, vcpu->arch.dawrx);
1369 		break;
1370 	case KVM_REG_PPC_CIABR:
1371 		*val = get_reg_val(id, vcpu->arch.ciabr);
1372 		break;
1373 	case KVM_REG_PPC_CSIGR:
1374 		*val = get_reg_val(id, vcpu->arch.csigr);
1375 		break;
1376 	case KVM_REG_PPC_TACR:
1377 		*val = get_reg_val(id, vcpu->arch.tacr);
1378 		break;
1379 	case KVM_REG_PPC_TCSCR:
1380 		*val = get_reg_val(id, vcpu->arch.tcscr);
1381 		break;
1382 	case KVM_REG_PPC_PID:
1383 		*val = get_reg_val(id, vcpu->arch.pid);
1384 		break;
1385 	case KVM_REG_PPC_ACOP:
1386 		*val = get_reg_val(id, vcpu->arch.acop);
1387 		break;
1388 	case KVM_REG_PPC_WORT:
1389 		*val = get_reg_val(id, vcpu->arch.wort);
1390 		break;
1391 	case KVM_REG_PPC_TIDR:
1392 		*val = get_reg_val(id, vcpu->arch.tid);
1393 		break;
1394 	case KVM_REG_PPC_PSSCR:
1395 		*val = get_reg_val(id, vcpu->arch.psscr);
1396 		break;
1397 	case KVM_REG_PPC_VPA_ADDR:
1398 		spin_lock(&vcpu->arch.vpa_update_lock);
1399 		*val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1400 		spin_unlock(&vcpu->arch.vpa_update_lock);
1401 		break;
1402 	case KVM_REG_PPC_VPA_SLB:
1403 		spin_lock(&vcpu->arch.vpa_update_lock);
1404 		val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1405 		val->vpaval.length = vcpu->arch.slb_shadow.len;
1406 		spin_unlock(&vcpu->arch.vpa_update_lock);
1407 		break;
1408 	case KVM_REG_PPC_VPA_DTL:
1409 		spin_lock(&vcpu->arch.vpa_update_lock);
1410 		val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1411 		val->vpaval.length = vcpu->arch.dtl.len;
1412 		spin_unlock(&vcpu->arch.vpa_update_lock);
1413 		break;
1414 	case KVM_REG_PPC_TB_OFFSET:
1415 		*val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1416 		break;
1417 	case KVM_REG_PPC_LPCR:
1418 	case KVM_REG_PPC_LPCR_64:
1419 		*val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1420 		break;
1421 	case KVM_REG_PPC_PPR:
1422 		*val = get_reg_val(id, vcpu->arch.ppr);
1423 		break;
1424 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1425 	case KVM_REG_PPC_TFHAR:
1426 		*val = get_reg_val(id, vcpu->arch.tfhar);
1427 		break;
1428 	case KVM_REG_PPC_TFIAR:
1429 		*val = get_reg_val(id, vcpu->arch.tfiar);
1430 		break;
1431 	case KVM_REG_PPC_TEXASR:
1432 		*val = get_reg_val(id, vcpu->arch.texasr);
1433 		break;
1434 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1435 		i = id - KVM_REG_PPC_TM_GPR0;
1436 		*val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1437 		break;
1438 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1439 	{
1440 		int j;
1441 		i = id - KVM_REG_PPC_TM_VSR0;
1442 		if (i < 32)
1443 			for (j = 0; j < TS_FPRWIDTH; j++)
1444 				val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1445 		else {
1446 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
1447 				val->vval = vcpu->arch.vr_tm.vr[i-32];
1448 			else
1449 				r = -ENXIO;
1450 		}
1451 		break;
1452 	}
1453 	case KVM_REG_PPC_TM_CR:
1454 		*val = get_reg_val(id, vcpu->arch.cr_tm);
1455 		break;
1456 	case KVM_REG_PPC_TM_XER:
1457 		*val = get_reg_val(id, vcpu->arch.xer_tm);
1458 		break;
1459 	case KVM_REG_PPC_TM_LR:
1460 		*val = get_reg_val(id, vcpu->arch.lr_tm);
1461 		break;
1462 	case KVM_REG_PPC_TM_CTR:
1463 		*val = get_reg_val(id, vcpu->arch.ctr_tm);
1464 		break;
1465 	case KVM_REG_PPC_TM_FPSCR:
1466 		*val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1467 		break;
1468 	case KVM_REG_PPC_TM_AMR:
1469 		*val = get_reg_val(id, vcpu->arch.amr_tm);
1470 		break;
1471 	case KVM_REG_PPC_TM_PPR:
1472 		*val = get_reg_val(id, vcpu->arch.ppr_tm);
1473 		break;
1474 	case KVM_REG_PPC_TM_VRSAVE:
1475 		*val = get_reg_val(id, vcpu->arch.vrsave_tm);
1476 		break;
1477 	case KVM_REG_PPC_TM_VSCR:
1478 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
1479 			*val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1480 		else
1481 			r = -ENXIO;
1482 		break;
1483 	case KVM_REG_PPC_TM_DSCR:
1484 		*val = get_reg_val(id, vcpu->arch.dscr_tm);
1485 		break;
1486 	case KVM_REG_PPC_TM_TAR:
1487 		*val = get_reg_val(id, vcpu->arch.tar_tm);
1488 		break;
1489 #endif
1490 	case KVM_REG_PPC_ARCH_COMPAT:
1491 		*val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1492 		break;
1493 	default:
1494 		r = -EINVAL;
1495 		break;
1496 	}
1497 
1498 	return r;
1499 }
1500 
1501 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1502 				 union kvmppc_one_reg *val)
1503 {
1504 	int r = 0;
1505 	long int i;
1506 	unsigned long addr, len;
1507 
1508 	switch (id) {
1509 	case KVM_REG_PPC_HIOR:
1510 		/* Only allow this to be set to zero */
1511 		if (set_reg_val(id, *val))
1512 			r = -EINVAL;
1513 		break;
1514 	case KVM_REG_PPC_DABR:
1515 		vcpu->arch.dabr = set_reg_val(id, *val);
1516 		break;
1517 	case KVM_REG_PPC_DABRX:
1518 		vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1519 		break;
1520 	case KVM_REG_PPC_DSCR:
1521 		vcpu->arch.dscr = set_reg_val(id, *val);
1522 		break;
1523 	case KVM_REG_PPC_PURR:
1524 		vcpu->arch.purr = set_reg_val(id, *val);
1525 		break;
1526 	case KVM_REG_PPC_SPURR:
1527 		vcpu->arch.spurr = set_reg_val(id, *val);
1528 		break;
1529 	case KVM_REG_PPC_AMR:
1530 		vcpu->arch.amr = set_reg_val(id, *val);
1531 		break;
1532 	case KVM_REG_PPC_UAMOR:
1533 		vcpu->arch.uamor = set_reg_val(id, *val);
1534 		break;
1535 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1536 		i = id - KVM_REG_PPC_MMCR0;
1537 		vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1538 		break;
1539 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1540 		i = id - KVM_REG_PPC_PMC1;
1541 		vcpu->arch.pmc[i] = set_reg_val(id, *val);
1542 		break;
1543 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1544 		i = id - KVM_REG_PPC_SPMC1;
1545 		vcpu->arch.spmc[i] = set_reg_val(id, *val);
1546 		break;
1547 	case KVM_REG_PPC_SIAR:
1548 		vcpu->arch.siar = set_reg_val(id, *val);
1549 		break;
1550 	case KVM_REG_PPC_SDAR:
1551 		vcpu->arch.sdar = set_reg_val(id, *val);
1552 		break;
1553 	case KVM_REG_PPC_SIER:
1554 		vcpu->arch.sier = set_reg_val(id, *val);
1555 		break;
1556 	case KVM_REG_PPC_IAMR:
1557 		vcpu->arch.iamr = set_reg_val(id, *val);
1558 		break;
1559 	case KVM_REG_PPC_PSPB:
1560 		vcpu->arch.pspb = set_reg_val(id, *val);
1561 		break;
1562 	case KVM_REG_PPC_DPDES:
1563 		vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1564 		break;
1565 	case KVM_REG_PPC_VTB:
1566 		vcpu->arch.vcore->vtb = set_reg_val(id, *val);
1567 		break;
1568 	case KVM_REG_PPC_DAWR:
1569 		vcpu->arch.dawr = set_reg_val(id, *val);
1570 		break;
1571 	case KVM_REG_PPC_DAWRX:
1572 		vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1573 		break;
1574 	case KVM_REG_PPC_CIABR:
1575 		vcpu->arch.ciabr = set_reg_val(id, *val);
1576 		/* Don't allow setting breakpoints in hypervisor code */
1577 		if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1578 			vcpu->arch.ciabr &= ~CIABR_PRIV;	/* disable */
1579 		break;
1580 	case KVM_REG_PPC_CSIGR:
1581 		vcpu->arch.csigr = set_reg_val(id, *val);
1582 		break;
1583 	case KVM_REG_PPC_TACR:
1584 		vcpu->arch.tacr = set_reg_val(id, *val);
1585 		break;
1586 	case KVM_REG_PPC_TCSCR:
1587 		vcpu->arch.tcscr = set_reg_val(id, *val);
1588 		break;
1589 	case KVM_REG_PPC_PID:
1590 		vcpu->arch.pid = set_reg_val(id, *val);
1591 		break;
1592 	case KVM_REG_PPC_ACOP:
1593 		vcpu->arch.acop = set_reg_val(id, *val);
1594 		break;
1595 	case KVM_REG_PPC_WORT:
1596 		vcpu->arch.wort = set_reg_val(id, *val);
1597 		break;
1598 	case KVM_REG_PPC_TIDR:
1599 		vcpu->arch.tid = set_reg_val(id, *val);
1600 		break;
1601 	case KVM_REG_PPC_PSSCR:
1602 		vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
1603 		break;
1604 	case KVM_REG_PPC_VPA_ADDR:
1605 		addr = set_reg_val(id, *val);
1606 		r = -EINVAL;
1607 		if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1608 			      vcpu->arch.dtl.next_gpa))
1609 			break;
1610 		r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1611 		break;
1612 	case KVM_REG_PPC_VPA_SLB:
1613 		addr = val->vpaval.addr;
1614 		len = val->vpaval.length;
1615 		r = -EINVAL;
1616 		if (addr && !vcpu->arch.vpa.next_gpa)
1617 			break;
1618 		r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1619 		break;
1620 	case KVM_REG_PPC_VPA_DTL:
1621 		addr = val->vpaval.addr;
1622 		len = val->vpaval.length;
1623 		r = -EINVAL;
1624 		if (addr && (len < sizeof(struct dtl_entry) ||
1625 			     !vcpu->arch.vpa.next_gpa))
1626 			break;
1627 		len -= len % sizeof(struct dtl_entry);
1628 		r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1629 		break;
1630 	case KVM_REG_PPC_TB_OFFSET:
1631 		/*
1632 		 * POWER9 DD1 has an erratum where writing TBU40 causes
1633 		 * the timebase to lose ticks.  So we don't let the
1634 		 * timebase offset be changed on P9 DD1.  (It is
1635 		 * initialized to zero.)
1636 		 */
1637 		if (cpu_has_feature(CPU_FTR_POWER9_DD1))
1638 			break;
1639 		/* round up to multiple of 2^24 */
1640 		vcpu->arch.vcore->tb_offset =
1641 			ALIGN(set_reg_val(id, *val), 1UL << 24);
1642 		break;
1643 	case KVM_REG_PPC_LPCR:
1644 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1645 		break;
1646 	case KVM_REG_PPC_LPCR_64:
1647 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1648 		break;
1649 	case KVM_REG_PPC_PPR:
1650 		vcpu->arch.ppr = set_reg_val(id, *val);
1651 		break;
1652 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1653 	case KVM_REG_PPC_TFHAR:
1654 		vcpu->arch.tfhar = set_reg_val(id, *val);
1655 		break;
1656 	case KVM_REG_PPC_TFIAR:
1657 		vcpu->arch.tfiar = set_reg_val(id, *val);
1658 		break;
1659 	case KVM_REG_PPC_TEXASR:
1660 		vcpu->arch.texasr = set_reg_val(id, *val);
1661 		break;
1662 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1663 		i = id - KVM_REG_PPC_TM_GPR0;
1664 		vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1665 		break;
1666 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1667 	{
1668 		int j;
1669 		i = id - KVM_REG_PPC_TM_VSR0;
1670 		if (i < 32)
1671 			for (j = 0; j < TS_FPRWIDTH; j++)
1672 				vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
1673 		else
1674 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
1675 				vcpu->arch.vr_tm.vr[i-32] = val->vval;
1676 			else
1677 				r = -ENXIO;
1678 		break;
1679 	}
1680 	case KVM_REG_PPC_TM_CR:
1681 		vcpu->arch.cr_tm = set_reg_val(id, *val);
1682 		break;
1683 	case KVM_REG_PPC_TM_XER:
1684 		vcpu->arch.xer_tm = set_reg_val(id, *val);
1685 		break;
1686 	case KVM_REG_PPC_TM_LR:
1687 		vcpu->arch.lr_tm = set_reg_val(id, *val);
1688 		break;
1689 	case KVM_REG_PPC_TM_CTR:
1690 		vcpu->arch.ctr_tm = set_reg_val(id, *val);
1691 		break;
1692 	case KVM_REG_PPC_TM_FPSCR:
1693 		vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
1694 		break;
1695 	case KVM_REG_PPC_TM_AMR:
1696 		vcpu->arch.amr_tm = set_reg_val(id, *val);
1697 		break;
1698 	case KVM_REG_PPC_TM_PPR:
1699 		vcpu->arch.ppr_tm = set_reg_val(id, *val);
1700 		break;
1701 	case KVM_REG_PPC_TM_VRSAVE:
1702 		vcpu->arch.vrsave_tm = set_reg_val(id, *val);
1703 		break;
1704 	case KVM_REG_PPC_TM_VSCR:
1705 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
1706 			vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
1707 		else
1708 			r = - ENXIO;
1709 		break;
1710 	case KVM_REG_PPC_TM_DSCR:
1711 		vcpu->arch.dscr_tm = set_reg_val(id, *val);
1712 		break;
1713 	case KVM_REG_PPC_TM_TAR:
1714 		vcpu->arch.tar_tm = set_reg_val(id, *val);
1715 		break;
1716 #endif
1717 	case KVM_REG_PPC_ARCH_COMPAT:
1718 		r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
1719 		break;
1720 	default:
1721 		r = -EINVAL;
1722 		break;
1723 	}
1724 
1725 	return r;
1726 }
1727 
1728 /*
1729  * On POWER9, threads are independent and can be in different partitions.
1730  * Therefore we consider each thread to be a subcore.
1731  * There is a restriction that all threads have to be in the same
1732  * MMU mode (radix or HPT), unfortunately, but since we only support
1733  * HPT guests on a HPT host so far, that isn't an impediment yet.
1734  */
1735 static int threads_per_vcore(void)
1736 {
1737 	if (cpu_has_feature(CPU_FTR_ARCH_300))
1738 		return 1;
1739 	return threads_per_subcore;
1740 }
1741 
1742 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
1743 {
1744 	struct kvmppc_vcore *vcore;
1745 
1746 	vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
1747 
1748 	if (vcore == NULL)
1749 		return NULL;
1750 
1751 	spin_lock_init(&vcore->lock);
1752 	spin_lock_init(&vcore->stoltb_lock);
1753 	init_swait_queue_head(&vcore->wq);
1754 	vcore->preempt_tb = TB_NIL;
1755 	vcore->lpcr = kvm->arch.lpcr;
1756 	vcore->first_vcpuid = core * kvm->arch.smt_mode;
1757 	vcore->kvm = kvm;
1758 	INIT_LIST_HEAD(&vcore->preempt_list);
1759 
1760 	return vcore;
1761 }
1762 
1763 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1764 static struct debugfs_timings_element {
1765 	const char *name;
1766 	size_t offset;
1767 } timings[] = {
1768 	{"rm_entry",	offsetof(struct kvm_vcpu, arch.rm_entry)},
1769 	{"rm_intr",	offsetof(struct kvm_vcpu, arch.rm_intr)},
1770 	{"rm_exit",	offsetof(struct kvm_vcpu, arch.rm_exit)},
1771 	{"guest",	offsetof(struct kvm_vcpu, arch.guest_time)},
1772 	{"cede",	offsetof(struct kvm_vcpu, arch.cede_time)},
1773 };
1774 
1775 #define N_TIMINGS	(sizeof(timings) / sizeof(timings[0]))
1776 
1777 struct debugfs_timings_state {
1778 	struct kvm_vcpu	*vcpu;
1779 	unsigned int	buflen;
1780 	char		buf[N_TIMINGS * 100];
1781 };
1782 
1783 static int debugfs_timings_open(struct inode *inode, struct file *file)
1784 {
1785 	struct kvm_vcpu *vcpu = inode->i_private;
1786 	struct debugfs_timings_state *p;
1787 
1788 	p = kzalloc(sizeof(*p), GFP_KERNEL);
1789 	if (!p)
1790 		return -ENOMEM;
1791 
1792 	kvm_get_kvm(vcpu->kvm);
1793 	p->vcpu = vcpu;
1794 	file->private_data = p;
1795 
1796 	return nonseekable_open(inode, file);
1797 }
1798 
1799 static int debugfs_timings_release(struct inode *inode, struct file *file)
1800 {
1801 	struct debugfs_timings_state *p = file->private_data;
1802 
1803 	kvm_put_kvm(p->vcpu->kvm);
1804 	kfree(p);
1805 	return 0;
1806 }
1807 
1808 static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
1809 				    size_t len, loff_t *ppos)
1810 {
1811 	struct debugfs_timings_state *p = file->private_data;
1812 	struct kvm_vcpu *vcpu = p->vcpu;
1813 	char *s, *buf_end;
1814 	struct kvmhv_tb_accumulator tb;
1815 	u64 count;
1816 	loff_t pos;
1817 	ssize_t n;
1818 	int i, loops;
1819 	bool ok;
1820 
1821 	if (!p->buflen) {
1822 		s = p->buf;
1823 		buf_end = s + sizeof(p->buf);
1824 		for (i = 0; i < N_TIMINGS; ++i) {
1825 			struct kvmhv_tb_accumulator *acc;
1826 
1827 			acc = (struct kvmhv_tb_accumulator *)
1828 				((unsigned long)vcpu + timings[i].offset);
1829 			ok = false;
1830 			for (loops = 0; loops < 1000; ++loops) {
1831 				count = acc->seqcount;
1832 				if (!(count & 1)) {
1833 					smp_rmb();
1834 					tb = *acc;
1835 					smp_rmb();
1836 					if (count == acc->seqcount) {
1837 						ok = true;
1838 						break;
1839 					}
1840 				}
1841 				udelay(1);
1842 			}
1843 			if (!ok)
1844 				snprintf(s, buf_end - s, "%s: stuck\n",
1845 					timings[i].name);
1846 			else
1847 				snprintf(s, buf_end - s,
1848 					"%s: %llu %llu %llu %llu\n",
1849 					timings[i].name, count / 2,
1850 					tb_to_ns(tb.tb_total),
1851 					tb_to_ns(tb.tb_min),
1852 					tb_to_ns(tb.tb_max));
1853 			s += strlen(s);
1854 		}
1855 		p->buflen = s - p->buf;
1856 	}
1857 
1858 	pos = *ppos;
1859 	if (pos >= p->buflen)
1860 		return 0;
1861 	if (len > p->buflen - pos)
1862 		len = p->buflen - pos;
1863 	n = copy_to_user(buf, p->buf + pos, len);
1864 	if (n) {
1865 		if (n == len)
1866 			return -EFAULT;
1867 		len -= n;
1868 	}
1869 	*ppos = pos + len;
1870 	return len;
1871 }
1872 
1873 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
1874 				     size_t len, loff_t *ppos)
1875 {
1876 	return -EACCES;
1877 }
1878 
1879 static const struct file_operations debugfs_timings_ops = {
1880 	.owner	 = THIS_MODULE,
1881 	.open	 = debugfs_timings_open,
1882 	.release = debugfs_timings_release,
1883 	.read	 = debugfs_timings_read,
1884 	.write	 = debugfs_timings_write,
1885 	.llseek	 = generic_file_llseek,
1886 };
1887 
1888 /* Create a debugfs directory for the vcpu */
1889 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
1890 {
1891 	char buf[16];
1892 	struct kvm *kvm = vcpu->kvm;
1893 
1894 	snprintf(buf, sizeof(buf), "vcpu%u", id);
1895 	if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
1896 		return;
1897 	vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
1898 	if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir))
1899 		return;
1900 	vcpu->arch.debugfs_timings =
1901 		debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir,
1902 				    vcpu, &debugfs_timings_ops);
1903 }
1904 
1905 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
1906 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
1907 {
1908 }
1909 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
1910 
1911 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
1912 						   unsigned int id)
1913 {
1914 	struct kvm_vcpu *vcpu;
1915 	int err;
1916 	int core;
1917 	struct kvmppc_vcore *vcore;
1918 
1919 	err = -ENOMEM;
1920 	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1921 	if (!vcpu)
1922 		goto out;
1923 
1924 	err = kvm_vcpu_init(vcpu, kvm, id);
1925 	if (err)
1926 		goto free_vcpu;
1927 
1928 	vcpu->arch.shared = &vcpu->arch.shregs;
1929 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1930 	/*
1931 	 * The shared struct is never shared on HV,
1932 	 * so we can always use host endianness
1933 	 */
1934 #ifdef __BIG_ENDIAN__
1935 	vcpu->arch.shared_big_endian = true;
1936 #else
1937 	vcpu->arch.shared_big_endian = false;
1938 #endif
1939 #endif
1940 	vcpu->arch.mmcr[0] = MMCR0_FC;
1941 	vcpu->arch.ctrl = CTRL_RUNLATCH;
1942 	/* default to host PVR, since we can't spoof it */
1943 	kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
1944 	spin_lock_init(&vcpu->arch.vpa_update_lock);
1945 	spin_lock_init(&vcpu->arch.tbacct_lock);
1946 	vcpu->arch.busy_preempt = TB_NIL;
1947 	vcpu->arch.intr_msr = MSR_SF | MSR_ME;
1948 
1949 	/*
1950 	 * Set the default HFSCR for the guest from the host value.
1951 	 * This value is only used on POWER9.
1952 	 * On POWER9 DD1, TM doesn't work, so we make sure to
1953 	 * prevent the guest from using it.
1954 	 * On POWER9, we want to virtualize the doorbell facility, so we
1955 	 * turn off the HFSCR bit, which causes those instructions to trap.
1956 	 */
1957 	vcpu->arch.hfscr = mfspr(SPRN_HFSCR);
1958 	if (!cpu_has_feature(CPU_FTR_TM))
1959 		vcpu->arch.hfscr &= ~HFSCR_TM;
1960 	if (cpu_has_feature(CPU_FTR_ARCH_300))
1961 		vcpu->arch.hfscr &= ~HFSCR_MSGP;
1962 
1963 	kvmppc_mmu_book3s_hv_init(vcpu);
1964 
1965 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
1966 
1967 	init_waitqueue_head(&vcpu->arch.cpu_run);
1968 
1969 	mutex_lock(&kvm->lock);
1970 	vcore = NULL;
1971 	err = -EINVAL;
1972 	core = id / kvm->arch.smt_mode;
1973 	if (core < KVM_MAX_VCORES) {
1974 		vcore = kvm->arch.vcores[core];
1975 		if (!vcore) {
1976 			err = -ENOMEM;
1977 			vcore = kvmppc_vcore_create(kvm, core);
1978 			kvm->arch.vcores[core] = vcore;
1979 			kvm->arch.online_vcores++;
1980 		}
1981 	}
1982 	mutex_unlock(&kvm->lock);
1983 
1984 	if (!vcore)
1985 		goto free_vcpu;
1986 
1987 	spin_lock(&vcore->lock);
1988 	++vcore->num_threads;
1989 	spin_unlock(&vcore->lock);
1990 	vcpu->arch.vcore = vcore;
1991 	vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
1992 	vcpu->arch.thread_cpu = -1;
1993 	vcpu->arch.prev_cpu = -1;
1994 
1995 	vcpu->arch.cpu_type = KVM_CPU_3S_64;
1996 	kvmppc_sanity_check(vcpu);
1997 
1998 	debugfs_vcpu_init(vcpu, id);
1999 
2000 	return vcpu;
2001 
2002 free_vcpu:
2003 	kmem_cache_free(kvm_vcpu_cache, vcpu);
2004 out:
2005 	return ERR_PTR(err);
2006 }
2007 
2008 static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2009 			      unsigned long flags)
2010 {
2011 	int err;
2012 	int esmt = 0;
2013 
2014 	if (flags)
2015 		return -EINVAL;
2016 	if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2017 		return -EINVAL;
2018 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2019 		/*
2020 		 * On POWER8 (or POWER7), the threading mode is "strict",
2021 		 * so we pack smt_mode vcpus per vcore.
2022 		 */
2023 		if (smt_mode > threads_per_subcore)
2024 			return -EINVAL;
2025 	} else {
2026 		/*
2027 		 * On POWER9, the threading mode is "loose",
2028 		 * so each vcpu gets its own vcore.
2029 		 */
2030 		esmt = smt_mode;
2031 		smt_mode = 1;
2032 	}
2033 	mutex_lock(&kvm->lock);
2034 	err = -EBUSY;
2035 	if (!kvm->arch.online_vcores) {
2036 		kvm->arch.smt_mode = smt_mode;
2037 		kvm->arch.emul_smt_mode = esmt;
2038 		err = 0;
2039 	}
2040 	mutex_unlock(&kvm->lock);
2041 
2042 	return err;
2043 }
2044 
2045 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2046 {
2047 	if (vpa->pinned_addr)
2048 		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2049 					vpa->dirty);
2050 }
2051 
2052 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2053 {
2054 	spin_lock(&vcpu->arch.vpa_update_lock);
2055 	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2056 	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2057 	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2058 	spin_unlock(&vcpu->arch.vpa_update_lock);
2059 	kvm_vcpu_uninit(vcpu);
2060 	kmem_cache_free(kvm_vcpu_cache, vcpu);
2061 }
2062 
2063 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2064 {
2065 	/* Indicate we want to get back into the guest */
2066 	return 1;
2067 }
2068 
2069 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2070 {
2071 	unsigned long dec_nsec, now;
2072 
2073 	now = get_tb();
2074 	if (now > vcpu->arch.dec_expires) {
2075 		/* decrementer has already gone negative */
2076 		kvmppc_core_queue_dec(vcpu);
2077 		kvmppc_core_prepare_to_enter(vcpu);
2078 		return;
2079 	}
2080 	dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
2081 		   / tb_ticks_per_sec;
2082 	hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2083 	vcpu->arch.timer_running = 1;
2084 }
2085 
2086 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
2087 {
2088 	vcpu->arch.ceded = 0;
2089 	if (vcpu->arch.timer_running) {
2090 		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2091 		vcpu->arch.timer_running = 0;
2092 	}
2093 }
2094 
2095 extern int __kvmppc_vcore_entry(void);
2096 
2097 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2098 				   struct kvm_vcpu *vcpu)
2099 {
2100 	u64 now;
2101 
2102 	if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2103 		return;
2104 	spin_lock_irq(&vcpu->arch.tbacct_lock);
2105 	now = mftb();
2106 	vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2107 		vcpu->arch.stolen_logged;
2108 	vcpu->arch.busy_preempt = now;
2109 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2110 	spin_unlock_irq(&vcpu->arch.tbacct_lock);
2111 	--vc->n_runnable;
2112 	WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2113 }
2114 
2115 static int kvmppc_grab_hwthread(int cpu)
2116 {
2117 	struct paca_struct *tpaca;
2118 	long timeout = 10000;
2119 
2120 	/*
2121 	 * ISA v3.0 idle routines do not set hwthread_state or test
2122 	 * hwthread_req, so they can not grab idle threads.
2123 	 */
2124 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2125 		WARN(1, "KVM: can not control sibling threads\n");
2126 		return -EBUSY;
2127 	}
2128 
2129 	tpaca = &paca[cpu];
2130 
2131 	/* Ensure the thread won't go into the kernel if it wakes */
2132 	tpaca->kvm_hstate.kvm_vcpu = NULL;
2133 	tpaca->kvm_hstate.kvm_vcore = NULL;
2134 	tpaca->kvm_hstate.napping = 0;
2135 	smp_wmb();
2136 	tpaca->kvm_hstate.hwthread_req = 1;
2137 
2138 	/*
2139 	 * If the thread is already executing in the kernel (e.g. handling
2140 	 * a stray interrupt), wait for it to get back to nap mode.
2141 	 * The smp_mb() is to ensure that our setting of hwthread_req
2142 	 * is visible before we look at hwthread_state, so if this
2143 	 * races with the code at system_reset_pSeries and the thread
2144 	 * misses our setting of hwthread_req, we are sure to see its
2145 	 * setting of hwthread_state, and vice versa.
2146 	 */
2147 	smp_mb();
2148 	while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2149 		if (--timeout <= 0) {
2150 			pr_err("KVM: couldn't grab cpu %d\n", cpu);
2151 			return -EBUSY;
2152 		}
2153 		udelay(1);
2154 	}
2155 	return 0;
2156 }
2157 
2158 static void kvmppc_release_hwthread(int cpu)
2159 {
2160 	struct paca_struct *tpaca;
2161 
2162 	tpaca = &paca[cpu];
2163 	tpaca->kvm_hstate.kvm_vcpu = NULL;
2164 	tpaca->kvm_hstate.kvm_vcore = NULL;
2165 	tpaca->kvm_hstate.kvm_split_mode = NULL;
2166 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
2167 		tpaca->kvm_hstate.hwthread_req = 0;
2168 
2169 }
2170 
2171 static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2172 {
2173 	int i;
2174 
2175 	cpu = cpu_first_thread_sibling(cpu);
2176 	cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2177 	/*
2178 	 * Make sure setting of bit in need_tlb_flush precedes
2179 	 * testing of cpu_in_guest bits.  The matching barrier on
2180 	 * the other side is the first smp_mb() in kvmppc_run_core().
2181 	 */
2182 	smp_mb();
2183 	for (i = 0; i < threads_per_core; ++i)
2184 		if (cpumask_test_cpu(cpu + i, &kvm->arch.cpu_in_guest))
2185 			smp_call_function_single(cpu + i, do_nothing, NULL, 1);
2186 }
2187 
2188 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2189 {
2190 	struct kvm *kvm = vcpu->kvm;
2191 
2192 	/*
2193 	 * With radix, the guest can do TLB invalidations itself,
2194 	 * and it could choose to use the local form (tlbiel) if
2195 	 * it is invalidating a translation that has only ever been
2196 	 * used on one vcpu.  However, that doesn't mean it has
2197 	 * only ever been used on one physical cpu, since vcpus
2198 	 * can move around between pcpus.  To cope with this, when
2199 	 * a vcpu moves from one pcpu to another, we need to tell
2200 	 * any vcpus running on the same core as this vcpu previously
2201 	 * ran to flush the TLB.  The TLB is shared between threads,
2202 	 * so we use a single bit in .need_tlb_flush for all 4 threads.
2203 	 */
2204 	if (vcpu->arch.prev_cpu != pcpu) {
2205 		if (vcpu->arch.prev_cpu >= 0 &&
2206 		    cpu_first_thread_sibling(vcpu->arch.prev_cpu) !=
2207 		    cpu_first_thread_sibling(pcpu))
2208 			radix_flush_cpu(kvm, vcpu->arch.prev_cpu, vcpu);
2209 		vcpu->arch.prev_cpu = pcpu;
2210 	}
2211 }
2212 
2213 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
2214 {
2215 	int cpu;
2216 	struct paca_struct *tpaca;
2217 	struct kvm *kvm = vc->kvm;
2218 
2219 	cpu = vc->pcpu;
2220 	if (vcpu) {
2221 		if (vcpu->arch.timer_running) {
2222 			hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2223 			vcpu->arch.timer_running = 0;
2224 		}
2225 		cpu += vcpu->arch.ptid;
2226 		vcpu->cpu = vc->pcpu;
2227 		vcpu->arch.thread_cpu = cpu;
2228 		cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
2229 	}
2230 	tpaca = &paca[cpu];
2231 	tpaca->kvm_hstate.kvm_vcpu = vcpu;
2232 	tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
2233 	/* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */
2234 	smp_wmb();
2235 	tpaca->kvm_hstate.kvm_vcore = vc;
2236 	if (cpu != smp_processor_id())
2237 		kvmppc_ipi_thread(cpu);
2238 }
2239 
2240 static void kvmppc_wait_for_nap(void)
2241 {
2242 	int cpu = smp_processor_id();
2243 	int i, loops;
2244 	int n_threads = threads_per_vcore();
2245 
2246 	if (n_threads <= 1)
2247 		return;
2248 	for (loops = 0; loops < 1000000; ++loops) {
2249 		/*
2250 		 * Check if all threads are finished.
2251 		 * We set the vcore pointer when starting a thread
2252 		 * and the thread clears it when finished, so we look
2253 		 * for any threads that still have a non-NULL vcore ptr.
2254 		 */
2255 		for (i = 1; i < n_threads; ++i)
2256 			if (paca[cpu + i].kvm_hstate.kvm_vcore)
2257 				break;
2258 		if (i == n_threads) {
2259 			HMT_medium();
2260 			return;
2261 		}
2262 		HMT_low();
2263 	}
2264 	HMT_medium();
2265 	for (i = 1; i < n_threads; ++i)
2266 		if (paca[cpu + i].kvm_hstate.kvm_vcore)
2267 			pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
2268 }
2269 
2270 /*
2271  * Check that we are on thread 0 and that any other threads in
2272  * this core are off-line.  Then grab the threads so they can't
2273  * enter the kernel.
2274  */
2275 static int on_primary_thread(void)
2276 {
2277 	int cpu = smp_processor_id();
2278 	int thr;
2279 
2280 	/* Are we on a primary subcore? */
2281 	if (cpu_thread_in_subcore(cpu))
2282 		return 0;
2283 
2284 	thr = 0;
2285 	while (++thr < threads_per_subcore)
2286 		if (cpu_online(cpu + thr))
2287 			return 0;
2288 
2289 	/* Grab all hw threads so they can't go into the kernel */
2290 	for (thr = 1; thr < threads_per_subcore; ++thr) {
2291 		if (kvmppc_grab_hwthread(cpu + thr)) {
2292 			/* Couldn't grab one; let the others go */
2293 			do {
2294 				kvmppc_release_hwthread(cpu + thr);
2295 			} while (--thr > 0);
2296 			return 0;
2297 		}
2298 	}
2299 	return 1;
2300 }
2301 
2302 /*
2303  * A list of virtual cores for each physical CPU.
2304  * These are vcores that could run but their runner VCPU tasks are
2305  * (or may be) preempted.
2306  */
2307 struct preempted_vcore_list {
2308 	struct list_head	list;
2309 	spinlock_t		lock;
2310 };
2311 
2312 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
2313 
2314 static void init_vcore_lists(void)
2315 {
2316 	int cpu;
2317 
2318 	for_each_possible_cpu(cpu) {
2319 		struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
2320 		spin_lock_init(&lp->lock);
2321 		INIT_LIST_HEAD(&lp->list);
2322 	}
2323 }
2324 
2325 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
2326 {
2327 	struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2328 
2329 	vc->vcore_state = VCORE_PREEMPT;
2330 	vc->pcpu = smp_processor_id();
2331 	if (vc->num_threads < threads_per_vcore()) {
2332 		spin_lock(&lp->lock);
2333 		list_add_tail(&vc->preempt_list, &lp->list);
2334 		spin_unlock(&lp->lock);
2335 	}
2336 
2337 	/* Start accumulating stolen time */
2338 	kvmppc_core_start_stolen(vc);
2339 }
2340 
2341 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
2342 {
2343 	struct preempted_vcore_list *lp;
2344 
2345 	kvmppc_core_end_stolen(vc);
2346 	if (!list_empty(&vc->preempt_list)) {
2347 		lp = &per_cpu(preempted_vcores, vc->pcpu);
2348 		spin_lock(&lp->lock);
2349 		list_del_init(&vc->preempt_list);
2350 		spin_unlock(&lp->lock);
2351 	}
2352 	vc->vcore_state = VCORE_INACTIVE;
2353 }
2354 
2355 /*
2356  * This stores information about the virtual cores currently
2357  * assigned to a physical core.
2358  */
2359 struct core_info {
2360 	int		n_subcores;
2361 	int		max_subcore_threads;
2362 	int		total_threads;
2363 	int		subcore_threads[MAX_SUBCORES];
2364 	struct kvmppc_vcore *vc[MAX_SUBCORES];
2365 };
2366 
2367 /*
2368  * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7
2369  * respectively in 2-way micro-threading (split-core) mode.
2370  */
2371 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
2372 
2373 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
2374 {
2375 	memset(cip, 0, sizeof(*cip));
2376 	cip->n_subcores = 1;
2377 	cip->max_subcore_threads = vc->num_threads;
2378 	cip->total_threads = vc->num_threads;
2379 	cip->subcore_threads[0] = vc->num_threads;
2380 	cip->vc[0] = vc;
2381 }
2382 
2383 static bool subcore_config_ok(int n_subcores, int n_threads)
2384 {
2385 	/* Can only dynamically split if unsplit to begin with */
2386 	if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
2387 		return false;
2388 	if (n_subcores > MAX_SUBCORES)
2389 		return false;
2390 	if (n_subcores > 1) {
2391 		if (!(dynamic_mt_modes & 2))
2392 			n_subcores = 4;
2393 		if (n_subcores > 2 && !(dynamic_mt_modes & 4))
2394 			return false;
2395 	}
2396 
2397 	return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
2398 }
2399 
2400 static void init_vcore_to_run(struct kvmppc_vcore *vc)
2401 {
2402 	vc->entry_exit_map = 0;
2403 	vc->in_guest = 0;
2404 	vc->napping_threads = 0;
2405 	vc->conferring_threads = 0;
2406 }
2407 
2408 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
2409 {
2410 	int n_threads = vc->num_threads;
2411 	int sub;
2412 
2413 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2414 		return false;
2415 
2416 	if (n_threads < cip->max_subcore_threads)
2417 		n_threads = cip->max_subcore_threads;
2418 	if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
2419 		return false;
2420 	cip->max_subcore_threads = n_threads;
2421 
2422 	sub = cip->n_subcores;
2423 	++cip->n_subcores;
2424 	cip->total_threads += vc->num_threads;
2425 	cip->subcore_threads[sub] = vc->num_threads;
2426 	cip->vc[sub] = vc;
2427 	init_vcore_to_run(vc);
2428 	list_del_init(&vc->preempt_list);
2429 
2430 	return true;
2431 }
2432 
2433 /*
2434  * Work out whether it is possible to piggyback the execution of
2435  * vcore *pvc onto the execution of the other vcores described in *cip.
2436  */
2437 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
2438 			  int target_threads)
2439 {
2440 	if (cip->total_threads + pvc->num_threads > target_threads)
2441 		return false;
2442 
2443 	return can_dynamic_split(pvc, cip);
2444 }
2445 
2446 static void prepare_threads(struct kvmppc_vcore *vc)
2447 {
2448 	int i;
2449 	struct kvm_vcpu *vcpu;
2450 
2451 	for_each_runnable_thread(i, vcpu, vc) {
2452 		if (signal_pending(vcpu->arch.run_task))
2453 			vcpu->arch.ret = -EINTR;
2454 		else if (vcpu->arch.vpa.update_pending ||
2455 			 vcpu->arch.slb_shadow.update_pending ||
2456 			 vcpu->arch.dtl.update_pending)
2457 			vcpu->arch.ret = RESUME_GUEST;
2458 		else
2459 			continue;
2460 		kvmppc_remove_runnable(vc, vcpu);
2461 		wake_up(&vcpu->arch.cpu_run);
2462 	}
2463 }
2464 
2465 static void collect_piggybacks(struct core_info *cip, int target_threads)
2466 {
2467 	struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2468 	struct kvmppc_vcore *pvc, *vcnext;
2469 
2470 	spin_lock(&lp->lock);
2471 	list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
2472 		if (!spin_trylock(&pvc->lock))
2473 			continue;
2474 		prepare_threads(pvc);
2475 		if (!pvc->n_runnable) {
2476 			list_del_init(&pvc->preempt_list);
2477 			if (pvc->runner == NULL) {
2478 				pvc->vcore_state = VCORE_INACTIVE;
2479 				kvmppc_core_end_stolen(pvc);
2480 			}
2481 			spin_unlock(&pvc->lock);
2482 			continue;
2483 		}
2484 		if (!can_piggyback(pvc, cip, target_threads)) {
2485 			spin_unlock(&pvc->lock);
2486 			continue;
2487 		}
2488 		kvmppc_core_end_stolen(pvc);
2489 		pvc->vcore_state = VCORE_PIGGYBACK;
2490 		if (cip->total_threads >= target_threads)
2491 			break;
2492 	}
2493 	spin_unlock(&lp->lock);
2494 }
2495 
2496 static bool recheck_signals(struct core_info *cip)
2497 {
2498 	int sub, i;
2499 	struct kvm_vcpu *vcpu;
2500 
2501 	for (sub = 0; sub < cip->n_subcores; ++sub)
2502 		for_each_runnable_thread(i, vcpu, cip->vc[sub])
2503 			if (signal_pending(vcpu->arch.run_task))
2504 				return true;
2505 	return false;
2506 }
2507 
2508 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
2509 {
2510 	int still_running = 0, i;
2511 	u64 now;
2512 	long ret;
2513 	struct kvm_vcpu *vcpu;
2514 
2515 	spin_lock(&vc->lock);
2516 	now = get_tb();
2517 	for_each_runnable_thread(i, vcpu, vc) {
2518 		/* cancel pending dec exception if dec is positive */
2519 		if (now < vcpu->arch.dec_expires &&
2520 		    kvmppc_core_pending_dec(vcpu))
2521 			kvmppc_core_dequeue_dec(vcpu);
2522 
2523 		trace_kvm_guest_exit(vcpu);
2524 
2525 		ret = RESUME_GUEST;
2526 		if (vcpu->arch.trap)
2527 			ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
2528 						    vcpu->arch.run_task);
2529 
2530 		vcpu->arch.ret = ret;
2531 		vcpu->arch.trap = 0;
2532 
2533 		if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
2534 			if (vcpu->arch.pending_exceptions)
2535 				kvmppc_core_prepare_to_enter(vcpu);
2536 			if (vcpu->arch.ceded)
2537 				kvmppc_set_timer(vcpu);
2538 			else
2539 				++still_running;
2540 		} else {
2541 			kvmppc_remove_runnable(vc, vcpu);
2542 			wake_up(&vcpu->arch.cpu_run);
2543 		}
2544 	}
2545 	if (!is_master) {
2546 		if (still_running > 0) {
2547 			kvmppc_vcore_preempt(vc);
2548 		} else if (vc->runner) {
2549 			vc->vcore_state = VCORE_PREEMPT;
2550 			kvmppc_core_start_stolen(vc);
2551 		} else {
2552 			vc->vcore_state = VCORE_INACTIVE;
2553 		}
2554 		if (vc->n_runnable > 0 && vc->runner == NULL) {
2555 			/* make sure there's a candidate runner awake */
2556 			i = -1;
2557 			vcpu = next_runnable_thread(vc, &i);
2558 			wake_up(&vcpu->arch.cpu_run);
2559 		}
2560 	}
2561 	spin_unlock(&vc->lock);
2562 }
2563 
2564 /*
2565  * Clear core from the list of active host cores as we are about to
2566  * enter the guest. Only do this if it is the primary thread of the
2567  * core (not if a subcore) that is entering the guest.
2568  */
2569 static inline int kvmppc_clear_host_core(unsigned int cpu)
2570 {
2571 	int core;
2572 
2573 	if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2574 		return 0;
2575 	/*
2576 	 * Memory barrier can be omitted here as we will do a smp_wmb()
2577 	 * later in kvmppc_start_thread and we need ensure that state is
2578 	 * visible to other CPUs only after we enter guest.
2579 	 */
2580 	core = cpu >> threads_shift;
2581 	kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
2582 	return 0;
2583 }
2584 
2585 /*
2586  * Advertise this core as an active host core since we exited the guest
2587  * Only need to do this if it is the primary thread of the core that is
2588  * exiting.
2589  */
2590 static inline int kvmppc_set_host_core(unsigned int cpu)
2591 {
2592 	int core;
2593 
2594 	if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2595 		return 0;
2596 
2597 	/*
2598 	 * Memory barrier can be omitted here because we do a spin_unlock
2599 	 * immediately after this which provides the memory barrier.
2600 	 */
2601 	core = cpu >> threads_shift;
2602 	kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
2603 	return 0;
2604 }
2605 
2606 static void set_irq_happened(int trap)
2607 {
2608 	switch (trap) {
2609 	case BOOK3S_INTERRUPT_EXTERNAL:
2610 		local_paca->irq_happened |= PACA_IRQ_EE;
2611 		break;
2612 	case BOOK3S_INTERRUPT_H_DOORBELL:
2613 		local_paca->irq_happened |= PACA_IRQ_DBELL;
2614 		break;
2615 	case BOOK3S_INTERRUPT_HMI:
2616 		local_paca->irq_happened |= PACA_IRQ_HMI;
2617 		break;
2618 	}
2619 }
2620 
2621 /*
2622  * Run a set of guest threads on a physical core.
2623  * Called with vc->lock held.
2624  */
2625 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
2626 {
2627 	struct kvm_vcpu *vcpu;
2628 	int i;
2629 	int srcu_idx;
2630 	struct core_info core_info;
2631 	struct kvmppc_vcore *pvc;
2632 	struct kvm_split_mode split_info, *sip;
2633 	int split, subcore_size, active;
2634 	int sub;
2635 	bool thr0_done;
2636 	unsigned long cmd_bit, stat_bit;
2637 	int pcpu, thr;
2638 	int target_threads;
2639 	int controlled_threads;
2640 	int trap;
2641 
2642 	/*
2643 	 * Remove from the list any threads that have a signal pending
2644 	 * or need a VPA update done
2645 	 */
2646 	prepare_threads(vc);
2647 
2648 	/* if the runner is no longer runnable, let the caller pick a new one */
2649 	if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
2650 		return;
2651 
2652 	/*
2653 	 * Initialize *vc.
2654 	 */
2655 	init_vcore_to_run(vc);
2656 	vc->preempt_tb = TB_NIL;
2657 
2658 	/*
2659 	 * Number of threads that we will be controlling: the same as
2660 	 * the number of threads per subcore, except on POWER9,
2661 	 * where it's 1 because the threads are (mostly) independent.
2662 	 */
2663 	controlled_threads = threads_per_vcore();
2664 
2665 	/*
2666 	 * Make sure we are running on primary threads, and that secondary
2667 	 * threads are offline.  Also check if the number of threads in this
2668 	 * guest are greater than the current system threads per guest.
2669 	 */
2670 	if ((controlled_threads > 1) &&
2671 	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
2672 		for_each_runnable_thread(i, vcpu, vc) {
2673 			vcpu->arch.ret = -EBUSY;
2674 			kvmppc_remove_runnable(vc, vcpu);
2675 			wake_up(&vcpu->arch.cpu_run);
2676 		}
2677 		goto out;
2678 	}
2679 
2680 	/*
2681 	 * See if we could run any other vcores on the physical core
2682 	 * along with this one.
2683 	 */
2684 	init_core_info(&core_info, vc);
2685 	pcpu = smp_processor_id();
2686 	target_threads = controlled_threads;
2687 	if (target_smt_mode && target_smt_mode < target_threads)
2688 		target_threads = target_smt_mode;
2689 	if (vc->num_threads < target_threads)
2690 		collect_piggybacks(&core_info, target_threads);
2691 
2692 	/*
2693 	 * On radix, arrange for TLB flushing if necessary.
2694 	 * This has to be done before disabling interrupts since
2695 	 * it uses smp_call_function().
2696 	 */
2697 	pcpu = smp_processor_id();
2698 	if (kvm_is_radix(vc->kvm)) {
2699 		for (sub = 0; sub < core_info.n_subcores; ++sub)
2700 			for_each_runnable_thread(i, vcpu, core_info.vc[sub])
2701 				kvmppc_prepare_radix_vcpu(vcpu, pcpu);
2702 	}
2703 
2704 	/*
2705 	 * Hard-disable interrupts, and check resched flag and signals.
2706 	 * If we need to reschedule or deliver a signal, clean up
2707 	 * and return without going into the guest(s).
2708 	 */
2709 	local_irq_disable();
2710 	hard_irq_disable();
2711 	if (lazy_irq_pending() || need_resched() ||
2712 	    recheck_signals(&core_info)) {
2713 		local_irq_enable();
2714 		vc->vcore_state = VCORE_INACTIVE;
2715 		/* Unlock all except the primary vcore */
2716 		for (sub = 1; sub < core_info.n_subcores; ++sub) {
2717 			pvc = core_info.vc[sub];
2718 			/* Put back on to the preempted vcores list */
2719 			kvmppc_vcore_preempt(pvc);
2720 			spin_unlock(&pvc->lock);
2721 		}
2722 		for (i = 0; i < controlled_threads; ++i)
2723 			kvmppc_release_hwthread(pcpu + i);
2724 		return;
2725 	}
2726 
2727 	kvmppc_clear_host_core(pcpu);
2728 
2729 	/* Decide on micro-threading (split-core) mode */
2730 	subcore_size = threads_per_subcore;
2731 	cmd_bit = stat_bit = 0;
2732 	split = core_info.n_subcores;
2733 	sip = NULL;
2734 	if (split > 1) {
2735 		/* threads_per_subcore must be MAX_SMT_THREADS (8) here */
2736 		if (split == 2 && (dynamic_mt_modes & 2)) {
2737 			cmd_bit = HID0_POWER8_1TO2LPAR;
2738 			stat_bit = HID0_POWER8_2LPARMODE;
2739 		} else {
2740 			split = 4;
2741 			cmd_bit = HID0_POWER8_1TO4LPAR;
2742 			stat_bit = HID0_POWER8_4LPARMODE;
2743 		}
2744 		subcore_size = MAX_SMT_THREADS / split;
2745 		sip = &split_info;
2746 		memset(&split_info, 0, sizeof(split_info));
2747 		split_info.rpr = mfspr(SPRN_RPR);
2748 		split_info.pmmar = mfspr(SPRN_PMMAR);
2749 		split_info.ldbar = mfspr(SPRN_LDBAR);
2750 		split_info.subcore_size = subcore_size;
2751 		for (sub = 0; sub < core_info.n_subcores; ++sub)
2752 			split_info.vc[sub] = core_info.vc[sub];
2753 		/* order writes to split_info before kvm_split_mode pointer */
2754 		smp_wmb();
2755 	}
2756 	for (thr = 0; thr < controlled_threads; ++thr)
2757 		paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip;
2758 
2759 	/* Initiate micro-threading (split-core) if required */
2760 	if (cmd_bit) {
2761 		unsigned long hid0 = mfspr(SPRN_HID0);
2762 
2763 		hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
2764 		mb();
2765 		mtspr(SPRN_HID0, hid0);
2766 		isync();
2767 		for (;;) {
2768 			hid0 = mfspr(SPRN_HID0);
2769 			if (hid0 & stat_bit)
2770 				break;
2771 			cpu_relax();
2772 		}
2773 	}
2774 
2775 	/* Start all the threads */
2776 	active = 0;
2777 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
2778 		thr = subcore_thread_map[sub];
2779 		thr0_done = false;
2780 		active |= 1 << thr;
2781 		pvc = core_info.vc[sub];
2782 		pvc->pcpu = pcpu + thr;
2783 		for_each_runnable_thread(i, vcpu, pvc) {
2784 			kvmppc_start_thread(vcpu, pvc);
2785 			kvmppc_create_dtl_entry(vcpu, pvc);
2786 			trace_kvm_guest_enter(vcpu);
2787 			if (!vcpu->arch.ptid)
2788 				thr0_done = true;
2789 			active |= 1 << (thr + vcpu->arch.ptid);
2790 		}
2791 		/*
2792 		 * We need to start the first thread of each subcore
2793 		 * even if it doesn't have a vcpu.
2794 		 */
2795 		if (!thr0_done)
2796 			kvmppc_start_thread(NULL, pvc);
2797 		thr += pvc->num_threads;
2798 	}
2799 
2800 	/*
2801 	 * Ensure that split_info.do_nap is set after setting
2802 	 * the vcore pointer in the PACA of the secondaries.
2803 	 */
2804 	smp_mb();
2805 	if (cmd_bit)
2806 		split_info.do_nap = 1;	/* ask secondaries to nap when done */
2807 
2808 	/*
2809 	 * When doing micro-threading, poke the inactive threads as well.
2810 	 * This gets them to the nap instruction after kvm_do_nap,
2811 	 * which reduces the time taken to unsplit later.
2812 	 */
2813 	if (split > 1)
2814 		for (thr = 1; thr < threads_per_subcore; ++thr)
2815 			if (!(active & (1 << thr)))
2816 				kvmppc_ipi_thread(pcpu + thr);
2817 
2818 	vc->vcore_state = VCORE_RUNNING;
2819 	preempt_disable();
2820 
2821 	trace_kvmppc_run_core(vc, 0);
2822 
2823 	for (sub = 0; sub < core_info.n_subcores; ++sub)
2824 		spin_unlock(&core_info.vc[sub]->lock);
2825 
2826 	/*
2827 	 * Interrupts will be enabled once we get into the guest,
2828 	 * so tell lockdep that we're about to enable interrupts.
2829 	 */
2830 	trace_hardirqs_on();
2831 
2832 	guest_enter();
2833 
2834 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
2835 
2836 	trap = __kvmppc_vcore_entry();
2837 
2838 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
2839 
2840 	guest_exit();
2841 
2842 	trace_hardirqs_off();
2843 	set_irq_happened(trap);
2844 
2845 	spin_lock(&vc->lock);
2846 	/* prevent other vcpu threads from doing kvmppc_start_thread() now */
2847 	vc->vcore_state = VCORE_EXITING;
2848 
2849 	/* wait for secondary threads to finish writing their state to memory */
2850 	kvmppc_wait_for_nap();
2851 
2852 	/* Return to whole-core mode if we split the core earlier */
2853 	if (split > 1) {
2854 		unsigned long hid0 = mfspr(SPRN_HID0);
2855 		unsigned long loops = 0;
2856 
2857 		hid0 &= ~HID0_POWER8_DYNLPARDIS;
2858 		stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
2859 		mb();
2860 		mtspr(SPRN_HID0, hid0);
2861 		isync();
2862 		for (;;) {
2863 			hid0 = mfspr(SPRN_HID0);
2864 			if (!(hid0 & stat_bit))
2865 				break;
2866 			cpu_relax();
2867 			++loops;
2868 		}
2869 		split_info.do_nap = 0;
2870 	}
2871 
2872 	kvmppc_set_host_core(pcpu);
2873 
2874 	local_irq_enable();
2875 
2876 	/* Let secondaries go back to the offline loop */
2877 	for (i = 0; i < controlled_threads; ++i) {
2878 		kvmppc_release_hwthread(pcpu + i);
2879 		if (sip && sip->napped[i])
2880 			kvmppc_ipi_thread(pcpu + i);
2881 		cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
2882 	}
2883 
2884 	spin_unlock(&vc->lock);
2885 
2886 	/* make sure updates to secondary vcpu structs are visible now */
2887 	smp_mb();
2888 
2889 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
2890 		pvc = core_info.vc[sub];
2891 		post_guest_process(pvc, pvc == vc);
2892 	}
2893 
2894 	spin_lock(&vc->lock);
2895 	preempt_enable();
2896 
2897  out:
2898 	vc->vcore_state = VCORE_INACTIVE;
2899 	trace_kvmppc_run_core(vc, 1);
2900 }
2901 
2902 /*
2903  * Wait for some other vcpu thread to execute us, and
2904  * wake us up when we need to handle something in the host.
2905  */
2906 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
2907 				 struct kvm_vcpu *vcpu, int wait_state)
2908 {
2909 	DEFINE_WAIT(wait);
2910 
2911 	prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
2912 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
2913 		spin_unlock(&vc->lock);
2914 		schedule();
2915 		spin_lock(&vc->lock);
2916 	}
2917 	finish_wait(&vcpu->arch.cpu_run, &wait);
2918 }
2919 
2920 static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
2921 {
2922 	/* 10us base */
2923 	if (vc->halt_poll_ns == 0 && halt_poll_ns_grow)
2924 		vc->halt_poll_ns = 10000;
2925 	else
2926 		vc->halt_poll_ns *= halt_poll_ns_grow;
2927 }
2928 
2929 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
2930 {
2931 	if (halt_poll_ns_shrink == 0)
2932 		vc->halt_poll_ns = 0;
2933 	else
2934 		vc->halt_poll_ns /= halt_poll_ns_shrink;
2935 }
2936 
2937 #ifdef CONFIG_KVM_XICS
2938 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
2939 {
2940 	if (!xive_enabled())
2941 		return false;
2942 	return vcpu->arch.xive_saved_state.pipr <
2943 		vcpu->arch.xive_saved_state.cppr;
2944 }
2945 #else
2946 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
2947 {
2948 	return false;
2949 }
2950 #endif /* CONFIG_KVM_XICS */
2951 
2952 static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
2953 {
2954 	if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
2955 	    kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
2956 		return true;
2957 
2958 	return false;
2959 }
2960 
2961 /*
2962  * Check to see if any of the runnable vcpus on the vcore have pending
2963  * exceptions or are no longer ceded
2964  */
2965 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
2966 {
2967 	struct kvm_vcpu *vcpu;
2968 	int i;
2969 
2970 	for_each_runnable_thread(i, vcpu, vc) {
2971 		if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
2972 			return 1;
2973 	}
2974 
2975 	return 0;
2976 }
2977 
2978 /*
2979  * All the vcpus in this vcore are idle, so wait for a decrementer
2980  * or external interrupt to one of the vcpus.  vc->lock is held.
2981  */
2982 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
2983 {
2984 	ktime_t cur, start_poll, start_wait;
2985 	int do_sleep = 1;
2986 	u64 block_ns;
2987 	DECLARE_SWAITQUEUE(wait);
2988 
2989 	/* Poll for pending exceptions and ceded state */
2990 	cur = start_poll = ktime_get();
2991 	if (vc->halt_poll_ns) {
2992 		ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
2993 		++vc->runner->stat.halt_attempted_poll;
2994 
2995 		vc->vcore_state = VCORE_POLLING;
2996 		spin_unlock(&vc->lock);
2997 
2998 		do {
2999 			if (kvmppc_vcore_check_block(vc)) {
3000 				do_sleep = 0;
3001 				break;
3002 			}
3003 			cur = ktime_get();
3004 		} while (single_task_running() && ktime_before(cur, stop));
3005 
3006 		spin_lock(&vc->lock);
3007 		vc->vcore_state = VCORE_INACTIVE;
3008 
3009 		if (!do_sleep) {
3010 			++vc->runner->stat.halt_successful_poll;
3011 			goto out;
3012 		}
3013 	}
3014 
3015 	prepare_to_swait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
3016 
3017 	if (kvmppc_vcore_check_block(vc)) {
3018 		finish_swait(&vc->wq, &wait);
3019 		do_sleep = 0;
3020 		/* If we polled, count this as a successful poll */
3021 		if (vc->halt_poll_ns)
3022 			++vc->runner->stat.halt_successful_poll;
3023 		goto out;
3024 	}
3025 
3026 	start_wait = ktime_get();
3027 
3028 	vc->vcore_state = VCORE_SLEEPING;
3029 	trace_kvmppc_vcore_blocked(vc, 0);
3030 	spin_unlock(&vc->lock);
3031 	schedule();
3032 	finish_swait(&vc->wq, &wait);
3033 	spin_lock(&vc->lock);
3034 	vc->vcore_state = VCORE_INACTIVE;
3035 	trace_kvmppc_vcore_blocked(vc, 1);
3036 	++vc->runner->stat.halt_successful_wait;
3037 
3038 	cur = ktime_get();
3039 
3040 out:
3041 	block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
3042 
3043 	/* Attribute wait time */
3044 	if (do_sleep) {
3045 		vc->runner->stat.halt_wait_ns +=
3046 			ktime_to_ns(cur) - ktime_to_ns(start_wait);
3047 		/* Attribute failed poll time */
3048 		if (vc->halt_poll_ns)
3049 			vc->runner->stat.halt_poll_fail_ns +=
3050 				ktime_to_ns(start_wait) -
3051 				ktime_to_ns(start_poll);
3052 	} else {
3053 		/* Attribute successful poll time */
3054 		if (vc->halt_poll_ns)
3055 			vc->runner->stat.halt_poll_success_ns +=
3056 				ktime_to_ns(cur) -
3057 				ktime_to_ns(start_poll);
3058 	}
3059 
3060 	/* Adjust poll time */
3061 	if (halt_poll_ns) {
3062 		if (block_ns <= vc->halt_poll_ns)
3063 			;
3064 		/* We slept and blocked for longer than the max halt time */
3065 		else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
3066 			shrink_halt_poll_ns(vc);
3067 		/* We slept and our poll time is too small */
3068 		else if (vc->halt_poll_ns < halt_poll_ns &&
3069 				block_ns < halt_poll_ns)
3070 			grow_halt_poll_ns(vc);
3071 		if (vc->halt_poll_ns > halt_poll_ns)
3072 			vc->halt_poll_ns = halt_poll_ns;
3073 	} else
3074 		vc->halt_poll_ns = 0;
3075 
3076 	trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
3077 }
3078 
3079 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
3080 {
3081 	int n_ceded, i;
3082 	struct kvmppc_vcore *vc;
3083 	struct kvm_vcpu *v;
3084 
3085 	trace_kvmppc_run_vcpu_enter(vcpu);
3086 
3087 	kvm_run->exit_reason = 0;
3088 	vcpu->arch.ret = RESUME_GUEST;
3089 	vcpu->arch.trap = 0;
3090 	kvmppc_update_vpas(vcpu);
3091 
3092 	/*
3093 	 * Synchronize with other threads in this virtual core
3094 	 */
3095 	vc = vcpu->arch.vcore;
3096 	spin_lock(&vc->lock);
3097 	vcpu->arch.ceded = 0;
3098 	vcpu->arch.run_task = current;
3099 	vcpu->arch.kvm_run = kvm_run;
3100 	vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
3101 	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
3102 	vcpu->arch.busy_preempt = TB_NIL;
3103 	WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
3104 	++vc->n_runnable;
3105 
3106 	/*
3107 	 * This happens the first time this is called for a vcpu.
3108 	 * If the vcore is already running, we may be able to start
3109 	 * this thread straight away and have it join in.
3110 	 */
3111 	if (!signal_pending(current)) {
3112 		if (vc->vcore_state == VCORE_PIGGYBACK) {
3113 			if (spin_trylock(&vc->lock)) {
3114 				if (vc->vcore_state == VCORE_RUNNING &&
3115 				    !VCORE_IS_EXITING(vc)) {
3116 					kvmppc_create_dtl_entry(vcpu, vc);
3117 					kvmppc_start_thread(vcpu, vc);
3118 					trace_kvm_guest_enter(vcpu);
3119 				}
3120 				spin_unlock(&vc->lock);
3121 			}
3122 		} else if (vc->vcore_state == VCORE_RUNNING &&
3123 			   !VCORE_IS_EXITING(vc)) {
3124 			kvmppc_create_dtl_entry(vcpu, vc);
3125 			kvmppc_start_thread(vcpu, vc);
3126 			trace_kvm_guest_enter(vcpu);
3127 		} else if (vc->vcore_state == VCORE_SLEEPING) {
3128 			swake_up(&vc->wq);
3129 		}
3130 
3131 	}
3132 
3133 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3134 	       !signal_pending(current)) {
3135 		if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3136 			kvmppc_vcore_end_preempt(vc);
3137 
3138 		if (vc->vcore_state != VCORE_INACTIVE) {
3139 			kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
3140 			continue;
3141 		}
3142 		for_each_runnable_thread(i, v, vc) {
3143 			kvmppc_core_prepare_to_enter(v);
3144 			if (signal_pending(v->arch.run_task)) {
3145 				kvmppc_remove_runnable(vc, v);
3146 				v->stat.signal_exits++;
3147 				v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
3148 				v->arch.ret = -EINTR;
3149 				wake_up(&v->arch.cpu_run);
3150 			}
3151 		}
3152 		if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
3153 			break;
3154 		n_ceded = 0;
3155 		for_each_runnable_thread(i, v, vc) {
3156 			if (!kvmppc_vcpu_woken(v))
3157 				n_ceded += v->arch.ceded;
3158 			else
3159 				v->arch.ceded = 0;
3160 		}
3161 		vc->runner = vcpu;
3162 		if (n_ceded == vc->n_runnable) {
3163 			kvmppc_vcore_blocked(vc);
3164 		} else if (need_resched()) {
3165 			kvmppc_vcore_preempt(vc);
3166 			/* Let something else run */
3167 			cond_resched_lock(&vc->lock);
3168 			if (vc->vcore_state == VCORE_PREEMPT)
3169 				kvmppc_vcore_end_preempt(vc);
3170 		} else {
3171 			kvmppc_run_core(vc);
3172 		}
3173 		vc->runner = NULL;
3174 	}
3175 
3176 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3177 	       (vc->vcore_state == VCORE_RUNNING ||
3178 		vc->vcore_state == VCORE_EXITING ||
3179 		vc->vcore_state == VCORE_PIGGYBACK))
3180 		kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
3181 
3182 	if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3183 		kvmppc_vcore_end_preempt(vc);
3184 
3185 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
3186 		kvmppc_remove_runnable(vc, vcpu);
3187 		vcpu->stat.signal_exits++;
3188 		kvm_run->exit_reason = KVM_EXIT_INTR;
3189 		vcpu->arch.ret = -EINTR;
3190 	}
3191 
3192 	if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
3193 		/* Wake up some vcpu to run the core */
3194 		i = -1;
3195 		v = next_runnable_thread(vc, &i);
3196 		wake_up(&v->arch.cpu_run);
3197 	}
3198 
3199 	trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
3200 	spin_unlock(&vc->lock);
3201 	return vcpu->arch.ret;
3202 }
3203 
3204 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
3205 {
3206 	int r;
3207 	int srcu_idx;
3208 	unsigned long ebb_regs[3] = {};	/* shut up GCC */
3209 	unsigned long user_tar = 0;
3210 	unsigned int user_vrsave;
3211 
3212 	if (!vcpu->arch.sane) {
3213 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3214 		return -EINVAL;
3215 	}
3216 
3217 	/*
3218 	 * Don't allow entry with a suspended transaction, because
3219 	 * the guest entry/exit code will lose it.
3220 	 * If the guest has TM enabled, save away their TM-related SPRs
3221 	 * (they will get restored by the TM unavailable interrupt).
3222 	 */
3223 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
3224 	if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
3225 	    (current->thread.regs->msr & MSR_TM)) {
3226 		if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
3227 			run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3228 			run->fail_entry.hardware_entry_failure_reason = 0;
3229 			return -EINVAL;
3230 		}
3231 		/* Enable TM so we can read the TM SPRs */
3232 		mtmsr(mfmsr() | MSR_TM);
3233 		current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
3234 		current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
3235 		current->thread.tm_texasr = mfspr(SPRN_TEXASR);
3236 		current->thread.regs->msr &= ~MSR_TM;
3237 	}
3238 #endif
3239 
3240 	kvmppc_core_prepare_to_enter(vcpu);
3241 
3242 	/* No need to go into the guest when all we'll do is come back out */
3243 	if (signal_pending(current)) {
3244 		run->exit_reason = KVM_EXIT_INTR;
3245 		return -EINTR;
3246 	}
3247 
3248 	atomic_inc(&vcpu->kvm->arch.vcpus_running);
3249 	/* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */
3250 	smp_mb();
3251 
3252 	/* On the first time here, set up HTAB and VRMA */
3253 	if (!kvm_is_radix(vcpu->kvm) && !vcpu->kvm->arch.hpte_setup_done) {
3254 		r = kvmppc_hv_setup_htab_rma(vcpu);
3255 		if (r)
3256 			goto out;
3257 	}
3258 
3259 	flush_all_to_thread(current);
3260 
3261 	/* Save userspace EBB and other register values */
3262 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
3263 		ebb_regs[0] = mfspr(SPRN_EBBHR);
3264 		ebb_regs[1] = mfspr(SPRN_EBBRR);
3265 		ebb_regs[2] = mfspr(SPRN_BESCR);
3266 		user_tar = mfspr(SPRN_TAR);
3267 	}
3268 	user_vrsave = mfspr(SPRN_VRSAVE);
3269 
3270 	vcpu->arch.wqp = &vcpu->arch.vcore->wq;
3271 	vcpu->arch.pgdir = current->mm->pgd;
3272 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
3273 
3274 	do {
3275 		r = kvmppc_run_vcpu(run, vcpu);
3276 
3277 		if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
3278 		    !(vcpu->arch.shregs.msr & MSR_PR)) {
3279 			trace_kvm_hcall_enter(vcpu);
3280 			r = kvmppc_pseries_do_hcall(vcpu);
3281 			trace_kvm_hcall_exit(vcpu, r);
3282 			kvmppc_core_prepare_to_enter(vcpu);
3283 		} else if (r == RESUME_PAGE_FAULT) {
3284 			srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3285 			r = kvmppc_book3s_hv_page_fault(run, vcpu,
3286 				vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
3287 			srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3288 		} else if (r == RESUME_PASSTHROUGH) {
3289 			if (WARN_ON(xive_enabled()))
3290 				r = H_SUCCESS;
3291 			else
3292 				r = kvmppc_xics_rm_complete(vcpu, 0);
3293 		}
3294 	} while (is_kvmppc_resume_guest(r));
3295 
3296 	/* Restore userspace EBB and other register values */
3297 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
3298 		mtspr(SPRN_EBBHR, ebb_regs[0]);
3299 		mtspr(SPRN_EBBRR, ebb_regs[1]);
3300 		mtspr(SPRN_BESCR, ebb_regs[2]);
3301 		mtspr(SPRN_TAR, user_tar);
3302 		mtspr(SPRN_FSCR, current->thread.fscr);
3303 	}
3304 	mtspr(SPRN_VRSAVE, user_vrsave);
3305 
3306  out:
3307 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
3308 	atomic_dec(&vcpu->kvm->arch.vcpus_running);
3309 	return r;
3310 }
3311 
3312 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
3313 				     int linux_psize)
3314 {
3315 	struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];
3316 
3317 	if (!def->shift)
3318 		return;
3319 	(*sps)->page_shift = def->shift;
3320 	(*sps)->slb_enc = def->sllp;
3321 	(*sps)->enc[0].page_shift = def->shift;
3322 	(*sps)->enc[0].pte_enc = def->penc[linux_psize];
3323 	/*
3324 	 * Add 16MB MPSS support if host supports it
3325 	 */
3326 	if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) {
3327 		(*sps)->enc[1].page_shift = 24;
3328 		(*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M];
3329 	}
3330 	(*sps)++;
3331 }
3332 
3333 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
3334 					 struct kvm_ppc_smmu_info *info)
3335 {
3336 	struct kvm_ppc_one_seg_page_size *sps;
3337 
3338 	/*
3339 	 * Since we don't yet support HPT guests on a radix host,
3340 	 * return an error if the host uses radix.
3341 	 */
3342 	if (radix_enabled())
3343 		return -EINVAL;
3344 
3345 	/*
3346 	 * POWER7, POWER8 and POWER9 all support 32 storage keys for data.
3347 	 * POWER7 doesn't support keys for instruction accesses,
3348 	 * POWER8 and POWER9 do.
3349 	 */
3350 	info->data_keys = 32;
3351 	info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
3352 
3353 	info->flags = KVM_PPC_PAGE_SIZES_REAL;
3354 	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
3355 		info->flags |= KVM_PPC_1T_SEGMENTS;
3356 	info->slb_size = mmu_slb_size;
3357 
3358 	/* We only support these sizes for now, and no muti-size segments */
3359 	sps = &info->sps[0];
3360 	kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K);
3361 	kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K);
3362 	kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M);
3363 
3364 	return 0;
3365 }
3366 
3367 /*
3368  * Get (and clear) the dirty memory log for a memory slot.
3369  */
3370 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
3371 					 struct kvm_dirty_log *log)
3372 {
3373 	struct kvm_memslots *slots;
3374 	struct kvm_memory_slot *memslot;
3375 	int i, r;
3376 	unsigned long n;
3377 	unsigned long *buf;
3378 	struct kvm_vcpu *vcpu;
3379 
3380 	mutex_lock(&kvm->slots_lock);
3381 
3382 	r = -EINVAL;
3383 	if (log->slot >= KVM_USER_MEM_SLOTS)
3384 		goto out;
3385 
3386 	slots = kvm_memslots(kvm);
3387 	memslot = id_to_memslot(slots, log->slot);
3388 	r = -ENOENT;
3389 	if (!memslot->dirty_bitmap)
3390 		goto out;
3391 
3392 	/*
3393 	 * Use second half of bitmap area because radix accumulates
3394 	 * bits in the first half.
3395 	 */
3396 	n = kvm_dirty_bitmap_bytes(memslot);
3397 	buf = memslot->dirty_bitmap + n / sizeof(long);
3398 	memset(buf, 0, n);
3399 
3400 	if (kvm_is_radix(kvm))
3401 		r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
3402 	else
3403 		r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
3404 	if (r)
3405 		goto out;
3406 
3407 	/* Harvest dirty bits from VPA and DTL updates */
3408 	/* Note: we never modify the SLB shadow buffer areas */
3409 	kvm_for_each_vcpu(i, vcpu, kvm) {
3410 		spin_lock(&vcpu->arch.vpa_update_lock);
3411 		kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
3412 		kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
3413 		spin_unlock(&vcpu->arch.vpa_update_lock);
3414 	}
3415 
3416 	r = -EFAULT;
3417 	if (copy_to_user(log->dirty_bitmap, buf, n))
3418 		goto out;
3419 
3420 	r = 0;
3421 out:
3422 	mutex_unlock(&kvm->slots_lock);
3423 	return r;
3424 }
3425 
3426 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
3427 					struct kvm_memory_slot *dont)
3428 {
3429 	if (!dont || free->arch.rmap != dont->arch.rmap) {
3430 		vfree(free->arch.rmap);
3431 		free->arch.rmap = NULL;
3432 	}
3433 }
3434 
3435 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
3436 					 unsigned long npages)
3437 {
3438 	/*
3439 	 * For now, if radix_enabled() then we only support radix guests,
3440 	 * and in that case we don't need the rmap array.
3441 	 */
3442 	if (radix_enabled()) {
3443 		slot->arch.rmap = NULL;
3444 		return 0;
3445 	}
3446 
3447 	slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
3448 	if (!slot->arch.rmap)
3449 		return -ENOMEM;
3450 
3451 	return 0;
3452 }
3453 
3454 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
3455 					struct kvm_memory_slot *memslot,
3456 					const struct kvm_userspace_memory_region *mem)
3457 {
3458 	return 0;
3459 }
3460 
3461 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
3462 				const struct kvm_userspace_memory_region *mem,
3463 				const struct kvm_memory_slot *old,
3464 				const struct kvm_memory_slot *new)
3465 {
3466 	unsigned long npages = mem->memory_size >> PAGE_SHIFT;
3467 	struct kvm_memslots *slots;
3468 	struct kvm_memory_slot *memslot;
3469 
3470 	/*
3471 	 * If we are making a new memslot, it might make
3472 	 * some address that was previously cached as emulated
3473 	 * MMIO be no longer emulated MMIO, so invalidate
3474 	 * all the caches of emulated MMIO translations.
3475 	 */
3476 	if (npages)
3477 		atomic64_inc(&kvm->arch.mmio_update);
3478 
3479 	if (npages && old->npages && !kvm_is_radix(kvm)) {
3480 		/*
3481 		 * If modifying a memslot, reset all the rmap dirty bits.
3482 		 * If this is a new memslot, we don't need to do anything
3483 		 * since the rmap array starts out as all zeroes,
3484 		 * i.e. no pages are dirty.
3485 		 */
3486 		slots = kvm_memslots(kvm);
3487 		memslot = id_to_memslot(slots, mem->slot);
3488 		kvmppc_hv_get_dirty_log_hpt(kvm, memslot, NULL);
3489 	}
3490 }
3491 
3492 /*
3493  * Update LPCR values in kvm->arch and in vcores.
3494  * Caller must hold kvm->lock.
3495  */
3496 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
3497 {
3498 	long int i;
3499 	u32 cores_done = 0;
3500 
3501 	if ((kvm->arch.lpcr & mask) == lpcr)
3502 		return;
3503 
3504 	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
3505 
3506 	for (i = 0; i < KVM_MAX_VCORES; ++i) {
3507 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];
3508 		if (!vc)
3509 			continue;
3510 		spin_lock(&vc->lock);
3511 		vc->lpcr = (vc->lpcr & ~mask) | lpcr;
3512 		spin_unlock(&vc->lock);
3513 		if (++cores_done >= kvm->arch.online_vcores)
3514 			break;
3515 	}
3516 }
3517 
3518 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
3519 {
3520 	return;
3521 }
3522 
3523 static void kvmppc_setup_partition_table(struct kvm *kvm)
3524 {
3525 	unsigned long dw0, dw1;
3526 
3527 	if (!kvm_is_radix(kvm)) {
3528 		/* PS field - page size for VRMA */
3529 		dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
3530 			((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
3531 		/* HTABSIZE and HTABORG fields */
3532 		dw0 |= kvm->arch.sdr1;
3533 
3534 		/* Second dword as set by userspace */
3535 		dw1 = kvm->arch.process_table;
3536 	} else {
3537 		dw0 = PATB_HR | radix__get_tree_size() |
3538 			__pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
3539 		dw1 = PATB_GR | kvm->arch.process_table;
3540 	}
3541 
3542 	mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1);
3543 }
3544 
3545 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
3546 {
3547 	int err = 0;
3548 	struct kvm *kvm = vcpu->kvm;
3549 	unsigned long hva;
3550 	struct kvm_memory_slot *memslot;
3551 	struct vm_area_struct *vma;
3552 	unsigned long lpcr = 0, senc;
3553 	unsigned long psize, porder;
3554 	int srcu_idx;
3555 
3556 	mutex_lock(&kvm->lock);
3557 	if (kvm->arch.hpte_setup_done)
3558 		goto out;	/* another vcpu beat us to it */
3559 
3560 	/* Allocate hashed page table (if not done already) and reset it */
3561 	if (!kvm->arch.hpt.virt) {
3562 		int order = KVM_DEFAULT_HPT_ORDER;
3563 		struct kvm_hpt_info info;
3564 
3565 		err = kvmppc_allocate_hpt(&info, order);
3566 		/* If we get here, it means userspace didn't specify a
3567 		 * size explicitly.  So, try successively smaller
3568 		 * sizes if the default failed. */
3569 		while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
3570 			err  = kvmppc_allocate_hpt(&info, order);
3571 
3572 		if (err < 0) {
3573 			pr_err("KVM: Couldn't alloc HPT\n");
3574 			goto out;
3575 		}
3576 
3577 		kvmppc_set_hpt(kvm, &info);
3578 	}
3579 
3580 	/* Look up the memslot for guest physical address 0 */
3581 	srcu_idx = srcu_read_lock(&kvm->srcu);
3582 	memslot = gfn_to_memslot(kvm, 0);
3583 
3584 	/* We must have some memory at 0 by now */
3585 	err = -EINVAL;
3586 	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
3587 		goto out_srcu;
3588 
3589 	/* Look up the VMA for the start of this memory slot */
3590 	hva = memslot->userspace_addr;
3591 	down_read(&current->mm->mmap_sem);
3592 	vma = find_vma(current->mm, hva);
3593 	if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
3594 		goto up_out;
3595 
3596 	psize = vma_kernel_pagesize(vma);
3597 	porder = __ilog2(psize);
3598 
3599 	up_read(&current->mm->mmap_sem);
3600 
3601 	/* We can handle 4k, 64k or 16M pages in the VRMA */
3602 	err = -EINVAL;
3603 	if (!(psize == 0x1000 || psize == 0x10000 ||
3604 	      psize == 0x1000000))
3605 		goto out_srcu;
3606 
3607 	senc = slb_pgsize_encoding(psize);
3608 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
3609 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
3610 	/* Create HPTEs in the hash page table for the VRMA */
3611 	kvmppc_map_vrma(vcpu, memslot, porder);
3612 
3613 	/* Update VRMASD field in the LPCR */
3614 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
3615 		/* the -4 is to account for senc values starting at 0x10 */
3616 		lpcr = senc << (LPCR_VRMASD_SH - 4);
3617 		kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
3618 	} else {
3619 		kvmppc_setup_partition_table(kvm);
3620 	}
3621 
3622 	/* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */
3623 	smp_wmb();
3624 	kvm->arch.hpte_setup_done = 1;
3625 	err = 0;
3626  out_srcu:
3627 	srcu_read_unlock(&kvm->srcu, srcu_idx);
3628  out:
3629 	mutex_unlock(&kvm->lock);
3630 	return err;
3631 
3632  up_out:
3633 	up_read(&current->mm->mmap_sem);
3634 	goto out_srcu;
3635 }
3636 
3637 #ifdef CONFIG_KVM_XICS
3638 /*
3639  * Allocate a per-core structure for managing state about which cores are
3640  * running in the host versus the guest and for exchanging data between
3641  * real mode KVM and CPU running in the host.
3642  * This is only done for the first VM.
3643  * The allocated structure stays even if all VMs have stopped.
3644  * It is only freed when the kvm-hv module is unloaded.
3645  * It's OK for this routine to fail, we just don't support host
3646  * core operations like redirecting H_IPI wakeups.
3647  */
3648 void kvmppc_alloc_host_rm_ops(void)
3649 {
3650 	struct kvmppc_host_rm_ops *ops;
3651 	unsigned long l_ops;
3652 	int cpu, core;
3653 	int size;
3654 
3655 	/* Not the first time here ? */
3656 	if (kvmppc_host_rm_ops_hv != NULL)
3657 		return;
3658 
3659 	ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
3660 	if (!ops)
3661 		return;
3662 
3663 	size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
3664 	ops->rm_core = kzalloc(size, GFP_KERNEL);
3665 
3666 	if (!ops->rm_core) {
3667 		kfree(ops);
3668 		return;
3669 	}
3670 
3671 	cpus_read_lock();
3672 
3673 	for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
3674 		if (!cpu_online(cpu))
3675 			continue;
3676 
3677 		core = cpu >> threads_shift;
3678 		ops->rm_core[core].rm_state.in_host = 1;
3679 	}
3680 
3681 	ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
3682 
3683 	/*
3684 	 * Make the contents of the kvmppc_host_rm_ops structure visible
3685 	 * to other CPUs before we assign it to the global variable.
3686 	 * Do an atomic assignment (no locks used here), but if someone
3687 	 * beats us to it, just free our copy and return.
3688 	 */
3689 	smp_wmb();
3690 	l_ops = (unsigned long) ops;
3691 
3692 	if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
3693 		cpus_read_unlock();
3694 		kfree(ops->rm_core);
3695 		kfree(ops);
3696 		return;
3697 	}
3698 
3699 	cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
3700 					     "ppc/kvm_book3s:prepare",
3701 					     kvmppc_set_host_core,
3702 					     kvmppc_clear_host_core);
3703 	cpus_read_unlock();
3704 }
3705 
3706 void kvmppc_free_host_rm_ops(void)
3707 {
3708 	if (kvmppc_host_rm_ops_hv) {
3709 		cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
3710 		kfree(kvmppc_host_rm_ops_hv->rm_core);
3711 		kfree(kvmppc_host_rm_ops_hv);
3712 		kvmppc_host_rm_ops_hv = NULL;
3713 	}
3714 }
3715 #endif
3716 
3717 static int kvmppc_core_init_vm_hv(struct kvm *kvm)
3718 {
3719 	unsigned long lpcr, lpid;
3720 	char buf[32];
3721 	int ret;
3722 
3723 	/* Allocate the guest's logical partition ID */
3724 
3725 	lpid = kvmppc_alloc_lpid();
3726 	if ((long)lpid < 0)
3727 		return -ENOMEM;
3728 	kvm->arch.lpid = lpid;
3729 
3730 	kvmppc_alloc_host_rm_ops();
3731 
3732 	/*
3733 	 * Since we don't flush the TLB when tearing down a VM,
3734 	 * and this lpid might have previously been used,
3735 	 * make sure we flush on each core before running the new VM.
3736 	 * On POWER9, the tlbie in mmu_partition_table_set_entry()
3737 	 * does this flush for us.
3738 	 */
3739 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
3740 		cpumask_setall(&kvm->arch.need_tlb_flush);
3741 
3742 	/* Start out with the default set of hcalls enabled */
3743 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
3744 	       sizeof(kvm->arch.enabled_hcalls));
3745 
3746 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
3747 		kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
3748 
3749 	/* Init LPCR for virtual RMA mode */
3750 	kvm->arch.host_lpid = mfspr(SPRN_LPID);
3751 	kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
3752 	lpcr &= LPCR_PECE | LPCR_LPES;
3753 	lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
3754 		LPCR_VPM0 | LPCR_VPM1;
3755 	kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
3756 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
3757 	/* On POWER8 turn on online bit to enable PURR/SPURR */
3758 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
3759 		lpcr |= LPCR_ONL;
3760 	/*
3761 	 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed)
3762 	 * Set HVICE bit to enable hypervisor virtualization interrupts.
3763 	 * Set HEIC to prevent OS interrupts to go to hypervisor (should
3764 	 * be unnecessary but better safe than sorry in case we re-enable
3765 	 * EE in HV mode with this LPCR still set)
3766 	 */
3767 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
3768 		lpcr &= ~LPCR_VPM0;
3769 		lpcr |= LPCR_HVICE | LPCR_HEIC;
3770 
3771 		/*
3772 		 * If xive is enabled, we route 0x500 interrupts directly
3773 		 * to the guest.
3774 		 */
3775 		if (xive_enabled())
3776 			lpcr |= LPCR_LPES;
3777 	}
3778 
3779 	/*
3780 	 * For now, if the host uses radix, the guest must be radix.
3781 	 */
3782 	if (radix_enabled()) {
3783 		kvm->arch.radix = 1;
3784 		lpcr &= ~LPCR_VPM1;
3785 		lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
3786 		ret = kvmppc_init_vm_radix(kvm);
3787 		if (ret) {
3788 			kvmppc_free_lpid(kvm->arch.lpid);
3789 			return ret;
3790 		}
3791 		kvmppc_setup_partition_table(kvm);
3792 	}
3793 
3794 	kvm->arch.lpcr = lpcr;
3795 
3796 	/* Initialization for future HPT resizes */
3797 	kvm->arch.resize_hpt = NULL;
3798 
3799 	/*
3800 	 * Work out how many sets the TLB has, for the use of
3801 	 * the TLB invalidation loop in book3s_hv_rmhandlers.S.
3802 	 */
3803 	if (kvm_is_radix(kvm))
3804 		kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;	/* 128 */
3805 	else if (cpu_has_feature(CPU_FTR_ARCH_300))
3806 		kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;	/* 256 */
3807 	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
3808 		kvm->arch.tlb_sets = POWER8_TLB_SETS;		/* 512 */
3809 	else
3810 		kvm->arch.tlb_sets = POWER7_TLB_SETS;		/* 128 */
3811 
3812 	/*
3813 	 * Track that we now have a HV mode VM active. This blocks secondary
3814 	 * CPU threads from coming online.
3815 	 * On POWER9, we only need to do this for HPT guests on a radix
3816 	 * host, which is not yet supported.
3817 	 */
3818 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
3819 		kvm_hv_vm_activated();
3820 
3821 	/*
3822 	 * Initialize smt_mode depending on processor.
3823 	 * POWER8 and earlier have to use "strict" threading, where
3824 	 * all vCPUs in a vcore have to run on the same (sub)core,
3825 	 * whereas on POWER9 the threads can each run a different
3826 	 * guest.
3827 	 */
3828 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
3829 		kvm->arch.smt_mode = threads_per_subcore;
3830 	else
3831 		kvm->arch.smt_mode = 1;
3832 	kvm->arch.emul_smt_mode = 1;
3833 
3834 	/*
3835 	 * Create a debugfs directory for the VM
3836 	 */
3837 	snprintf(buf, sizeof(buf), "vm%d", current->pid);
3838 	kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
3839 	if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
3840 		kvmppc_mmu_debugfs_init(kvm);
3841 
3842 	return 0;
3843 }
3844 
3845 static void kvmppc_free_vcores(struct kvm *kvm)
3846 {
3847 	long int i;
3848 
3849 	for (i = 0; i < KVM_MAX_VCORES; ++i)
3850 		kfree(kvm->arch.vcores[i]);
3851 	kvm->arch.online_vcores = 0;
3852 }
3853 
3854 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
3855 {
3856 	debugfs_remove_recursive(kvm->arch.debugfs_dir);
3857 
3858 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
3859 		kvm_hv_vm_deactivated();
3860 
3861 	kvmppc_free_vcores(kvm);
3862 
3863 	kvmppc_free_lpid(kvm->arch.lpid);
3864 
3865 	if (kvm_is_radix(kvm))
3866 		kvmppc_free_radix(kvm);
3867 	else
3868 		kvmppc_free_hpt(&kvm->arch.hpt);
3869 
3870 	kvmppc_free_pimap(kvm);
3871 }
3872 
3873 /* We don't need to emulate any privileged instructions or dcbz */
3874 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
3875 				     unsigned int inst, int *advance)
3876 {
3877 	return EMULATE_FAIL;
3878 }
3879 
3880 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
3881 					ulong spr_val)
3882 {
3883 	return EMULATE_FAIL;
3884 }
3885 
3886 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
3887 					ulong *spr_val)
3888 {
3889 	return EMULATE_FAIL;
3890 }
3891 
3892 static int kvmppc_core_check_processor_compat_hv(void)
3893 {
3894 	if (!cpu_has_feature(CPU_FTR_HVMODE) ||
3895 	    !cpu_has_feature(CPU_FTR_ARCH_206))
3896 		return -EIO;
3897 
3898 	return 0;
3899 }
3900 
3901 #ifdef CONFIG_KVM_XICS
3902 
3903 void kvmppc_free_pimap(struct kvm *kvm)
3904 {
3905 	kfree(kvm->arch.pimap);
3906 }
3907 
3908 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
3909 {
3910 	return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
3911 }
3912 
3913 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
3914 {
3915 	struct irq_desc *desc;
3916 	struct kvmppc_irq_map *irq_map;
3917 	struct kvmppc_passthru_irqmap *pimap;
3918 	struct irq_chip *chip;
3919 	int i, rc = 0;
3920 
3921 	if (!kvm_irq_bypass)
3922 		return 1;
3923 
3924 	desc = irq_to_desc(host_irq);
3925 	if (!desc)
3926 		return -EIO;
3927 
3928 	mutex_lock(&kvm->lock);
3929 
3930 	pimap = kvm->arch.pimap;
3931 	if (pimap == NULL) {
3932 		/* First call, allocate structure to hold IRQ map */
3933 		pimap = kvmppc_alloc_pimap();
3934 		if (pimap == NULL) {
3935 			mutex_unlock(&kvm->lock);
3936 			return -ENOMEM;
3937 		}
3938 		kvm->arch.pimap = pimap;
3939 	}
3940 
3941 	/*
3942 	 * For now, we only support interrupts for which the EOI operation
3943 	 * is an OPAL call followed by a write to XIRR, since that's
3944 	 * what our real-mode EOI code does, or a XIVE interrupt
3945 	 */
3946 	chip = irq_data_get_irq_chip(&desc->irq_data);
3947 	if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) {
3948 		pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
3949 			host_irq, guest_gsi);
3950 		mutex_unlock(&kvm->lock);
3951 		return -ENOENT;
3952 	}
3953 
3954 	/*
3955 	 * See if we already have an entry for this guest IRQ number.
3956 	 * If it's mapped to a hardware IRQ number, that's an error,
3957 	 * otherwise re-use this entry.
3958 	 */
3959 	for (i = 0; i < pimap->n_mapped; i++) {
3960 		if (guest_gsi == pimap->mapped[i].v_hwirq) {
3961 			if (pimap->mapped[i].r_hwirq) {
3962 				mutex_unlock(&kvm->lock);
3963 				return -EINVAL;
3964 			}
3965 			break;
3966 		}
3967 	}
3968 
3969 	if (i == KVMPPC_PIRQ_MAPPED) {
3970 		mutex_unlock(&kvm->lock);
3971 		return -EAGAIN;		/* table is full */
3972 	}
3973 
3974 	irq_map = &pimap->mapped[i];
3975 
3976 	irq_map->v_hwirq = guest_gsi;
3977 	irq_map->desc = desc;
3978 
3979 	/*
3980 	 * Order the above two stores before the next to serialize with
3981 	 * the KVM real mode handler.
3982 	 */
3983 	smp_wmb();
3984 	irq_map->r_hwirq = desc->irq_data.hwirq;
3985 
3986 	if (i == pimap->n_mapped)
3987 		pimap->n_mapped++;
3988 
3989 	if (xive_enabled())
3990 		rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc);
3991 	else
3992 		kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq);
3993 	if (rc)
3994 		irq_map->r_hwirq = 0;
3995 
3996 	mutex_unlock(&kvm->lock);
3997 
3998 	return 0;
3999 }
4000 
4001 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
4002 {
4003 	struct irq_desc *desc;
4004 	struct kvmppc_passthru_irqmap *pimap;
4005 	int i, rc = 0;
4006 
4007 	if (!kvm_irq_bypass)
4008 		return 0;
4009 
4010 	desc = irq_to_desc(host_irq);
4011 	if (!desc)
4012 		return -EIO;
4013 
4014 	mutex_lock(&kvm->lock);
4015 	if (!kvm->arch.pimap)
4016 		goto unlock;
4017 
4018 	pimap = kvm->arch.pimap;
4019 
4020 	for (i = 0; i < pimap->n_mapped; i++) {
4021 		if (guest_gsi == pimap->mapped[i].v_hwirq)
4022 			break;
4023 	}
4024 
4025 	if (i == pimap->n_mapped) {
4026 		mutex_unlock(&kvm->lock);
4027 		return -ENODEV;
4028 	}
4029 
4030 	if (xive_enabled())
4031 		rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc);
4032 	else
4033 		kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
4034 
4035 	/* invalidate the entry (what do do on error from the above ?) */
4036 	pimap->mapped[i].r_hwirq = 0;
4037 
4038 	/*
4039 	 * We don't free this structure even when the count goes to
4040 	 * zero. The structure is freed when we destroy the VM.
4041 	 */
4042  unlock:
4043 	mutex_unlock(&kvm->lock);
4044 	return rc;
4045 }
4046 
4047 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
4048 					     struct irq_bypass_producer *prod)
4049 {
4050 	int ret = 0;
4051 	struct kvm_kernel_irqfd *irqfd =
4052 		container_of(cons, struct kvm_kernel_irqfd, consumer);
4053 
4054 	irqfd->producer = prod;
4055 
4056 	ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
4057 	if (ret)
4058 		pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
4059 			prod->irq, irqfd->gsi, ret);
4060 
4061 	return ret;
4062 }
4063 
4064 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
4065 					      struct irq_bypass_producer *prod)
4066 {
4067 	int ret;
4068 	struct kvm_kernel_irqfd *irqfd =
4069 		container_of(cons, struct kvm_kernel_irqfd, consumer);
4070 
4071 	irqfd->producer = NULL;
4072 
4073 	/*
4074 	 * When producer of consumer is unregistered, we change back to
4075 	 * default external interrupt handling mode - KVM real mode
4076 	 * will switch back to host.
4077 	 */
4078 	ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
4079 	if (ret)
4080 		pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
4081 			prod->irq, irqfd->gsi, ret);
4082 }
4083 #endif
4084 
4085 static long kvm_arch_vm_ioctl_hv(struct file *filp,
4086 				 unsigned int ioctl, unsigned long arg)
4087 {
4088 	struct kvm *kvm __maybe_unused = filp->private_data;
4089 	void __user *argp = (void __user *)arg;
4090 	long r;
4091 
4092 	switch (ioctl) {
4093 
4094 	case KVM_PPC_ALLOCATE_HTAB: {
4095 		u32 htab_order;
4096 
4097 		r = -EFAULT;
4098 		if (get_user(htab_order, (u32 __user *)argp))
4099 			break;
4100 		r = kvmppc_alloc_reset_hpt(kvm, htab_order);
4101 		if (r)
4102 			break;
4103 		r = 0;
4104 		break;
4105 	}
4106 
4107 	case KVM_PPC_GET_HTAB_FD: {
4108 		struct kvm_get_htab_fd ghf;
4109 
4110 		r = -EFAULT;
4111 		if (copy_from_user(&ghf, argp, sizeof(ghf)))
4112 			break;
4113 		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
4114 		break;
4115 	}
4116 
4117 	case KVM_PPC_RESIZE_HPT_PREPARE: {
4118 		struct kvm_ppc_resize_hpt rhpt;
4119 
4120 		r = -EFAULT;
4121 		if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
4122 			break;
4123 
4124 		r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
4125 		break;
4126 	}
4127 
4128 	case KVM_PPC_RESIZE_HPT_COMMIT: {
4129 		struct kvm_ppc_resize_hpt rhpt;
4130 
4131 		r = -EFAULT;
4132 		if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
4133 			break;
4134 
4135 		r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
4136 		break;
4137 	}
4138 
4139 	default:
4140 		r = -ENOTTY;
4141 	}
4142 
4143 	return r;
4144 }
4145 
4146 /*
4147  * List of hcall numbers to enable by default.
4148  * For compatibility with old userspace, we enable by default
4149  * all hcalls that were implemented before the hcall-enabling
4150  * facility was added.  Note this list should not include H_RTAS.
4151  */
4152 static unsigned int default_hcall_list[] = {
4153 	H_REMOVE,
4154 	H_ENTER,
4155 	H_READ,
4156 	H_PROTECT,
4157 	H_BULK_REMOVE,
4158 	H_GET_TCE,
4159 	H_PUT_TCE,
4160 	H_SET_DABR,
4161 	H_SET_XDABR,
4162 	H_CEDE,
4163 	H_PROD,
4164 	H_CONFER,
4165 	H_REGISTER_VPA,
4166 #ifdef CONFIG_KVM_XICS
4167 	H_EOI,
4168 	H_CPPR,
4169 	H_IPI,
4170 	H_IPOLL,
4171 	H_XIRR,
4172 	H_XIRR_X,
4173 #endif
4174 	0
4175 };
4176 
4177 static void init_default_hcalls(void)
4178 {
4179 	int i;
4180 	unsigned int hcall;
4181 
4182 	for (i = 0; default_hcall_list[i]; ++i) {
4183 		hcall = default_hcall_list[i];
4184 		WARN_ON(!kvmppc_hcall_impl_hv(hcall));
4185 		__set_bit(hcall / 4, default_enabled_hcalls);
4186 	}
4187 }
4188 
4189 static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
4190 {
4191 	unsigned long lpcr;
4192 	int radix;
4193 
4194 	/* If not on a POWER9, reject it */
4195 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
4196 		return -ENODEV;
4197 
4198 	/* If any unknown flags set, reject it */
4199 	if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
4200 		return -EINVAL;
4201 
4202 	/* We can't change a guest to/from radix yet */
4203 	radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
4204 	if (radix != kvm_is_radix(kvm))
4205 		return -EINVAL;
4206 
4207 	/* GR (guest radix) bit in process_table field must match */
4208 	if (!!(cfg->process_table & PATB_GR) != radix)
4209 		return -EINVAL;
4210 
4211 	/* Process table size field must be reasonable, i.e. <= 24 */
4212 	if ((cfg->process_table & PRTS_MASK) > 24)
4213 		return -EINVAL;
4214 
4215 	mutex_lock(&kvm->lock);
4216 	kvm->arch.process_table = cfg->process_table;
4217 	kvmppc_setup_partition_table(kvm);
4218 
4219 	lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
4220 	kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
4221 	mutex_unlock(&kvm->lock);
4222 
4223 	return 0;
4224 }
4225 
4226 static struct kvmppc_ops kvm_ops_hv = {
4227 	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
4228 	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
4229 	.get_one_reg = kvmppc_get_one_reg_hv,
4230 	.set_one_reg = kvmppc_set_one_reg_hv,
4231 	.vcpu_load   = kvmppc_core_vcpu_load_hv,
4232 	.vcpu_put    = kvmppc_core_vcpu_put_hv,
4233 	.set_msr     = kvmppc_set_msr_hv,
4234 	.vcpu_run    = kvmppc_vcpu_run_hv,
4235 	.vcpu_create = kvmppc_core_vcpu_create_hv,
4236 	.vcpu_free   = kvmppc_core_vcpu_free_hv,
4237 	.check_requests = kvmppc_core_check_requests_hv,
4238 	.get_dirty_log  = kvm_vm_ioctl_get_dirty_log_hv,
4239 	.flush_memslot  = kvmppc_core_flush_memslot_hv,
4240 	.prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
4241 	.commit_memory_region  = kvmppc_core_commit_memory_region_hv,
4242 	.unmap_hva = kvm_unmap_hva_hv,
4243 	.unmap_hva_range = kvm_unmap_hva_range_hv,
4244 	.age_hva  = kvm_age_hva_hv,
4245 	.test_age_hva = kvm_test_age_hva_hv,
4246 	.set_spte_hva = kvm_set_spte_hva_hv,
4247 	.mmu_destroy  = kvmppc_mmu_destroy_hv,
4248 	.free_memslot = kvmppc_core_free_memslot_hv,
4249 	.create_memslot = kvmppc_core_create_memslot_hv,
4250 	.init_vm =  kvmppc_core_init_vm_hv,
4251 	.destroy_vm = kvmppc_core_destroy_vm_hv,
4252 	.get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
4253 	.emulate_op = kvmppc_core_emulate_op_hv,
4254 	.emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
4255 	.emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
4256 	.fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
4257 	.arch_vm_ioctl  = kvm_arch_vm_ioctl_hv,
4258 	.hcall_implemented = kvmppc_hcall_impl_hv,
4259 #ifdef CONFIG_KVM_XICS
4260 	.irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
4261 	.irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
4262 #endif
4263 	.configure_mmu = kvmhv_configure_mmu,
4264 	.get_rmmu_info = kvmhv_get_rmmu_info,
4265 	.set_smt_mode = kvmhv_set_smt_mode,
4266 };
4267 
4268 static int kvm_init_subcore_bitmap(void)
4269 {
4270 	int i, j;
4271 	int nr_cores = cpu_nr_cores();
4272 	struct sibling_subcore_state *sibling_subcore_state;
4273 
4274 	for (i = 0; i < nr_cores; i++) {
4275 		int first_cpu = i * threads_per_core;
4276 		int node = cpu_to_node(first_cpu);
4277 
4278 		/* Ignore if it is already allocated. */
4279 		if (paca[first_cpu].sibling_subcore_state)
4280 			continue;
4281 
4282 		sibling_subcore_state =
4283 			kmalloc_node(sizeof(struct sibling_subcore_state),
4284 							GFP_KERNEL, node);
4285 		if (!sibling_subcore_state)
4286 			return -ENOMEM;
4287 
4288 		memset(sibling_subcore_state, 0,
4289 				sizeof(struct sibling_subcore_state));
4290 
4291 		for (j = 0; j < threads_per_core; j++) {
4292 			int cpu = first_cpu + j;
4293 
4294 			paca[cpu].sibling_subcore_state = sibling_subcore_state;
4295 		}
4296 	}
4297 	return 0;
4298 }
4299 
4300 static int kvmppc_radix_possible(void)
4301 {
4302 	return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
4303 }
4304 
4305 static int kvmppc_book3s_init_hv(void)
4306 {
4307 	int r;
4308 	/*
4309 	 * FIXME!! Do we need to check on all cpus ?
4310 	 */
4311 	r = kvmppc_core_check_processor_compat_hv();
4312 	if (r < 0)
4313 		return -ENODEV;
4314 
4315 	r = kvm_init_subcore_bitmap();
4316 	if (r)
4317 		return r;
4318 
4319 	/*
4320 	 * We need a way of accessing the XICS interrupt controller,
4321 	 * either directly, via paca[cpu].kvm_hstate.xics_phys, or
4322 	 * indirectly, via OPAL.
4323 	 */
4324 #ifdef CONFIG_SMP
4325 	if (!xive_enabled() && !local_paca->kvm_hstate.xics_phys) {
4326 		struct device_node *np;
4327 
4328 		np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
4329 		if (!np) {
4330 			pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
4331 			return -ENODEV;
4332 		}
4333 	}
4334 #endif
4335 
4336 	kvm_ops_hv.owner = THIS_MODULE;
4337 	kvmppc_hv_ops = &kvm_ops_hv;
4338 
4339 	init_default_hcalls();
4340 
4341 	init_vcore_lists();
4342 
4343 	r = kvmppc_mmu_hv_init();
4344 	if (r)
4345 		return r;
4346 
4347 	if (kvmppc_radix_possible())
4348 		r = kvmppc_radix_init();
4349 	return r;
4350 }
4351 
4352 static void kvmppc_book3s_exit_hv(void)
4353 {
4354 	kvmppc_free_host_rm_ops();
4355 	if (kvmppc_radix_possible())
4356 		kvmppc_radix_exit();
4357 	kvmppc_hv_ops = NULL;
4358 }
4359 
4360 module_init(kvmppc_book3s_init_hv);
4361 module_exit(kvmppc_book3s_exit_hv);
4362 MODULE_LICENSE("GPL");
4363 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4364 MODULE_ALIAS("devname:kvm");
4365 
4366