xref: /openbmc/linux/arch/powerpc/kvm/book3s_hv.c (revision 94588c1b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
4  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
5  *
6  * Authors:
7  *    Paul Mackerras <paulus@au1.ibm.com>
8  *    Alexander Graf <agraf@suse.de>
9  *    Kevin Wolf <mail@kevin-wolf.de>
10  *
11  * Description: KVM functions specific to running on Book 3S
12  * processors in hypervisor mode (specifically POWER7 and later).
13  *
14  * This file is derived from arch/powerpc/kvm/book3s.c,
15  * by Alexander Graf <agraf@suse.de>.
16  */
17 
18 #include <linux/kvm_host.h>
19 #include <linux/kernel.h>
20 #include <linux/err.h>
21 #include <linux/slab.h>
22 #include <linux/preempt.h>
23 #include <linux/sched/signal.h>
24 #include <linux/sched/stat.h>
25 #include <linux/delay.h>
26 #include <linux/export.h>
27 #include <linux/fs.h>
28 #include <linux/anon_inodes.h>
29 #include <linux/cpu.h>
30 #include <linux/cpumask.h>
31 #include <linux/spinlock.h>
32 #include <linux/page-flags.h>
33 #include <linux/srcu.h>
34 #include <linux/miscdevice.h>
35 #include <linux/debugfs.h>
36 #include <linux/gfp.h>
37 #include <linux/vmalloc.h>
38 #include <linux/highmem.h>
39 #include <linux/hugetlb.h>
40 #include <linux/kvm_irqfd.h>
41 #include <linux/irqbypass.h>
42 #include <linux/module.h>
43 #include <linux/compiler.h>
44 #include <linux/of.h>
45 
46 #include <asm/ftrace.h>
47 #include <asm/reg.h>
48 #include <asm/ppc-opcode.h>
49 #include <asm/asm-prototypes.h>
50 #include <asm/archrandom.h>
51 #include <asm/debug.h>
52 #include <asm/disassemble.h>
53 #include <asm/cputable.h>
54 #include <asm/cacheflush.h>
55 #include <linux/uaccess.h>
56 #include <asm/interrupt.h>
57 #include <asm/io.h>
58 #include <asm/kvm_ppc.h>
59 #include <asm/kvm_book3s.h>
60 #include <asm/mmu_context.h>
61 #include <asm/lppaca.h>
62 #include <asm/processor.h>
63 #include <asm/cputhreads.h>
64 #include <asm/page.h>
65 #include <asm/hvcall.h>
66 #include <asm/switch_to.h>
67 #include <asm/smp.h>
68 #include <asm/dbell.h>
69 #include <asm/hmi.h>
70 #include <asm/pnv-pci.h>
71 #include <asm/mmu.h>
72 #include <asm/opal.h>
73 #include <asm/xics.h>
74 #include <asm/xive.h>
75 #include <asm/hw_breakpoint.h>
76 #include <asm/kvm_book3s_uvmem.h>
77 #include <asm/ultravisor.h>
78 #include <asm/dtl.h>
79 
80 #include "book3s.h"
81 
82 #define CREATE_TRACE_POINTS
83 #include "trace_hv.h"
84 
85 /* #define EXIT_DEBUG */
86 /* #define EXIT_DEBUG_SIMPLE */
87 /* #define EXIT_DEBUG_INT */
88 
89 /* Used to indicate that a guest page fault needs to be handled */
90 #define RESUME_PAGE_FAULT	(RESUME_GUEST | RESUME_FLAG_ARCH1)
91 /* Used to indicate that a guest passthrough interrupt needs to be handled */
92 #define RESUME_PASSTHROUGH	(RESUME_GUEST | RESUME_FLAG_ARCH2)
93 
94 /* Used as a "null" value for timebase values */
95 #define TB_NIL	(~(u64)0)
96 
97 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
98 
99 static int dynamic_mt_modes = 6;
100 module_param(dynamic_mt_modes, int, 0644);
101 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
102 static int target_smt_mode;
103 module_param(target_smt_mode, int, 0644);
104 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
105 
106 static bool indep_threads_mode = true;
107 module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR);
108 MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)");
109 
110 static bool one_vm_per_core;
111 module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
112 MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires indep_threads_mode=N)");
113 
114 #ifdef CONFIG_KVM_XICS
115 static const struct kernel_param_ops module_param_ops = {
116 	.set = param_set_int,
117 	.get = param_get_int,
118 };
119 
120 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
121 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
122 
123 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
124 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
125 #endif
126 
127 /* If set, guests are allowed to create and control nested guests */
128 static bool nested = true;
129 module_param(nested, bool, S_IRUGO | S_IWUSR);
130 MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
131 
132 static inline bool nesting_enabled(struct kvm *kvm)
133 {
134 	return kvm->arch.nested_enable && kvm_is_radix(kvm);
135 }
136 
137 /* If set, the threads on each CPU core have to be in the same MMU mode */
138 static bool no_mixing_hpt_and_radix __read_mostly;
139 
140 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
141 
142 /*
143  * RWMR values for POWER8.  These control the rate at which PURR
144  * and SPURR count and should be set according to the number of
145  * online threads in the vcore being run.
146  */
147 #define RWMR_RPA_P8_1THREAD	0x164520C62609AECAUL
148 #define RWMR_RPA_P8_2THREAD	0x7FFF2908450D8DA9UL
149 #define RWMR_RPA_P8_3THREAD	0x164520C62609AECAUL
150 #define RWMR_RPA_P8_4THREAD	0x199A421245058DA9UL
151 #define RWMR_RPA_P8_5THREAD	0x164520C62609AECAUL
152 #define RWMR_RPA_P8_6THREAD	0x164520C62609AECAUL
153 #define RWMR_RPA_P8_7THREAD	0x164520C62609AECAUL
154 #define RWMR_RPA_P8_8THREAD	0x164520C62609AECAUL
155 
156 static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
157 	RWMR_RPA_P8_1THREAD,
158 	RWMR_RPA_P8_1THREAD,
159 	RWMR_RPA_P8_2THREAD,
160 	RWMR_RPA_P8_3THREAD,
161 	RWMR_RPA_P8_4THREAD,
162 	RWMR_RPA_P8_5THREAD,
163 	RWMR_RPA_P8_6THREAD,
164 	RWMR_RPA_P8_7THREAD,
165 	RWMR_RPA_P8_8THREAD,
166 };
167 
168 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
169 		int *ip)
170 {
171 	int i = *ip;
172 	struct kvm_vcpu *vcpu;
173 
174 	while (++i < MAX_SMT_THREADS) {
175 		vcpu = READ_ONCE(vc->runnable_threads[i]);
176 		if (vcpu) {
177 			*ip = i;
178 			return vcpu;
179 		}
180 	}
181 	return NULL;
182 }
183 
184 /* Used to traverse the list of runnable threads for a given vcore */
185 #define for_each_runnable_thread(i, vcpu, vc) \
186 	for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
187 
188 static bool kvmppc_ipi_thread(int cpu)
189 {
190 	unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
191 
192 	/* If we're a nested hypervisor, fall back to ordinary IPIs for now */
193 	if (kvmhv_on_pseries())
194 		return false;
195 
196 	/* On POWER9 we can use msgsnd to IPI any cpu */
197 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
198 		msg |= get_hard_smp_processor_id(cpu);
199 		smp_mb();
200 		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
201 		return true;
202 	}
203 
204 	/* On POWER8 for IPIs to threads in the same core, use msgsnd */
205 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
206 		preempt_disable();
207 		if (cpu_first_thread_sibling(cpu) ==
208 		    cpu_first_thread_sibling(smp_processor_id())) {
209 			msg |= cpu_thread_in_core(cpu);
210 			smp_mb();
211 			__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
212 			preempt_enable();
213 			return true;
214 		}
215 		preempt_enable();
216 	}
217 
218 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
219 	if (cpu >= 0 && cpu < nr_cpu_ids) {
220 		if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
221 			xics_wake_cpu(cpu);
222 			return true;
223 		}
224 		opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
225 		return true;
226 	}
227 #endif
228 
229 	return false;
230 }
231 
232 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
233 {
234 	int cpu;
235 	struct rcuwait *waitp;
236 
237 	waitp = kvm_arch_vcpu_get_wait(vcpu);
238 	if (rcuwait_wake_up(waitp))
239 		++vcpu->stat.halt_wakeup;
240 
241 	cpu = READ_ONCE(vcpu->arch.thread_cpu);
242 	if (cpu >= 0 && kvmppc_ipi_thread(cpu))
243 		return;
244 
245 	/* CPU points to the first thread of the core */
246 	cpu = vcpu->cpu;
247 	if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
248 		smp_send_reschedule(cpu);
249 }
250 
251 /*
252  * We use the vcpu_load/put functions to measure stolen time.
253  * Stolen time is counted as time when either the vcpu is able to
254  * run as part of a virtual core, but the task running the vcore
255  * is preempted or sleeping, or when the vcpu needs something done
256  * in the kernel by the task running the vcpu, but that task is
257  * preempted or sleeping.  Those two things have to be counted
258  * separately, since one of the vcpu tasks will take on the job
259  * of running the core, and the other vcpu tasks in the vcore will
260  * sleep waiting for it to do that, but that sleep shouldn't count
261  * as stolen time.
262  *
263  * Hence we accumulate stolen time when the vcpu can run as part of
264  * a vcore using vc->stolen_tb, and the stolen time when the vcpu
265  * needs its task to do other things in the kernel (for example,
266  * service a page fault) in busy_stolen.  We don't accumulate
267  * stolen time for a vcore when it is inactive, or for a vcpu
268  * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
269  * a misnomer; it means that the vcpu task is not executing in
270  * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
271  * the kernel.  We don't have any way of dividing up that time
272  * between time that the vcpu is genuinely stopped, time that
273  * the task is actively working on behalf of the vcpu, and time
274  * that the task is preempted, so we don't count any of it as
275  * stolen.
276  *
277  * Updates to busy_stolen are protected by arch.tbacct_lock;
278  * updates to vc->stolen_tb are protected by the vcore->stoltb_lock
279  * lock.  The stolen times are measured in units of timebase ticks.
280  * (Note that the != TB_NIL checks below are purely defensive;
281  * they should never fail.)
282  */
283 
284 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
285 {
286 	unsigned long flags;
287 
288 	spin_lock_irqsave(&vc->stoltb_lock, flags);
289 	vc->preempt_tb = mftb();
290 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
291 }
292 
293 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
294 {
295 	unsigned long flags;
296 
297 	spin_lock_irqsave(&vc->stoltb_lock, flags);
298 	if (vc->preempt_tb != TB_NIL) {
299 		vc->stolen_tb += mftb() - vc->preempt_tb;
300 		vc->preempt_tb = TB_NIL;
301 	}
302 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
303 }
304 
305 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
306 {
307 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
308 	unsigned long flags;
309 
310 	/*
311 	 * We can test vc->runner without taking the vcore lock,
312 	 * because only this task ever sets vc->runner to this
313 	 * vcpu, and once it is set to this vcpu, only this task
314 	 * ever sets it to NULL.
315 	 */
316 	if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
317 		kvmppc_core_end_stolen(vc);
318 
319 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
320 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
321 	    vcpu->arch.busy_preempt != TB_NIL) {
322 		vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
323 		vcpu->arch.busy_preempt = TB_NIL;
324 	}
325 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
326 }
327 
328 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
329 {
330 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
331 	unsigned long flags;
332 
333 	if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
334 		kvmppc_core_start_stolen(vc);
335 
336 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
337 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
338 		vcpu->arch.busy_preempt = mftb();
339 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
340 }
341 
342 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
343 {
344 	vcpu->arch.pvr = pvr;
345 }
346 
347 /* Dummy value used in computing PCR value below */
348 #define PCR_ARCH_31    (PCR_ARCH_300 << 1)
349 
350 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
351 {
352 	unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
353 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
354 
355 	/* We can (emulate) our own architecture version and anything older */
356 	if (cpu_has_feature(CPU_FTR_ARCH_31))
357 		host_pcr_bit = PCR_ARCH_31;
358 	else if (cpu_has_feature(CPU_FTR_ARCH_300))
359 		host_pcr_bit = PCR_ARCH_300;
360 	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
361 		host_pcr_bit = PCR_ARCH_207;
362 	else if (cpu_has_feature(CPU_FTR_ARCH_206))
363 		host_pcr_bit = PCR_ARCH_206;
364 	else
365 		host_pcr_bit = PCR_ARCH_205;
366 
367 	/* Determine lowest PCR bit needed to run guest in given PVR level */
368 	guest_pcr_bit = host_pcr_bit;
369 	if (arch_compat) {
370 		switch (arch_compat) {
371 		case PVR_ARCH_205:
372 			guest_pcr_bit = PCR_ARCH_205;
373 			break;
374 		case PVR_ARCH_206:
375 		case PVR_ARCH_206p:
376 			guest_pcr_bit = PCR_ARCH_206;
377 			break;
378 		case PVR_ARCH_207:
379 			guest_pcr_bit = PCR_ARCH_207;
380 			break;
381 		case PVR_ARCH_300:
382 			guest_pcr_bit = PCR_ARCH_300;
383 			break;
384 		case PVR_ARCH_31:
385 			guest_pcr_bit = PCR_ARCH_31;
386 			break;
387 		default:
388 			return -EINVAL;
389 		}
390 	}
391 
392 	/* Check requested PCR bits don't exceed our capabilities */
393 	if (guest_pcr_bit > host_pcr_bit)
394 		return -EINVAL;
395 
396 	spin_lock(&vc->lock);
397 	vc->arch_compat = arch_compat;
398 	/*
399 	 * Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit
400 	 * Also set all reserved PCR bits
401 	 */
402 	vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK;
403 	spin_unlock(&vc->lock);
404 
405 	return 0;
406 }
407 
408 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
409 {
410 	int r;
411 
412 	pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
413 	pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
414 	       vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
415 	for (r = 0; r < 16; ++r)
416 		pr_err("r%2d = %.16lx  r%d = %.16lx\n",
417 		       r, kvmppc_get_gpr(vcpu, r),
418 		       r+16, kvmppc_get_gpr(vcpu, r+16));
419 	pr_err("ctr = %.16lx  lr  = %.16lx\n",
420 	       vcpu->arch.regs.ctr, vcpu->arch.regs.link);
421 	pr_err("srr0 = %.16llx srr1 = %.16llx\n",
422 	       vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
423 	pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
424 	       vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
425 	pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
426 	       vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
427 	pr_err("cr = %.8lx  xer = %.16lx  dsisr = %.8x\n",
428 	       vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
429 	pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
430 	pr_err("fault dar = %.16lx dsisr = %.8x\n",
431 	       vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
432 	pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
433 	for (r = 0; r < vcpu->arch.slb_max; ++r)
434 		pr_err("  ESID = %.16llx VSID = %.16llx\n",
435 		       vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
436 	pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
437 	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
438 	       vcpu->arch.last_inst);
439 }
440 
441 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
442 {
443 	return kvm_get_vcpu_by_id(kvm, id);
444 }
445 
446 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
447 {
448 	vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
449 	vpa->yield_count = cpu_to_be32(1);
450 }
451 
452 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
453 		   unsigned long addr, unsigned long len)
454 {
455 	/* check address is cacheline aligned */
456 	if (addr & (L1_CACHE_BYTES - 1))
457 		return -EINVAL;
458 	spin_lock(&vcpu->arch.vpa_update_lock);
459 	if (v->next_gpa != addr || v->len != len) {
460 		v->next_gpa = addr;
461 		v->len = addr ? len : 0;
462 		v->update_pending = 1;
463 	}
464 	spin_unlock(&vcpu->arch.vpa_update_lock);
465 	return 0;
466 }
467 
468 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
469 struct reg_vpa {
470 	u32 dummy;
471 	union {
472 		__be16 hword;
473 		__be32 word;
474 	} length;
475 };
476 
477 static int vpa_is_registered(struct kvmppc_vpa *vpap)
478 {
479 	if (vpap->update_pending)
480 		return vpap->next_gpa != 0;
481 	return vpap->pinned_addr != NULL;
482 }
483 
484 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
485 				       unsigned long flags,
486 				       unsigned long vcpuid, unsigned long vpa)
487 {
488 	struct kvm *kvm = vcpu->kvm;
489 	unsigned long len, nb;
490 	void *va;
491 	struct kvm_vcpu *tvcpu;
492 	int err;
493 	int subfunc;
494 	struct kvmppc_vpa *vpap;
495 
496 	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
497 	if (!tvcpu)
498 		return H_PARAMETER;
499 
500 	subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
501 	if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
502 	    subfunc == H_VPA_REG_SLB) {
503 		/* Registering new area - address must be cache-line aligned */
504 		if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
505 			return H_PARAMETER;
506 
507 		/* convert logical addr to kernel addr and read length */
508 		va = kvmppc_pin_guest_page(kvm, vpa, &nb);
509 		if (va == NULL)
510 			return H_PARAMETER;
511 		if (subfunc == H_VPA_REG_VPA)
512 			len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
513 		else
514 			len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
515 		kvmppc_unpin_guest_page(kvm, va, vpa, false);
516 
517 		/* Check length */
518 		if (len > nb || len < sizeof(struct reg_vpa))
519 			return H_PARAMETER;
520 	} else {
521 		vpa = 0;
522 		len = 0;
523 	}
524 
525 	err = H_PARAMETER;
526 	vpap = NULL;
527 	spin_lock(&tvcpu->arch.vpa_update_lock);
528 
529 	switch (subfunc) {
530 	case H_VPA_REG_VPA:		/* register VPA */
531 		/*
532 		 * The size of our lppaca is 1kB because of the way we align
533 		 * it for the guest to avoid crossing a 4kB boundary. We only
534 		 * use 640 bytes of the structure though, so we should accept
535 		 * clients that set a size of 640.
536 		 */
537 		BUILD_BUG_ON(sizeof(struct lppaca) != 640);
538 		if (len < sizeof(struct lppaca))
539 			break;
540 		vpap = &tvcpu->arch.vpa;
541 		err = 0;
542 		break;
543 
544 	case H_VPA_REG_DTL:		/* register DTL */
545 		if (len < sizeof(struct dtl_entry))
546 			break;
547 		len -= len % sizeof(struct dtl_entry);
548 
549 		/* Check that they have previously registered a VPA */
550 		err = H_RESOURCE;
551 		if (!vpa_is_registered(&tvcpu->arch.vpa))
552 			break;
553 
554 		vpap = &tvcpu->arch.dtl;
555 		err = 0;
556 		break;
557 
558 	case H_VPA_REG_SLB:		/* register SLB shadow buffer */
559 		/* Check that they have previously registered a VPA */
560 		err = H_RESOURCE;
561 		if (!vpa_is_registered(&tvcpu->arch.vpa))
562 			break;
563 
564 		vpap = &tvcpu->arch.slb_shadow;
565 		err = 0;
566 		break;
567 
568 	case H_VPA_DEREG_VPA:		/* deregister VPA */
569 		/* Check they don't still have a DTL or SLB buf registered */
570 		err = H_RESOURCE;
571 		if (vpa_is_registered(&tvcpu->arch.dtl) ||
572 		    vpa_is_registered(&tvcpu->arch.slb_shadow))
573 			break;
574 
575 		vpap = &tvcpu->arch.vpa;
576 		err = 0;
577 		break;
578 
579 	case H_VPA_DEREG_DTL:		/* deregister DTL */
580 		vpap = &tvcpu->arch.dtl;
581 		err = 0;
582 		break;
583 
584 	case H_VPA_DEREG_SLB:		/* deregister SLB shadow buffer */
585 		vpap = &tvcpu->arch.slb_shadow;
586 		err = 0;
587 		break;
588 	}
589 
590 	if (vpap) {
591 		vpap->next_gpa = vpa;
592 		vpap->len = len;
593 		vpap->update_pending = 1;
594 	}
595 
596 	spin_unlock(&tvcpu->arch.vpa_update_lock);
597 
598 	return err;
599 }
600 
601 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
602 {
603 	struct kvm *kvm = vcpu->kvm;
604 	void *va;
605 	unsigned long nb;
606 	unsigned long gpa;
607 
608 	/*
609 	 * We need to pin the page pointed to by vpap->next_gpa,
610 	 * but we can't call kvmppc_pin_guest_page under the lock
611 	 * as it does get_user_pages() and down_read().  So we
612 	 * have to drop the lock, pin the page, then get the lock
613 	 * again and check that a new area didn't get registered
614 	 * in the meantime.
615 	 */
616 	for (;;) {
617 		gpa = vpap->next_gpa;
618 		spin_unlock(&vcpu->arch.vpa_update_lock);
619 		va = NULL;
620 		nb = 0;
621 		if (gpa)
622 			va = kvmppc_pin_guest_page(kvm, gpa, &nb);
623 		spin_lock(&vcpu->arch.vpa_update_lock);
624 		if (gpa == vpap->next_gpa)
625 			break;
626 		/* sigh... unpin that one and try again */
627 		if (va)
628 			kvmppc_unpin_guest_page(kvm, va, gpa, false);
629 	}
630 
631 	vpap->update_pending = 0;
632 	if (va && nb < vpap->len) {
633 		/*
634 		 * If it's now too short, it must be that userspace
635 		 * has changed the mappings underlying guest memory,
636 		 * so unregister the region.
637 		 */
638 		kvmppc_unpin_guest_page(kvm, va, gpa, false);
639 		va = NULL;
640 	}
641 	if (vpap->pinned_addr)
642 		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
643 					vpap->dirty);
644 	vpap->gpa = gpa;
645 	vpap->pinned_addr = va;
646 	vpap->dirty = false;
647 	if (va)
648 		vpap->pinned_end = va + vpap->len;
649 }
650 
651 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
652 {
653 	if (!(vcpu->arch.vpa.update_pending ||
654 	      vcpu->arch.slb_shadow.update_pending ||
655 	      vcpu->arch.dtl.update_pending))
656 		return;
657 
658 	spin_lock(&vcpu->arch.vpa_update_lock);
659 	if (vcpu->arch.vpa.update_pending) {
660 		kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
661 		if (vcpu->arch.vpa.pinned_addr)
662 			init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
663 	}
664 	if (vcpu->arch.dtl.update_pending) {
665 		kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
666 		vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
667 		vcpu->arch.dtl_index = 0;
668 	}
669 	if (vcpu->arch.slb_shadow.update_pending)
670 		kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
671 	spin_unlock(&vcpu->arch.vpa_update_lock);
672 }
673 
674 /*
675  * Return the accumulated stolen time for the vcore up until `now'.
676  * The caller should hold the vcore lock.
677  */
678 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
679 {
680 	u64 p;
681 	unsigned long flags;
682 
683 	spin_lock_irqsave(&vc->stoltb_lock, flags);
684 	p = vc->stolen_tb;
685 	if (vc->vcore_state != VCORE_INACTIVE &&
686 	    vc->preempt_tb != TB_NIL)
687 		p += now - vc->preempt_tb;
688 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
689 	return p;
690 }
691 
692 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
693 				    struct kvmppc_vcore *vc)
694 {
695 	struct dtl_entry *dt;
696 	struct lppaca *vpa;
697 	unsigned long stolen;
698 	unsigned long core_stolen;
699 	u64 now;
700 	unsigned long flags;
701 
702 	dt = vcpu->arch.dtl_ptr;
703 	vpa = vcpu->arch.vpa.pinned_addr;
704 	now = mftb();
705 	core_stolen = vcore_stolen_time(vc, now);
706 	stolen = core_stolen - vcpu->arch.stolen_logged;
707 	vcpu->arch.stolen_logged = core_stolen;
708 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
709 	stolen += vcpu->arch.busy_stolen;
710 	vcpu->arch.busy_stolen = 0;
711 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
712 	if (!dt || !vpa)
713 		return;
714 	memset(dt, 0, sizeof(struct dtl_entry));
715 	dt->dispatch_reason = 7;
716 	dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
717 	dt->timebase = cpu_to_be64(now + vc->tb_offset);
718 	dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
719 	dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
720 	dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
721 	++dt;
722 	if (dt == vcpu->arch.dtl.pinned_end)
723 		dt = vcpu->arch.dtl.pinned_addr;
724 	vcpu->arch.dtl_ptr = dt;
725 	/* order writing *dt vs. writing vpa->dtl_idx */
726 	smp_wmb();
727 	vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
728 	vcpu->arch.dtl.dirty = true;
729 }
730 
731 /* See if there is a doorbell interrupt pending for a vcpu */
732 static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
733 {
734 	int thr;
735 	struct kvmppc_vcore *vc;
736 
737 	if (vcpu->arch.doorbell_request)
738 		return true;
739 	/*
740 	 * Ensure that the read of vcore->dpdes comes after the read
741 	 * of vcpu->doorbell_request.  This barrier matches the
742 	 * smp_wmb() in kvmppc_guest_entry_inject().
743 	 */
744 	smp_rmb();
745 	vc = vcpu->arch.vcore;
746 	thr = vcpu->vcpu_id - vc->first_vcpuid;
747 	return !!(vc->dpdes & (1 << thr));
748 }
749 
750 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
751 {
752 	if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
753 		return true;
754 	if ((!vcpu->arch.vcore->arch_compat) &&
755 	    cpu_has_feature(CPU_FTR_ARCH_207S))
756 		return true;
757 	return false;
758 }
759 
760 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
761 			     unsigned long resource, unsigned long value1,
762 			     unsigned long value2)
763 {
764 	switch (resource) {
765 	case H_SET_MODE_RESOURCE_SET_CIABR:
766 		if (!kvmppc_power8_compatible(vcpu))
767 			return H_P2;
768 		if (value2)
769 			return H_P4;
770 		if (mflags)
771 			return H_UNSUPPORTED_FLAG_START;
772 		/* Guests can't breakpoint the hypervisor */
773 		if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
774 			return H_P3;
775 		vcpu->arch.ciabr  = value1;
776 		return H_SUCCESS;
777 	case H_SET_MODE_RESOURCE_SET_DAWR0:
778 		if (!kvmppc_power8_compatible(vcpu))
779 			return H_P2;
780 		if (!ppc_breakpoint_available())
781 			return H_P2;
782 		if (mflags)
783 			return H_UNSUPPORTED_FLAG_START;
784 		if (value2 & DABRX_HYP)
785 			return H_P4;
786 		vcpu->arch.dawr0  = value1;
787 		vcpu->arch.dawrx0 = value2;
788 		return H_SUCCESS;
789 	case H_SET_MODE_RESOURCE_SET_DAWR1:
790 		if (!kvmppc_power8_compatible(vcpu))
791 			return H_P2;
792 		if (!ppc_breakpoint_available())
793 			return H_P2;
794 		if (!cpu_has_feature(CPU_FTR_DAWR1))
795 			return H_P2;
796 		if (!vcpu->kvm->arch.dawr1_enabled)
797 			return H_FUNCTION;
798 		if (mflags)
799 			return H_UNSUPPORTED_FLAG_START;
800 		if (value2 & DABRX_HYP)
801 			return H_P4;
802 		vcpu->arch.dawr1  = value1;
803 		vcpu->arch.dawrx1 = value2;
804 		return H_SUCCESS;
805 	case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
806 		/*
807 		 * KVM does not support mflags=2 (AIL=2) and AIL=1 is reserved.
808 		 * Keep this in synch with kvmppc_filter_guest_lpcr_hv.
809 		 */
810 		if (mflags != 0 && mflags != 3)
811 			return H_UNSUPPORTED_FLAG_START;
812 		return H_TOO_HARD;
813 	default:
814 		return H_TOO_HARD;
815 	}
816 }
817 
818 /* Copy guest memory in place - must reside within a single memslot */
819 static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
820 				  unsigned long len)
821 {
822 	struct kvm_memory_slot *to_memslot = NULL;
823 	struct kvm_memory_slot *from_memslot = NULL;
824 	unsigned long to_addr, from_addr;
825 	int r;
826 
827 	/* Get HPA for from address */
828 	from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
829 	if (!from_memslot)
830 		return -EFAULT;
831 	if ((from + len) >= ((from_memslot->base_gfn + from_memslot->npages)
832 			     << PAGE_SHIFT))
833 		return -EINVAL;
834 	from_addr = gfn_to_hva_memslot(from_memslot, from >> PAGE_SHIFT);
835 	if (kvm_is_error_hva(from_addr))
836 		return -EFAULT;
837 	from_addr |= (from & (PAGE_SIZE - 1));
838 
839 	/* Get HPA for to address */
840 	to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
841 	if (!to_memslot)
842 		return -EFAULT;
843 	if ((to + len) >= ((to_memslot->base_gfn + to_memslot->npages)
844 			   << PAGE_SHIFT))
845 		return -EINVAL;
846 	to_addr = gfn_to_hva_memslot(to_memslot, to >> PAGE_SHIFT);
847 	if (kvm_is_error_hva(to_addr))
848 		return -EFAULT;
849 	to_addr |= (to & (PAGE_SIZE - 1));
850 
851 	/* Perform copy */
852 	r = raw_copy_in_user((void __user *)to_addr, (void __user *)from_addr,
853 			     len);
854 	if (r)
855 		return -EFAULT;
856 	mark_page_dirty(kvm, to >> PAGE_SHIFT);
857 	return 0;
858 }
859 
860 static long kvmppc_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags,
861 			       unsigned long dest, unsigned long src)
862 {
863 	u64 pg_sz = SZ_4K;		/* 4K page size */
864 	u64 pg_mask = SZ_4K - 1;
865 	int ret;
866 
867 	/* Check for invalid flags (H_PAGE_SET_LOANED covers all CMO flags) */
868 	if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE |
869 		      H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED))
870 		return H_PARAMETER;
871 
872 	/* dest (and src if copy_page flag set) must be page aligned */
873 	if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask)))
874 		return H_PARAMETER;
875 
876 	/* zero and/or copy the page as determined by the flags */
877 	if (flags & H_COPY_PAGE) {
878 		ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
879 		if (ret < 0)
880 			return H_PARAMETER;
881 	} else if (flags & H_ZERO_PAGE) {
882 		ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
883 		if (ret < 0)
884 			return H_PARAMETER;
885 	}
886 
887 	/* We can ignore the remaining flags */
888 
889 	return H_SUCCESS;
890 }
891 
892 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
893 {
894 	struct kvmppc_vcore *vcore = target->arch.vcore;
895 
896 	/*
897 	 * We expect to have been called by the real mode handler
898 	 * (kvmppc_rm_h_confer()) which would have directly returned
899 	 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may
900 	 * have useful work to do and should not confer) so we don't
901 	 * recheck that here.
902 	 */
903 
904 	spin_lock(&vcore->lock);
905 	if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
906 	    vcore->vcore_state != VCORE_INACTIVE &&
907 	    vcore->runner)
908 		target = vcore->runner;
909 	spin_unlock(&vcore->lock);
910 
911 	return kvm_vcpu_yield_to(target);
912 }
913 
914 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
915 {
916 	int yield_count = 0;
917 	struct lppaca *lppaca;
918 
919 	spin_lock(&vcpu->arch.vpa_update_lock);
920 	lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
921 	if (lppaca)
922 		yield_count = be32_to_cpu(lppaca->yield_count);
923 	spin_unlock(&vcpu->arch.vpa_update_lock);
924 	return yield_count;
925 }
926 
927 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
928 {
929 	unsigned long req = kvmppc_get_gpr(vcpu, 3);
930 	unsigned long target, ret = H_SUCCESS;
931 	int yield_count;
932 	struct kvm_vcpu *tvcpu;
933 	int idx, rc;
934 
935 	if (req <= MAX_HCALL_OPCODE &&
936 	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
937 		return RESUME_HOST;
938 
939 	switch (req) {
940 	case H_CEDE:
941 		break;
942 	case H_PROD:
943 		target = kvmppc_get_gpr(vcpu, 4);
944 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
945 		if (!tvcpu) {
946 			ret = H_PARAMETER;
947 			break;
948 		}
949 		tvcpu->arch.prodded = 1;
950 		smp_mb();
951 		if (tvcpu->arch.ceded)
952 			kvmppc_fast_vcpu_kick_hv(tvcpu);
953 		break;
954 	case H_CONFER:
955 		target = kvmppc_get_gpr(vcpu, 4);
956 		if (target == -1)
957 			break;
958 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
959 		if (!tvcpu) {
960 			ret = H_PARAMETER;
961 			break;
962 		}
963 		yield_count = kvmppc_get_gpr(vcpu, 5);
964 		if (kvmppc_get_yield_count(tvcpu) != yield_count)
965 			break;
966 		kvm_arch_vcpu_yield_to(tvcpu);
967 		break;
968 	case H_REGISTER_VPA:
969 		ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
970 					kvmppc_get_gpr(vcpu, 5),
971 					kvmppc_get_gpr(vcpu, 6));
972 		break;
973 	case H_RTAS:
974 		if (list_empty(&vcpu->kvm->arch.rtas_tokens))
975 			return RESUME_HOST;
976 
977 		idx = srcu_read_lock(&vcpu->kvm->srcu);
978 		rc = kvmppc_rtas_hcall(vcpu);
979 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
980 
981 		if (rc == -ENOENT)
982 			return RESUME_HOST;
983 		else if (rc == 0)
984 			break;
985 
986 		/* Send the error out to userspace via KVM_RUN */
987 		return rc;
988 	case H_LOGICAL_CI_LOAD:
989 		ret = kvmppc_h_logical_ci_load(vcpu);
990 		if (ret == H_TOO_HARD)
991 			return RESUME_HOST;
992 		break;
993 	case H_LOGICAL_CI_STORE:
994 		ret = kvmppc_h_logical_ci_store(vcpu);
995 		if (ret == H_TOO_HARD)
996 			return RESUME_HOST;
997 		break;
998 	case H_SET_MODE:
999 		ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
1000 					kvmppc_get_gpr(vcpu, 5),
1001 					kvmppc_get_gpr(vcpu, 6),
1002 					kvmppc_get_gpr(vcpu, 7));
1003 		if (ret == H_TOO_HARD)
1004 			return RESUME_HOST;
1005 		break;
1006 	case H_XIRR:
1007 	case H_CPPR:
1008 	case H_EOI:
1009 	case H_IPI:
1010 	case H_IPOLL:
1011 	case H_XIRR_X:
1012 		if (kvmppc_xics_enabled(vcpu)) {
1013 			if (xics_on_xive()) {
1014 				ret = H_NOT_AVAILABLE;
1015 				return RESUME_GUEST;
1016 			}
1017 			ret = kvmppc_xics_hcall(vcpu, req);
1018 			break;
1019 		}
1020 		return RESUME_HOST;
1021 	case H_SET_DABR:
1022 		ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
1023 		break;
1024 	case H_SET_XDABR:
1025 		ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
1026 						kvmppc_get_gpr(vcpu, 5));
1027 		break;
1028 #ifdef CONFIG_SPAPR_TCE_IOMMU
1029 	case H_GET_TCE:
1030 		ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1031 						kvmppc_get_gpr(vcpu, 5));
1032 		if (ret == H_TOO_HARD)
1033 			return RESUME_HOST;
1034 		break;
1035 	case H_PUT_TCE:
1036 		ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1037 						kvmppc_get_gpr(vcpu, 5),
1038 						kvmppc_get_gpr(vcpu, 6));
1039 		if (ret == H_TOO_HARD)
1040 			return RESUME_HOST;
1041 		break;
1042 	case H_PUT_TCE_INDIRECT:
1043 		ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
1044 						kvmppc_get_gpr(vcpu, 5),
1045 						kvmppc_get_gpr(vcpu, 6),
1046 						kvmppc_get_gpr(vcpu, 7));
1047 		if (ret == H_TOO_HARD)
1048 			return RESUME_HOST;
1049 		break;
1050 	case H_STUFF_TCE:
1051 		ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1052 						kvmppc_get_gpr(vcpu, 5),
1053 						kvmppc_get_gpr(vcpu, 6),
1054 						kvmppc_get_gpr(vcpu, 7));
1055 		if (ret == H_TOO_HARD)
1056 			return RESUME_HOST;
1057 		break;
1058 #endif
1059 	case H_RANDOM:
1060 		if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4]))
1061 			ret = H_HARDWARE;
1062 		break;
1063 
1064 	case H_SET_PARTITION_TABLE:
1065 		ret = H_FUNCTION;
1066 		if (nesting_enabled(vcpu->kvm))
1067 			ret = kvmhv_set_partition_table(vcpu);
1068 		break;
1069 	case H_ENTER_NESTED:
1070 		ret = H_FUNCTION;
1071 		if (!nesting_enabled(vcpu->kvm))
1072 			break;
1073 		ret = kvmhv_enter_nested_guest(vcpu);
1074 		if (ret == H_INTERRUPT) {
1075 			kvmppc_set_gpr(vcpu, 3, 0);
1076 			vcpu->arch.hcall_needed = 0;
1077 			return -EINTR;
1078 		} else if (ret == H_TOO_HARD) {
1079 			kvmppc_set_gpr(vcpu, 3, 0);
1080 			vcpu->arch.hcall_needed = 0;
1081 			return RESUME_HOST;
1082 		}
1083 		break;
1084 	case H_TLB_INVALIDATE:
1085 		ret = H_FUNCTION;
1086 		if (nesting_enabled(vcpu->kvm))
1087 			ret = kvmhv_do_nested_tlbie(vcpu);
1088 		break;
1089 	case H_COPY_TOFROM_GUEST:
1090 		ret = H_FUNCTION;
1091 		if (nesting_enabled(vcpu->kvm))
1092 			ret = kvmhv_copy_tofrom_guest_nested(vcpu);
1093 		break;
1094 	case H_PAGE_INIT:
1095 		ret = kvmppc_h_page_init(vcpu, kvmppc_get_gpr(vcpu, 4),
1096 					 kvmppc_get_gpr(vcpu, 5),
1097 					 kvmppc_get_gpr(vcpu, 6));
1098 		break;
1099 	case H_SVM_PAGE_IN:
1100 		ret = H_UNSUPPORTED;
1101 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1102 			ret = kvmppc_h_svm_page_in(vcpu->kvm,
1103 						   kvmppc_get_gpr(vcpu, 4),
1104 						   kvmppc_get_gpr(vcpu, 5),
1105 						   kvmppc_get_gpr(vcpu, 6));
1106 		break;
1107 	case H_SVM_PAGE_OUT:
1108 		ret = H_UNSUPPORTED;
1109 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1110 			ret = kvmppc_h_svm_page_out(vcpu->kvm,
1111 						    kvmppc_get_gpr(vcpu, 4),
1112 						    kvmppc_get_gpr(vcpu, 5),
1113 						    kvmppc_get_gpr(vcpu, 6));
1114 		break;
1115 	case H_SVM_INIT_START:
1116 		ret = H_UNSUPPORTED;
1117 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1118 			ret = kvmppc_h_svm_init_start(vcpu->kvm);
1119 		break;
1120 	case H_SVM_INIT_DONE:
1121 		ret = H_UNSUPPORTED;
1122 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1123 			ret = kvmppc_h_svm_init_done(vcpu->kvm);
1124 		break;
1125 	case H_SVM_INIT_ABORT:
1126 		/*
1127 		 * Even if that call is made by the Ultravisor, the SSR1 value
1128 		 * is the guest context one, with the secure bit clear as it has
1129 		 * not yet been secured. So we can't check it here.
1130 		 * Instead the kvm->arch.secure_guest flag is checked inside
1131 		 * kvmppc_h_svm_init_abort().
1132 		 */
1133 		ret = kvmppc_h_svm_init_abort(vcpu->kvm);
1134 		break;
1135 
1136 	default:
1137 		return RESUME_HOST;
1138 	}
1139 	kvmppc_set_gpr(vcpu, 3, ret);
1140 	vcpu->arch.hcall_needed = 0;
1141 	return RESUME_GUEST;
1142 }
1143 
1144 /*
1145  * Handle H_CEDE in the nested virtualization case where we haven't
1146  * called the real-mode hcall handlers in book3s_hv_rmhandlers.S.
1147  * This has to be done early, not in kvmppc_pseries_do_hcall(), so
1148  * that the cede logic in kvmppc_run_single_vcpu() works properly.
1149  */
1150 static void kvmppc_nested_cede(struct kvm_vcpu *vcpu)
1151 {
1152 	vcpu->arch.shregs.msr |= MSR_EE;
1153 	vcpu->arch.ceded = 1;
1154 	smp_mb();
1155 	if (vcpu->arch.prodded) {
1156 		vcpu->arch.prodded = 0;
1157 		smp_mb();
1158 		vcpu->arch.ceded = 0;
1159 	}
1160 }
1161 
1162 static int kvmppc_hcall_impl_hv(unsigned long cmd)
1163 {
1164 	switch (cmd) {
1165 	case H_CEDE:
1166 	case H_PROD:
1167 	case H_CONFER:
1168 	case H_REGISTER_VPA:
1169 	case H_SET_MODE:
1170 	case H_LOGICAL_CI_LOAD:
1171 	case H_LOGICAL_CI_STORE:
1172 #ifdef CONFIG_KVM_XICS
1173 	case H_XIRR:
1174 	case H_CPPR:
1175 	case H_EOI:
1176 	case H_IPI:
1177 	case H_IPOLL:
1178 	case H_XIRR_X:
1179 #endif
1180 	case H_PAGE_INIT:
1181 		return 1;
1182 	}
1183 
1184 	/* See if it's in the real-mode table */
1185 	return kvmppc_hcall_impl_hv_realmode(cmd);
1186 }
1187 
1188 static int kvmppc_emulate_debug_inst(struct kvm_vcpu *vcpu)
1189 {
1190 	u32 last_inst;
1191 
1192 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
1193 					EMULATE_DONE) {
1194 		/*
1195 		 * Fetch failed, so return to guest and
1196 		 * try executing it again.
1197 		 */
1198 		return RESUME_GUEST;
1199 	}
1200 
1201 	if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
1202 		vcpu->run->exit_reason = KVM_EXIT_DEBUG;
1203 		vcpu->run->debug.arch.address = kvmppc_get_pc(vcpu);
1204 		return RESUME_HOST;
1205 	} else {
1206 		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1207 		return RESUME_GUEST;
1208 	}
1209 }
1210 
1211 static void do_nothing(void *x)
1212 {
1213 }
1214 
1215 static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1216 {
1217 	int thr, cpu, pcpu, nthreads;
1218 	struct kvm_vcpu *v;
1219 	unsigned long dpdes;
1220 
1221 	nthreads = vcpu->kvm->arch.emul_smt_mode;
1222 	dpdes = 0;
1223 	cpu = vcpu->vcpu_id & ~(nthreads - 1);
1224 	for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1225 		v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1226 		if (!v)
1227 			continue;
1228 		/*
1229 		 * If the vcpu is currently running on a physical cpu thread,
1230 		 * interrupt it in order to pull it out of the guest briefly,
1231 		 * which will update its vcore->dpdes value.
1232 		 */
1233 		pcpu = READ_ONCE(v->cpu);
1234 		if (pcpu >= 0)
1235 			smp_call_function_single(pcpu, do_nothing, NULL, 1);
1236 		if (kvmppc_doorbell_pending(v))
1237 			dpdes |= 1 << thr;
1238 	}
1239 	return dpdes;
1240 }
1241 
1242 /*
1243  * On POWER9, emulate doorbell-related instructions in order to
1244  * give the guest the illusion of running on a multi-threaded core.
1245  * The instructions emulated are msgsndp, msgclrp, mfspr TIR,
1246  * and mfspr DPDES.
1247  */
1248 static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1249 {
1250 	u32 inst, rb, thr;
1251 	unsigned long arg;
1252 	struct kvm *kvm = vcpu->kvm;
1253 	struct kvm_vcpu *tvcpu;
1254 
1255 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1256 		return RESUME_GUEST;
1257 	if (get_op(inst) != 31)
1258 		return EMULATE_FAIL;
1259 	rb = get_rb(inst);
1260 	thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1261 	switch (get_xop(inst)) {
1262 	case OP_31_XOP_MSGSNDP:
1263 		arg = kvmppc_get_gpr(vcpu, rb);
1264 		if (((arg >> 27) & 0x1f) != PPC_DBELL_SERVER)
1265 			break;
1266 		arg &= 0x7f;
1267 		if (arg >= kvm->arch.emul_smt_mode)
1268 			break;
1269 		tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1270 		if (!tvcpu)
1271 			break;
1272 		if (!tvcpu->arch.doorbell_request) {
1273 			tvcpu->arch.doorbell_request = 1;
1274 			kvmppc_fast_vcpu_kick_hv(tvcpu);
1275 		}
1276 		break;
1277 	case OP_31_XOP_MSGCLRP:
1278 		arg = kvmppc_get_gpr(vcpu, rb);
1279 		if (((arg >> 27) & 0x1f) != PPC_DBELL_SERVER)
1280 			break;
1281 		vcpu->arch.vcore->dpdes = 0;
1282 		vcpu->arch.doorbell_request = 0;
1283 		break;
1284 	case OP_31_XOP_MFSPR:
1285 		switch (get_sprn(inst)) {
1286 		case SPRN_TIR:
1287 			arg = thr;
1288 			break;
1289 		case SPRN_DPDES:
1290 			arg = kvmppc_read_dpdes(vcpu);
1291 			break;
1292 		default:
1293 			return EMULATE_FAIL;
1294 		}
1295 		kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1296 		break;
1297 	default:
1298 		return EMULATE_FAIL;
1299 	}
1300 	kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1301 	return RESUME_GUEST;
1302 }
1303 
1304 static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
1305 				 struct task_struct *tsk)
1306 {
1307 	struct kvm_run *run = vcpu->run;
1308 	int r = RESUME_HOST;
1309 
1310 	vcpu->stat.sum_exits++;
1311 
1312 	/*
1313 	 * This can happen if an interrupt occurs in the last stages
1314 	 * of guest entry or the first stages of guest exit (i.e. after
1315 	 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1316 	 * and before setting it to KVM_GUEST_MODE_HOST_HV).
1317 	 * That can happen due to a bug, or due to a machine check
1318 	 * occurring at just the wrong time.
1319 	 */
1320 	if (vcpu->arch.shregs.msr & MSR_HV) {
1321 		printk(KERN_EMERG "KVM trap in HV mode!\n");
1322 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1323 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
1324 			vcpu->arch.shregs.msr);
1325 		kvmppc_dump_regs(vcpu);
1326 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1327 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1328 		return RESUME_HOST;
1329 	}
1330 	run->exit_reason = KVM_EXIT_UNKNOWN;
1331 	run->ready_for_interrupt_injection = 1;
1332 	switch (vcpu->arch.trap) {
1333 	/* We're good on these - the host merely wanted to get our attention */
1334 	case BOOK3S_INTERRUPT_HV_DECREMENTER:
1335 		vcpu->stat.dec_exits++;
1336 		r = RESUME_GUEST;
1337 		break;
1338 	case BOOK3S_INTERRUPT_EXTERNAL:
1339 	case BOOK3S_INTERRUPT_H_DOORBELL:
1340 	case BOOK3S_INTERRUPT_H_VIRT:
1341 		vcpu->stat.ext_intr_exits++;
1342 		r = RESUME_GUEST;
1343 		break;
1344 	/* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
1345 	case BOOK3S_INTERRUPT_HMI:
1346 	case BOOK3S_INTERRUPT_PERFMON:
1347 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
1348 		r = RESUME_GUEST;
1349 		break;
1350 	case BOOK3S_INTERRUPT_MACHINE_CHECK: {
1351 		static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1352 					      DEFAULT_RATELIMIT_BURST);
1353 		/*
1354 		 * Print the MCE event to host console. Ratelimit so the guest
1355 		 * can't flood the host log.
1356 		 */
1357 		if (__ratelimit(&rs))
1358 			machine_check_print_event_info(&vcpu->arch.mce_evt,false, true);
1359 
1360 		/*
1361 		 * If the guest can do FWNMI, exit to userspace so it can
1362 		 * deliver a FWNMI to the guest.
1363 		 * Otherwise we synthesize a machine check for the guest
1364 		 * so that it knows that the machine check occurred.
1365 		 */
1366 		if (!vcpu->kvm->arch.fwnmi_enabled) {
1367 			ulong flags = vcpu->arch.shregs.msr & 0x083c0000;
1368 			kvmppc_core_queue_machine_check(vcpu, flags);
1369 			r = RESUME_GUEST;
1370 			break;
1371 		}
1372 
1373 		/* Exit to guest with KVM_EXIT_NMI as exit reason */
1374 		run->exit_reason = KVM_EXIT_NMI;
1375 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1376 		/* Clear out the old NMI status from run->flags */
1377 		run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1378 		/* Now set the NMI status */
1379 		if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1380 			run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1381 		else
1382 			run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1383 
1384 		r = RESUME_HOST;
1385 		break;
1386 	}
1387 	case BOOK3S_INTERRUPT_PROGRAM:
1388 	{
1389 		ulong flags;
1390 		/*
1391 		 * Normally program interrupts are delivered directly
1392 		 * to the guest by the hardware, but we can get here
1393 		 * as a result of a hypervisor emulation interrupt
1394 		 * (e40) getting turned into a 700 by BML RTAS.
1395 		 */
1396 		flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1397 		kvmppc_core_queue_program(vcpu, flags);
1398 		r = RESUME_GUEST;
1399 		break;
1400 	}
1401 	case BOOK3S_INTERRUPT_SYSCALL:
1402 	{
1403 		/* hcall - punt to userspace */
1404 		int i;
1405 
1406 		/* hypercall with MSR_PR has already been handled in rmode,
1407 		 * and never reaches here.
1408 		 */
1409 
1410 		run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1411 		for (i = 0; i < 9; ++i)
1412 			run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1413 		run->exit_reason = KVM_EXIT_PAPR_HCALL;
1414 		vcpu->arch.hcall_needed = 1;
1415 		r = RESUME_HOST;
1416 		break;
1417 	}
1418 	/*
1419 	 * We get these next two if the guest accesses a page which it thinks
1420 	 * it has mapped but which is not actually present, either because
1421 	 * it is for an emulated I/O device or because the corresonding
1422 	 * host page has been paged out.  Any other HDSI/HISI interrupts
1423 	 * have been handled already.
1424 	 */
1425 	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1426 		r = RESUME_PAGE_FAULT;
1427 		break;
1428 	case BOOK3S_INTERRUPT_H_INST_STORAGE:
1429 		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1430 		vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
1431 			DSISR_SRR1_MATCH_64S;
1432 		if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1433 			vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1434 		r = RESUME_PAGE_FAULT;
1435 		break;
1436 	/*
1437 	 * This occurs if the guest executes an illegal instruction.
1438 	 * If the guest debug is disabled, generate a program interrupt
1439 	 * to the guest. If guest debug is enabled, we need to check
1440 	 * whether the instruction is a software breakpoint instruction.
1441 	 * Accordingly return to Guest or Host.
1442 	 */
1443 	case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1444 		if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1445 			vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1446 				swab32(vcpu->arch.emul_inst) :
1447 				vcpu->arch.emul_inst;
1448 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1449 			r = kvmppc_emulate_debug_inst(vcpu);
1450 		} else {
1451 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1452 			r = RESUME_GUEST;
1453 		}
1454 		break;
1455 	/*
1456 	 * This occurs if the guest (kernel or userspace), does something that
1457 	 * is prohibited by HFSCR.
1458 	 * On POWER9, this could be a doorbell instruction that we need
1459 	 * to emulate.
1460 	 * Otherwise, we just generate a program interrupt to the guest.
1461 	 */
1462 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1463 		r = EMULATE_FAIL;
1464 		if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
1465 		    cpu_has_feature(CPU_FTR_ARCH_300))
1466 			r = kvmppc_emulate_doorbell_instr(vcpu);
1467 		if (r == EMULATE_FAIL) {
1468 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1469 			r = RESUME_GUEST;
1470 		}
1471 		break;
1472 
1473 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1474 	case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1475 		/*
1476 		 * This occurs for various TM-related instructions that
1477 		 * we need to emulate on POWER9 DD2.2.  We have already
1478 		 * handled the cases where the guest was in real-suspend
1479 		 * mode and was transitioning to transactional state.
1480 		 */
1481 		r = kvmhv_p9_tm_emulation(vcpu);
1482 		break;
1483 #endif
1484 
1485 	case BOOK3S_INTERRUPT_HV_RM_HARD:
1486 		r = RESUME_PASSTHROUGH;
1487 		break;
1488 	default:
1489 		kvmppc_dump_regs(vcpu);
1490 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1491 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
1492 			vcpu->arch.shregs.msr);
1493 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1494 		r = RESUME_HOST;
1495 		break;
1496 	}
1497 
1498 	return r;
1499 }
1500 
1501 static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
1502 {
1503 	int r;
1504 	int srcu_idx;
1505 
1506 	vcpu->stat.sum_exits++;
1507 
1508 	/*
1509 	 * This can happen if an interrupt occurs in the last stages
1510 	 * of guest entry or the first stages of guest exit (i.e. after
1511 	 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1512 	 * and before setting it to KVM_GUEST_MODE_HOST_HV).
1513 	 * That can happen due to a bug, or due to a machine check
1514 	 * occurring at just the wrong time.
1515 	 */
1516 	if (vcpu->arch.shregs.msr & MSR_HV) {
1517 		pr_emerg("KVM trap in HV mode while nested!\n");
1518 		pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1519 			 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1520 			 vcpu->arch.shregs.msr);
1521 		kvmppc_dump_regs(vcpu);
1522 		return RESUME_HOST;
1523 	}
1524 	switch (vcpu->arch.trap) {
1525 	/* We're good on these - the host merely wanted to get our attention */
1526 	case BOOK3S_INTERRUPT_HV_DECREMENTER:
1527 		vcpu->stat.dec_exits++;
1528 		r = RESUME_GUEST;
1529 		break;
1530 	case BOOK3S_INTERRUPT_EXTERNAL:
1531 		vcpu->stat.ext_intr_exits++;
1532 		r = RESUME_HOST;
1533 		break;
1534 	case BOOK3S_INTERRUPT_H_DOORBELL:
1535 	case BOOK3S_INTERRUPT_H_VIRT:
1536 		vcpu->stat.ext_intr_exits++;
1537 		r = RESUME_GUEST;
1538 		break;
1539 	/* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
1540 	case BOOK3S_INTERRUPT_HMI:
1541 	case BOOK3S_INTERRUPT_PERFMON:
1542 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
1543 		r = RESUME_GUEST;
1544 		break;
1545 	case BOOK3S_INTERRUPT_MACHINE_CHECK:
1546 	{
1547 		static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1548 					      DEFAULT_RATELIMIT_BURST);
1549 		/* Pass the machine check to the L1 guest */
1550 		r = RESUME_HOST;
1551 		/* Print the MCE event to host console. */
1552 		if (__ratelimit(&rs))
1553 			machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1554 		break;
1555 	}
1556 	/*
1557 	 * We get these next two if the guest accesses a page which it thinks
1558 	 * it has mapped but which is not actually present, either because
1559 	 * it is for an emulated I/O device or because the corresonding
1560 	 * host page has been paged out.
1561 	 */
1562 	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1563 		srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1564 		r = kvmhv_nested_page_fault(vcpu);
1565 		srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1566 		break;
1567 	case BOOK3S_INTERRUPT_H_INST_STORAGE:
1568 		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1569 		vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
1570 					 DSISR_SRR1_MATCH_64S;
1571 		if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1572 			vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1573 		srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1574 		r = kvmhv_nested_page_fault(vcpu);
1575 		srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1576 		break;
1577 
1578 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1579 	case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1580 		/*
1581 		 * This occurs for various TM-related instructions that
1582 		 * we need to emulate on POWER9 DD2.2.  We have already
1583 		 * handled the cases where the guest was in real-suspend
1584 		 * mode and was transitioning to transactional state.
1585 		 */
1586 		r = kvmhv_p9_tm_emulation(vcpu);
1587 		break;
1588 #endif
1589 
1590 	case BOOK3S_INTERRUPT_HV_RM_HARD:
1591 		vcpu->arch.trap = 0;
1592 		r = RESUME_GUEST;
1593 		if (!xics_on_xive())
1594 			kvmppc_xics_rm_complete(vcpu, 0);
1595 		break;
1596 	default:
1597 		r = RESUME_HOST;
1598 		break;
1599 	}
1600 
1601 	return r;
1602 }
1603 
1604 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1605 					    struct kvm_sregs *sregs)
1606 {
1607 	int i;
1608 
1609 	memset(sregs, 0, sizeof(struct kvm_sregs));
1610 	sregs->pvr = vcpu->arch.pvr;
1611 	for (i = 0; i < vcpu->arch.slb_max; i++) {
1612 		sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1613 		sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1614 	}
1615 
1616 	return 0;
1617 }
1618 
1619 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1620 					    struct kvm_sregs *sregs)
1621 {
1622 	int i, j;
1623 
1624 	/* Only accept the same PVR as the host's, since we can't spoof it */
1625 	if (sregs->pvr != vcpu->arch.pvr)
1626 		return -EINVAL;
1627 
1628 	j = 0;
1629 	for (i = 0; i < vcpu->arch.slb_nr; i++) {
1630 		if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1631 			vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1632 			vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1633 			++j;
1634 		}
1635 	}
1636 	vcpu->arch.slb_max = j;
1637 
1638 	return 0;
1639 }
1640 
1641 /*
1642  * Enforce limits on guest LPCR values based on hardware availability,
1643  * guest configuration, and possibly hypervisor support and security
1644  * concerns.
1645  */
1646 unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr)
1647 {
1648 	/* LPCR_TC only applies to HPT guests */
1649 	if (kvm_is_radix(kvm))
1650 		lpcr &= ~LPCR_TC;
1651 
1652 	/* On POWER8 and above, userspace can modify AIL */
1653 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
1654 		lpcr &= ~LPCR_AIL;
1655 	if ((lpcr & LPCR_AIL) != LPCR_AIL_3)
1656 		lpcr &= ~LPCR_AIL; /* LPCR[AIL]=1/2 is disallowed */
1657 
1658 	/*
1659 	 * On POWER9, allow userspace to enable large decrementer for the
1660 	 * guest, whether or not the host has it enabled.
1661 	 */
1662 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
1663 		lpcr &= ~LPCR_LD;
1664 
1665 	return lpcr;
1666 }
1667 
1668 static void verify_lpcr(struct kvm *kvm, unsigned long lpcr)
1669 {
1670 	if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) {
1671 		WARN_ONCE(1, "lpcr 0x%lx differs from filtered 0x%lx\n",
1672 			  lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr));
1673 	}
1674 }
1675 
1676 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1677 		bool preserve_top32)
1678 {
1679 	struct kvm *kvm = vcpu->kvm;
1680 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
1681 	u64 mask;
1682 
1683 	spin_lock(&vc->lock);
1684 
1685 	/*
1686 	 * Userspace can only modify
1687 	 * DPFD (default prefetch depth), ILE (interrupt little-endian),
1688 	 * TC (translation control), AIL (alternate interrupt location),
1689 	 * LD (large decrementer).
1690 	 * These are subject to restrictions from kvmppc_filter_lcpr_hv().
1691 	 */
1692 	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD;
1693 
1694 	/* Broken 32-bit version of LPCR must not clear top bits */
1695 	if (preserve_top32)
1696 		mask &= 0xFFFFFFFF;
1697 
1698 	new_lpcr = kvmppc_filter_lpcr_hv(kvm,
1699 			(vc->lpcr & ~mask) | (new_lpcr & mask));
1700 
1701 	/*
1702 	 * If ILE (interrupt little-endian) has changed, update the
1703 	 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
1704 	 */
1705 	if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1706 		struct kvm_vcpu *vcpu;
1707 		int i;
1708 
1709 		kvm_for_each_vcpu(i, vcpu, kvm) {
1710 			if (vcpu->arch.vcore != vc)
1711 				continue;
1712 			if (new_lpcr & LPCR_ILE)
1713 				vcpu->arch.intr_msr |= MSR_LE;
1714 			else
1715 				vcpu->arch.intr_msr &= ~MSR_LE;
1716 		}
1717 	}
1718 
1719 	vc->lpcr = new_lpcr;
1720 
1721 	spin_unlock(&vc->lock);
1722 }
1723 
1724 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1725 				 union kvmppc_one_reg *val)
1726 {
1727 	int r = 0;
1728 	long int i;
1729 
1730 	switch (id) {
1731 	case KVM_REG_PPC_DEBUG_INST:
1732 		*val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1733 		break;
1734 	case KVM_REG_PPC_HIOR:
1735 		*val = get_reg_val(id, 0);
1736 		break;
1737 	case KVM_REG_PPC_DABR:
1738 		*val = get_reg_val(id, vcpu->arch.dabr);
1739 		break;
1740 	case KVM_REG_PPC_DABRX:
1741 		*val = get_reg_val(id, vcpu->arch.dabrx);
1742 		break;
1743 	case KVM_REG_PPC_DSCR:
1744 		*val = get_reg_val(id, vcpu->arch.dscr);
1745 		break;
1746 	case KVM_REG_PPC_PURR:
1747 		*val = get_reg_val(id, vcpu->arch.purr);
1748 		break;
1749 	case KVM_REG_PPC_SPURR:
1750 		*val = get_reg_val(id, vcpu->arch.spurr);
1751 		break;
1752 	case KVM_REG_PPC_AMR:
1753 		*val = get_reg_val(id, vcpu->arch.amr);
1754 		break;
1755 	case KVM_REG_PPC_UAMOR:
1756 		*val = get_reg_val(id, vcpu->arch.uamor);
1757 		break;
1758 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
1759 		i = id - KVM_REG_PPC_MMCR0;
1760 		*val = get_reg_val(id, vcpu->arch.mmcr[i]);
1761 		break;
1762 	case KVM_REG_PPC_MMCR2:
1763 		*val = get_reg_val(id, vcpu->arch.mmcr[2]);
1764 		break;
1765 	case KVM_REG_PPC_MMCRA:
1766 		*val = get_reg_val(id, vcpu->arch.mmcra);
1767 		break;
1768 	case KVM_REG_PPC_MMCRS:
1769 		*val = get_reg_val(id, vcpu->arch.mmcrs);
1770 		break;
1771 	case KVM_REG_PPC_MMCR3:
1772 		*val = get_reg_val(id, vcpu->arch.mmcr[3]);
1773 		break;
1774 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1775 		i = id - KVM_REG_PPC_PMC1;
1776 		*val = get_reg_val(id, vcpu->arch.pmc[i]);
1777 		break;
1778 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1779 		i = id - KVM_REG_PPC_SPMC1;
1780 		*val = get_reg_val(id, vcpu->arch.spmc[i]);
1781 		break;
1782 	case KVM_REG_PPC_SIAR:
1783 		*val = get_reg_val(id, vcpu->arch.siar);
1784 		break;
1785 	case KVM_REG_PPC_SDAR:
1786 		*val = get_reg_val(id, vcpu->arch.sdar);
1787 		break;
1788 	case KVM_REG_PPC_SIER:
1789 		*val = get_reg_val(id, vcpu->arch.sier[0]);
1790 		break;
1791 	case KVM_REG_PPC_SIER2:
1792 		*val = get_reg_val(id, vcpu->arch.sier[1]);
1793 		break;
1794 	case KVM_REG_PPC_SIER3:
1795 		*val = get_reg_val(id, vcpu->arch.sier[2]);
1796 		break;
1797 	case KVM_REG_PPC_IAMR:
1798 		*val = get_reg_val(id, vcpu->arch.iamr);
1799 		break;
1800 	case KVM_REG_PPC_PSPB:
1801 		*val = get_reg_val(id, vcpu->arch.pspb);
1802 		break;
1803 	case KVM_REG_PPC_DPDES:
1804 		/*
1805 		 * On POWER9, where we are emulating msgsndp etc.,
1806 		 * we return 1 bit for each vcpu, which can come from
1807 		 * either vcore->dpdes or doorbell_request.
1808 		 * On POWER8, doorbell_request is 0.
1809 		 */
1810 		*val = get_reg_val(id, vcpu->arch.vcore->dpdes |
1811 				   vcpu->arch.doorbell_request);
1812 		break;
1813 	case KVM_REG_PPC_VTB:
1814 		*val = get_reg_val(id, vcpu->arch.vcore->vtb);
1815 		break;
1816 	case KVM_REG_PPC_DAWR:
1817 		*val = get_reg_val(id, vcpu->arch.dawr0);
1818 		break;
1819 	case KVM_REG_PPC_DAWRX:
1820 		*val = get_reg_val(id, vcpu->arch.dawrx0);
1821 		break;
1822 	case KVM_REG_PPC_DAWR1:
1823 		*val = get_reg_val(id, vcpu->arch.dawr1);
1824 		break;
1825 	case KVM_REG_PPC_DAWRX1:
1826 		*val = get_reg_val(id, vcpu->arch.dawrx1);
1827 		break;
1828 	case KVM_REG_PPC_CIABR:
1829 		*val = get_reg_val(id, vcpu->arch.ciabr);
1830 		break;
1831 	case KVM_REG_PPC_CSIGR:
1832 		*val = get_reg_val(id, vcpu->arch.csigr);
1833 		break;
1834 	case KVM_REG_PPC_TACR:
1835 		*val = get_reg_val(id, vcpu->arch.tacr);
1836 		break;
1837 	case KVM_REG_PPC_TCSCR:
1838 		*val = get_reg_val(id, vcpu->arch.tcscr);
1839 		break;
1840 	case KVM_REG_PPC_PID:
1841 		*val = get_reg_val(id, vcpu->arch.pid);
1842 		break;
1843 	case KVM_REG_PPC_ACOP:
1844 		*val = get_reg_val(id, vcpu->arch.acop);
1845 		break;
1846 	case KVM_REG_PPC_WORT:
1847 		*val = get_reg_val(id, vcpu->arch.wort);
1848 		break;
1849 	case KVM_REG_PPC_TIDR:
1850 		*val = get_reg_val(id, vcpu->arch.tid);
1851 		break;
1852 	case KVM_REG_PPC_PSSCR:
1853 		*val = get_reg_val(id, vcpu->arch.psscr);
1854 		break;
1855 	case KVM_REG_PPC_VPA_ADDR:
1856 		spin_lock(&vcpu->arch.vpa_update_lock);
1857 		*val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1858 		spin_unlock(&vcpu->arch.vpa_update_lock);
1859 		break;
1860 	case KVM_REG_PPC_VPA_SLB:
1861 		spin_lock(&vcpu->arch.vpa_update_lock);
1862 		val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1863 		val->vpaval.length = vcpu->arch.slb_shadow.len;
1864 		spin_unlock(&vcpu->arch.vpa_update_lock);
1865 		break;
1866 	case KVM_REG_PPC_VPA_DTL:
1867 		spin_lock(&vcpu->arch.vpa_update_lock);
1868 		val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1869 		val->vpaval.length = vcpu->arch.dtl.len;
1870 		spin_unlock(&vcpu->arch.vpa_update_lock);
1871 		break;
1872 	case KVM_REG_PPC_TB_OFFSET:
1873 		*val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1874 		break;
1875 	case KVM_REG_PPC_LPCR:
1876 	case KVM_REG_PPC_LPCR_64:
1877 		*val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1878 		break;
1879 	case KVM_REG_PPC_PPR:
1880 		*val = get_reg_val(id, vcpu->arch.ppr);
1881 		break;
1882 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1883 	case KVM_REG_PPC_TFHAR:
1884 		*val = get_reg_val(id, vcpu->arch.tfhar);
1885 		break;
1886 	case KVM_REG_PPC_TFIAR:
1887 		*val = get_reg_val(id, vcpu->arch.tfiar);
1888 		break;
1889 	case KVM_REG_PPC_TEXASR:
1890 		*val = get_reg_val(id, vcpu->arch.texasr);
1891 		break;
1892 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1893 		i = id - KVM_REG_PPC_TM_GPR0;
1894 		*val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1895 		break;
1896 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1897 	{
1898 		int j;
1899 		i = id - KVM_REG_PPC_TM_VSR0;
1900 		if (i < 32)
1901 			for (j = 0; j < TS_FPRWIDTH; j++)
1902 				val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1903 		else {
1904 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
1905 				val->vval = vcpu->arch.vr_tm.vr[i-32];
1906 			else
1907 				r = -ENXIO;
1908 		}
1909 		break;
1910 	}
1911 	case KVM_REG_PPC_TM_CR:
1912 		*val = get_reg_val(id, vcpu->arch.cr_tm);
1913 		break;
1914 	case KVM_REG_PPC_TM_XER:
1915 		*val = get_reg_val(id, vcpu->arch.xer_tm);
1916 		break;
1917 	case KVM_REG_PPC_TM_LR:
1918 		*val = get_reg_val(id, vcpu->arch.lr_tm);
1919 		break;
1920 	case KVM_REG_PPC_TM_CTR:
1921 		*val = get_reg_val(id, vcpu->arch.ctr_tm);
1922 		break;
1923 	case KVM_REG_PPC_TM_FPSCR:
1924 		*val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1925 		break;
1926 	case KVM_REG_PPC_TM_AMR:
1927 		*val = get_reg_val(id, vcpu->arch.amr_tm);
1928 		break;
1929 	case KVM_REG_PPC_TM_PPR:
1930 		*val = get_reg_val(id, vcpu->arch.ppr_tm);
1931 		break;
1932 	case KVM_REG_PPC_TM_VRSAVE:
1933 		*val = get_reg_val(id, vcpu->arch.vrsave_tm);
1934 		break;
1935 	case KVM_REG_PPC_TM_VSCR:
1936 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
1937 			*val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1938 		else
1939 			r = -ENXIO;
1940 		break;
1941 	case KVM_REG_PPC_TM_DSCR:
1942 		*val = get_reg_val(id, vcpu->arch.dscr_tm);
1943 		break;
1944 	case KVM_REG_PPC_TM_TAR:
1945 		*val = get_reg_val(id, vcpu->arch.tar_tm);
1946 		break;
1947 #endif
1948 	case KVM_REG_PPC_ARCH_COMPAT:
1949 		*val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1950 		break;
1951 	case KVM_REG_PPC_DEC_EXPIRY:
1952 		*val = get_reg_val(id, vcpu->arch.dec_expires +
1953 				   vcpu->arch.vcore->tb_offset);
1954 		break;
1955 	case KVM_REG_PPC_ONLINE:
1956 		*val = get_reg_val(id, vcpu->arch.online);
1957 		break;
1958 	case KVM_REG_PPC_PTCR:
1959 		*val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
1960 		break;
1961 	default:
1962 		r = -EINVAL;
1963 		break;
1964 	}
1965 
1966 	return r;
1967 }
1968 
1969 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1970 				 union kvmppc_one_reg *val)
1971 {
1972 	int r = 0;
1973 	long int i;
1974 	unsigned long addr, len;
1975 
1976 	switch (id) {
1977 	case KVM_REG_PPC_HIOR:
1978 		/* Only allow this to be set to zero */
1979 		if (set_reg_val(id, *val))
1980 			r = -EINVAL;
1981 		break;
1982 	case KVM_REG_PPC_DABR:
1983 		vcpu->arch.dabr = set_reg_val(id, *val);
1984 		break;
1985 	case KVM_REG_PPC_DABRX:
1986 		vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1987 		break;
1988 	case KVM_REG_PPC_DSCR:
1989 		vcpu->arch.dscr = set_reg_val(id, *val);
1990 		break;
1991 	case KVM_REG_PPC_PURR:
1992 		vcpu->arch.purr = set_reg_val(id, *val);
1993 		break;
1994 	case KVM_REG_PPC_SPURR:
1995 		vcpu->arch.spurr = set_reg_val(id, *val);
1996 		break;
1997 	case KVM_REG_PPC_AMR:
1998 		vcpu->arch.amr = set_reg_val(id, *val);
1999 		break;
2000 	case KVM_REG_PPC_UAMOR:
2001 		vcpu->arch.uamor = set_reg_val(id, *val);
2002 		break;
2003 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
2004 		i = id - KVM_REG_PPC_MMCR0;
2005 		vcpu->arch.mmcr[i] = set_reg_val(id, *val);
2006 		break;
2007 	case KVM_REG_PPC_MMCR2:
2008 		vcpu->arch.mmcr[2] = set_reg_val(id, *val);
2009 		break;
2010 	case KVM_REG_PPC_MMCRA:
2011 		vcpu->arch.mmcra = set_reg_val(id, *val);
2012 		break;
2013 	case KVM_REG_PPC_MMCRS:
2014 		vcpu->arch.mmcrs = set_reg_val(id, *val);
2015 		break;
2016 	case KVM_REG_PPC_MMCR3:
2017 		*val = get_reg_val(id, vcpu->arch.mmcr[3]);
2018 		break;
2019 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
2020 		i = id - KVM_REG_PPC_PMC1;
2021 		vcpu->arch.pmc[i] = set_reg_val(id, *val);
2022 		break;
2023 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
2024 		i = id - KVM_REG_PPC_SPMC1;
2025 		vcpu->arch.spmc[i] = set_reg_val(id, *val);
2026 		break;
2027 	case KVM_REG_PPC_SIAR:
2028 		vcpu->arch.siar = set_reg_val(id, *val);
2029 		break;
2030 	case KVM_REG_PPC_SDAR:
2031 		vcpu->arch.sdar = set_reg_val(id, *val);
2032 		break;
2033 	case KVM_REG_PPC_SIER:
2034 		vcpu->arch.sier[0] = set_reg_val(id, *val);
2035 		break;
2036 	case KVM_REG_PPC_SIER2:
2037 		vcpu->arch.sier[1] = set_reg_val(id, *val);
2038 		break;
2039 	case KVM_REG_PPC_SIER3:
2040 		vcpu->arch.sier[2] = set_reg_val(id, *val);
2041 		break;
2042 	case KVM_REG_PPC_IAMR:
2043 		vcpu->arch.iamr = set_reg_val(id, *val);
2044 		break;
2045 	case KVM_REG_PPC_PSPB:
2046 		vcpu->arch.pspb = set_reg_val(id, *val);
2047 		break;
2048 	case KVM_REG_PPC_DPDES:
2049 		vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
2050 		break;
2051 	case KVM_REG_PPC_VTB:
2052 		vcpu->arch.vcore->vtb = set_reg_val(id, *val);
2053 		break;
2054 	case KVM_REG_PPC_DAWR:
2055 		vcpu->arch.dawr0 = set_reg_val(id, *val);
2056 		break;
2057 	case KVM_REG_PPC_DAWRX:
2058 		vcpu->arch.dawrx0 = set_reg_val(id, *val) & ~DAWRX_HYP;
2059 		break;
2060 	case KVM_REG_PPC_DAWR1:
2061 		vcpu->arch.dawr1 = set_reg_val(id, *val);
2062 		break;
2063 	case KVM_REG_PPC_DAWRX1:
2064 		vcpu->arch.dawrx1 = set_reg_val(id, *val) & ~DAWRX_HYP;
2065 		break;
2066 	case KVM_REG_PPC_CIABR:
2067 		vcpu->arch.ciabr = set_reg_val(id, *val);
2068 		/* Don't allow setting breakpoints in hypervisor code */
2069 		if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
2070 			vcpu->arch.ciabr &= ~CIABR_PRIV;	/* disable */
2071 		break;
2072 	case KVM_REG_PPC_CSIGR:
2073 		vcpu->arch.csigr = set_reg_val(id, *val);
2074 		break;
2075 	case KVM_REG_PPC_TACR:
2076 		vcpu->arch.tacr = set_reg_val(id, *val);
2077 		break;
2078 	case KVM_REG_PPC_TCSCR:
2079 		vcpu->arch.tcscr = set_reg_val(id, *val);
2080 		break;
2081 	case KVM_REG_PPC_PID:
2082 		vcpu->arch.pid = set_reg_val(id, *val);
2083 		break;
2084 	case KVM_REG_PPC_ACOP:
2085 		vcpu->arch.acop = set_reg_val(id, *val);
2086 		break;
2087 	case KVM_REG_PPC_WORT:
2088 		vcpu->arch.wort = set_reg_val(id, *val);
2089 		break;
2090 	case KVM_REG_PPC_TIDR:
2091 		vcpu->arch.tid = set_reg_val(id, *val);
2092 		break;
2093 	case KVM_REG_PPC_PSSCR:
2094 		vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
2095 		break;
2096 	case KVM_REG_PPC_VPA_ADDR:
2097 		addr = set_reg_val(id, *val);
2098 		r = -EINVAL;
2099 		if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
2100 			      vcpu->arch.dtl.next_gpa))
2101 			break;
2102 		r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
2103 		break;
2104 	case KVM_REG_PPC_VPA_SLB:
2105 		addr = val->vpaval.addr;
2106 		len = val->vpaval.length;
2107 		r = -EINVAL;
2108 		if (addr && !vcpu->arch.vpa.next_gpa)
2109 			break;
2110 		r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
2111 		break;
2112 	case KVM_REG_PPC_VPA_DTL:
2113 		addr = val->vpaval.addr;
2114 		len = val->vpaval.length;
2115 		r = -EINVAL;
2116 		if (addr && (len < sizeof(struct dtl_entry) ||
2117 			     !vcpu->arch.vpa.next_gpa))
2118 			break;
2119 		len -= len % sizeof(struct dtl_entry);
2120 		r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
2121 		break;
2122 	case KVM_REG_PPC_TB_OFFSET:
2123 		/* round up to multiple of 2^24 */
2124 		vcpu->arch.vcore->tb_offset =
2125 			ALIGN(set_reg_val(id, *val), 1UL << 24);
2126 		break;
2127 	case KVM_REG_PPC_LPCR:
2128 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
2129 		break;
2130 	case KVM_REG_PPC_LPCR_64:
2131 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
2132 		break;
2133 	case KVM_REG_PPC_PPR:
2134 		vcpu->arch.ppr = set_reg_val(id, *val);
2135 		break;
2136 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2137 	case KVM_REG_PPC_TFHAR:
2138 		vcpu->arch.tfhar = set_reg_val(id, *val);
2139 		break;
2140 	case KVM_REG_PPC_TFIAR:
2141 		vcpu->arch.tfiar = set_reg_val(id, *val);
2142 		break;
2143 	case KVM_REG_PPC_TEXASR:
2144 		vcpu->arch.texasr = set_reg_val(id, *val);
2145 		break;
2146 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
2147 		i = id - KVM_REG_PPC_TM_GPR0;
2148 		vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
2149 		break;
2150 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
2151 	{
2152 		int j;
2153 		i = id - KVM_REG_PPC_TM_VSR0;
2154 		if (i < 32)
2155 			for (j = 0; j < TS_FPRWIDTH; j++)
2156 				vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
2157 		else
2158 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
2159 				vcpu->arch.vr_tm.vr[i-32] = val->vval;
2160 			else
2161 				r = -ENXIO;
2162 		break;
2163 	}
2164 	case KVM_REG_PPC_TM_CR:
2165 		vcpu->arch.cr_tm = set_reg_val(id, *val);
2166 		break;
2167 	case KVM_REG_PPC_TM_XER:
2168 		vcpu->arch.xer_tm = set_reg_val(id, *val);
2169 		break;
2170 	case KVM_REG_PPC_TM_LR:
2171 		vcpu->arch.lr_tm = set_reg_val(id, *val);
2172 		break;
2173 	case KVM_REG_PPC_TM_CTR:
2174 		vcpu->arch.ctr_tm = set_reg_val(id, *val);
2175 		break;
2176 	case KVM_REG_PPC_TM_FPSCR:
2177 		vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
2178 		break;
2179 	case KVM_REG_PPC_TM_AMR:
2180 		vcpu->arch.amr_tm = set_reg_val(id, *val);
2181 		break;
2182 	case KVM_REG_PPC_TM_PPR:
2183 		vcpu->arch.ppr_tm = set_reg_val(id, *val);
2184 		break;
2185 	case KVM_REG_PPC_TM_VRSAVE:
2186 		vcpu->arch.vrsave_tm = set_reg_val(id, *val);
2187 		break;
2188 	case KVM_REG_PPC_TM_VSCR:
2189 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
2190 			vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
2191 		else
2192 			r = - ENXIO;
2193 		break;
2194 	case KVM_REG_PPC_TM_DSCR:
2195 		vcpu->arch.dscr_tm = set_reg_val(id, *val);
2196 		break;
2197 	case KVM_REG_PPC_TM_TAR:
2198 		vcpu->arch.tar_tm = set_reg_val(id, *val);
2199 		break;
2200 #endif
2201 	case KVM_REG_PPC_ARCH_COMPAT:
2202 		r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
2203 		break;
2204 	case KVM_REG_PPC_DEC_EXPIRY:
2205 		vcpu->arch.dec_expires = set_reg_val(id, *val) -
2206 			vcpu->arch.vcore->tb_offset;
2207 		break;
2208 	case KVM_REG_PPC_ONLINE:
2209 		i = set_reg_val(id, *val);
2210 		if (i && !vcpu->arch.online)
2211 			atomic_inc(&vcpu->arch.vcore->online_count);
2212 		else if (!i && vcpu->arch.online)
2213 			atomic_dec(&vcpu->arch.vcore->online_count);
2214 		vcpu->arch.online = i;
2215 		break;
2216 	case KVM_REG_PPC_PTCR:
2217 		vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
2218 		break;
2219 	default:
2220 		r = -EINVAL;
2221 		break;
2222 	}
2223 
2224 	return r;
2225 }
2226 
2227 /*
2228  * On POWER9, threads are independent and can be in different partitions.
2229  * Therefore we consider each thread to be a subcore.
2230  * There is a restriction that all threads have to be in the same
2231  * MMU mode (radix or HPT), unfortunately, but since we only support
2232  * HPT guests on a HPT host so far, that isn't an impediment yet.
2233  */
2234 static int threads_per_vcore(struct kvm *kvm)
2235 {
2236 	if (kvm->arch.threads_indep)
2237 		return 1;
2238 	return threads_per_subcore;
2239 }
2240 
2241 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
2242 {
2243 	struct kvmppc_vcore *vcore;
2244 
2245 	vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
2246 
2247 	if (vcore == NULL)
2248 		return NULL;
2249 
2250 	spin_lock_init(&vcore->lock);
2251 	spin_lock_init(&vcore->stoltb_lock);
2252 	rcuwait_init(&vcore->wait);
2253 	vcore->preempt_tb = TB_NIL;
2254 	vcore->lpcr = kvm->arch.lpcr;
2255 	vcore->first_vcpuid = id;
2256 	vcore->kvm = kvm;
2257 	INIT_LIST_HEAD(&vcore->preempt_list);
2258 
2259 	return vcore;
2260 }
2261 
2262 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2263 static struct debugfs_timings_element {
2264 	const char *name;
2265 	size_t offset;
2266 } timings[] = {
2267 	{"rm_entry",	offsetof(struct kvm_vcpu, arch.rm_entry)},
2268 	{"rm_intr",	offsetof(struct kvm_vcpu, arch.rm_intr)},
2269 	{"rm_exit",	offsetof(struct kvm_vcpu, arch.rm_exit)},
2270 	{"guest",	offsetof(struct kvm_vcpu, arch.guest_time)},
2271 	{"cede",	offsetof(struct kvm_vcpu, arch.cede_time)},
2272 };
2273 
2274 #define N_TIMINGS	(ARRAY_SIZE(timings))
2275 
2276 struct debugfs_timings_state {
2277 	struct kvm_vcpu	*vcpu;
2278 	unsigned int	buflen;
2279 	char		buf[N_TIMINGS * 100];
2280 };
2281 
2282 static int debugfs_timings_open(struct inode *inode, struct file *file)
2283 {
2284 	struct kvm_vcpu *vcpu = inode->i_private;
2285 	struct debugfs_timings_state *p;
2286 
2287 	p = kzalloc(sizeof(*p), GFP_KERNEL);
2288 	if (!p)
2289 		return -ENOMEM;
2290 
2291 	kvm_get_kvm(vcpu->kvm);
2292 	p->vcpu = vcpu;
2293 	file->private_data = p;
2294 
2295 	return nonseekable_open(inode, file);
2296 }
2297 
2298 static int debugfs_timings_release(struct inode *inode, struct file *file)
2299 {
2300 	struct debugfs_timings_state *p = file->private_data;
2301 
2302 	kvm_put_kvm(p->vcpu->kvm);
2303 	kfree(p);
2304 	return 0;
2305 }
2306 
2307 static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
2308 				    size_t len, loff_t *ppos)
2309 {
2310 	struct debugfs_timings_state *p = file->private_data;
2311 	struct kvm_vcpu *vcpu = p->vcpu;
2312 	char *s, *buf_end;
2313 	struct kvmhv_tb_accumulator tb;
2314 	u64 count;
2315 	loff_t pos;
2316 	ssize_t n;
2317 	int i, loops;
2318 	bool ok;
2319 
2320 	if (!p->buflen) {
2321 		s = p->buf;
2322 		buf_end = s + sizeof(p->buf);
2323 		for (i = 0; i < N_TIMINGS; ++i) {
2324 			struct kvmhv_tb_accumulator *acc;
2325 
2326 			acc = (struct kvmhv_tb_accumulator *)
2327 				((unsigned long)vcpu + timings[i].offset);
2328 			ok = false;
2329 			for (loops = 0; loops < 1000; ++loops) {
2330 				count = acc->seqcount;
2331 				if (!(count & 1)) {
2332 					smp_rmb();
2333 					tb = *acc;
2334 					smp_rmb();
2335 					if (count == acc->seqcount) {
2336 						ok = true;
2337 						break;
2338 					}
2339 				}
2340 				udelay(1);
2341 			}
2342 			if (!ok)
2343 				snprintf(s, buf_end - s, "%s: stuck\n",
2344 					timings[i].name);
2345 			else
2346 				snprintf(s, buf_end - s,
2347 					"%s: %llu %llu %llu %llu\n",
2348 					timings[i].name, count / 2,
2349 					tb_to_ns(tb.tb_total),
2350 					tb_to_ns(tb.tb_min),
2351 					tb_to_ns(tb.tb_max));
2352 			s += strlen(s);
2353 		}
2354 		p->buflen = s - p->buf;
2355 	}
2356 
2357 	pos = *ppos;
2358 	if (pos >= p->buflen)
2359 		return 0;
2360 	if (len > p->buflen - pos)
2361 		len = p->buflen - pos;
2362 	n = copy_to_user(buf, p->buf + pos, len);
2363 	if (n) {
2364 		if (n == len)
2365 			return -EFAULT;
2366 		len -= n;
2367 	}
2368 	*ppos = pos + len;
2369 	return len;
2370 }
2371 
2372 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
2373 				     size_t len, loff_t *ppos)
2374 {
2375 	return -EACCES;
2376 }
2377 
2378 static const struct file_operations debugfs_timings_ops = {
2379 	.owner	 = THIS_MODULE,
2380 	.open	 = debugfs_timings_open,
2381 	.release = debugfs_timings_release,
2382 	.read	 = debugfs_timings_read,
2383 	.write	 = debugfs_timings_write,
2384 	.llseek	 = generic_file_llseek,
2385 };
2386 
2387 /* Create a debugfs directory for the vcpu */
2388 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2389 {
2390 	char buf[16];
2391 	struct kvm *kvm = vcpu->kvm;
2392 
2393 	snprintf(buf, sizeof(buf), "vcpu%u", id);
2394 	vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
2395 	debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, vcpu,
2396 			    &debugfs_timings_ops);
2397 }
2398 
2399 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
2400 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2401 {
2402 }
2403 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
2404 
2405 static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu)
2406 {
2407 	int err;
2408 	int core;
2409 	struct kvmppc_vcore *vcore;
2410 	struct kvm *kvm;
2411 	unsigned int id;
2412 
2413 	kvm = vcpu->kvm;
2414 	id = vcpu->vcpu_id;
2415 
2416 	vcpu->arch.shared = &vcpu->arch.shregs;
2417 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2418 	/*
2419 	 * The shared struct is never shared on HV,
2420 	 * so we can always use host endianness
2421 	 */
2422 #ifdef __BIG_ENDIAN__
2423 	vcpu->arch.shared_big_endian = true;
2424 #else
2425 	vcpu->arch.shared_big_endian = false;
2426 #endif
2427 #endif
2428 	vcpu->arch.mmcr[0] = MMCR0_FC;
2429 	vcpu->arch.ctrl = CTRL_RUNLATCH;
2430 	/* default to host PVR, since we can't spoof it */
2431 	kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2432 	spin_lock_init(&vcpu->arch.vpa_update_lock);
2433 	spin_lock_init(&vcpu->arch.tbacct_lock);
2434 	vcpu->arch.busy_preempt = TB_NIL;
2435 	vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2436 
2437 	/*
2438 	 * Set the default HFSCR for the guest from the host value.
2439 	 * This value is only used on POWER9.
2440 	 * On POWER9, we want to virtualize the doorbell facility, so we
2441 	 * don't set the HFSCR_MSGP bit, and that causes those instructions
2442 	 * to trap and then we emulate them.
2443 	 */
2444 	vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
2445 		HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP | HFSCR_PREFIX;
2446 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
2447 		vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
2448 		if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2449 			vcpu->arch.hfscr |= HFSCR_TM;
2450 	}
2451 	if (cpu_has_feature(CPU_FTR_TM_COMP))
2452 		vcpu->arch.hfscr |= HFSCR_TM;
2453 
2454 	kvmppc_mmu_book3s_hv_init(vcpu);
2455 
2456 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2457 
2458 	init_waitqueue_head(&vcpu->arch.cpu_run);
2459 
2460 	mutex_lock(&kvm->lock);
2461 	vcore = NULL;
2462 	err = -EINVAL;
2463 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2464 		if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2465 			pr_devel("KVM: VCPU ID too high\n");
2466 			core = KVM_MAX_VCORES;
2467 		} else {
2468 			BUG_ON(kvm->arch.smt_mode != 1);
2469 			core = kvmppc_pack_vcpu_id(kvm, id);
2470 		}
2471 	} else {
2472 		core = id / kvm->arch.smt_mode;
2473 	}
2474 	if (core < KVM_MAX_VCORES) {
2475 		vcore = kvm->arch.vcores[core];
2476 		if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2477 			pr_devel("KVM: collision on id %u", id);
2478 			vcore = NULL;
2479 		} else if (!vcore) {
2480 			/*
2481 			 * Take mmu_setup_lock for mutual exclusion
2482 			 * with kvmppc_update_lpcr().
2483 			 */
2484 			err = -ENOMEM;
2485 			vcore = kvmppc_vcore_create(kvm,
2486 					id & ~(kvm->arch.smt_mode - 1));
2487 			mutex_lock(&kvm->arch.mmu_setup_lock);
2488 			kvm->arch.vcores[core] = vcore;
2489 			kvm->arch.online_vcores++;
2490 			mutex_unlock(&kvm->arch.mmu_setup_lock);
2491 		}
2492 	}
2493 	mutex_unlock(&kvm->lock);
2494 
2495 	if (!vcore)
2496 		return err;
2497 
2498 	spin_lock(&vcore->lock);
2499 	++vcore->num_threads;
2500 	spin_unlock(&vcore->lock);
2501 	vcpu->arch.vcore = vcore;
2502 	vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2503 	vcpu->arch.thread_cpu = -1;
2504 	vcpu->arch.prev_cpu = -1;
2505 
2506 	vcpu->arch.cpu_type = KVM_CPU_3S_64;
2507 	kvmppc_sanity_check(vcpu);
2508 
2509 	debugfs_vcpu_init(vcpu, id);
2510 
2511 	return 0;
2512 }
2513 
2514 static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2515 			      unsigned long flags)
2516 {
2517 	int err;
2518 	int esmt = 0;
2519 
2520 	if (flags)
2521 		return -EINVAL;
2522 	if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2523 		return -EINVAL;
2524 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2525 		/*
2526 		 * On POWER8 (or POWER7), the threading mode is "strict",
2527 		 * so we pack smt_mode vcpus per vcore.
2528 		 */
2529 		if (smt_mode > threads_per_subcore)
2530 			return -EINVAL;
2531 	} else {
2532 		/*
2533 		 * On POWER9, the threading mode is "loose",
2534 		 * so each vcpu gets its own vcore.
2535 		 */
2536 		esmt = smt_mode;
2537 		smt_mode = 1;
2538 	}
2539 	mutex_lock(&kvm->lock);
2540 	err = -EBUSY;
2541 	if (!kvm->arch.online_vcores) {
2542 		kvm->arch.smt_mode = smt_mode;
2543 		kvm->arch.emul_smt_mode = esmt;
2544 		err = 0;
2545 	}
2546 	mutex_unlock(&kvm->lock);
2547 
2548 	return err;
2549 }
2550 
2551 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2552 {
2553 	if (vpa->pinned_addr)
2554 		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2555 					vpa->dirty);
2556 }
2557 
2558 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2559 {
2560 	spin_lock(&vcpu->arch.vpa_update_lock);
2561 	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2562 	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2563 	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2564 	spin_unlock(&vcpu->arch.vpa_update_lock);
2565 }
2566 
2567 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2568 {
2569 	/* Indicate we want to get back into the guest */
2570 	return 1;
2571 }
2572 
2573 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2574 {
2575 	unsigned long dec_nsec, now;
2576 
2577 	now = get_tb();
2578 	if (now > vcpu->arch.dec_expires) {
2579 		/* decrementer has already gone negative */
2580 		kvmppc_core_queue_dec(vcpu);
2581 		kvmppc_core_prepare_to_enter(vcpu);
2582 		return;
2583 	}
2584 	dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now);
2585 	hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2586 	vcpu->arch.timer_running = 1;
2587 }
2588 
2589 extern int __kvmppc_vcore_entry(void);
2590 
2591 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2592 				   struct kvm_vcpu *vcpu)
2593 {
2594 	u64 now;
2595 
2596 	if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2597 		return;
2598 	spin_lock_irq(&vcpu->arch.tbacct_lock);
2599 	now = mftb();
2600 	vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2601 		vcpu->arch.stolen_logged;
2602 	vcpu->arch.busy_preempt = now;
2603 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2604 	spin_unlock_irq(&vcpu->arch.tbacct_lock);
2605 	--vc->n_runnable;
2606 	WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2607 }
2608 
2609 static int kvmppc_grab_hwthread(int cpu)
2610 {
2611 	struct paca_struct *tpaca;
2612 	long timeout = 10000;
2613 
2614 	tpaca = paca_ptrs[cpu];
2615 
2616 	/* Ensure the thread won't go into the kernel if it wakes */
2617 	tpaca->kvm_hstate.kvm_vcpu = NULL;
2618 	tpaca->kvm_hstate.kvm_vcore = NULL;
2619 	tpaca->kvm_hstate.napping = 0;
2620 	smp_wmb();
2621 	tpaca->kvm_hstate.hwthread_req = 1;
2622 
2623 	/*
2624 	 * If the thread is already executing in the kernel (e.g. handling
2625 	 * a stray interrupt), wait for it to get back to nap mode.
2626 	 * The smp_mb() is to ensure that our setting of hwthread_req
2627 	 * is visible before we look at hwthread_state, so if this
2628 	 * races with the code at system_reset_pSeries and the thread
2629 	 * misses our setting of hwthread_req, we are sure to see its
2630 	 * setting of hwthread_state, and vice versa.
2631 	 */
2632 	smp_mb();
2633 	while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2634 		if (--timeout <= 0) {
2635 			pr_err("KVM: couldn't grab cpu %d\n", cpu);
2636 			return -EBUSY;
2637 		}
2638 		udelay(1);
2639 	}
2640 	return 0;
2641 }
2642 
2643 static void kvmppc_release_hwthread(int cpu)
2644 {
2645 	struct paca_struct *tpaca;
2646 
2647 	tpaca = paca_ptrs[cpu];
2648 	tpaca->kvm_hstate.hwthread_req = 0;
2649 	tpaca->kvm_hstate.kvm_vcpu = NULL;
2650 	tpaca->kvm_hstate.kvm_vcore = NULL;
2651 	tpaca->kvm_hstate.kvm_split_mode = NULL;
2652 }
2653 
2654 static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2655 {
2656 	struct kvm_nested_guest *nested = vcpu->arch.nested;
2657 	cpumask_t *cpu_in_guest;
2658 	int i;
2659 
2660 	cpu = cpu_first_thread_sibling(cpu);
2661 	if (nested) {
2662 		cpumask_set_cpu(cpu, &nested->need_tlb_flush);
2663 		cpu_in_guest = &nested->cpu_in_guest;
2664 	} else {
2665 		cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2666 		cpu_in_guest = &kvm->arch.cpu_in_guest;
2667 	}
2668 	/*
2669 	 * Make sure setting of bit in need_tlb_flush precedes
2670 	 * testing of cpu_in_guest bits.  The matching barrier on
2671 	 * the other side is the first smp_mb() in kvmppc_run_core().
2672 	 */
2673 	smp_mb();
2674 	for (i = 0; i < threads_per_core; ++i)
2675 		if (cpumask_test_cpu(cpu + i, cpu_in_guest))
2676 			smp_call_function_single(cpu + i, do_nothing, NULL, 1);
2677 }
2678 
2679 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2680 {
2681 	struct kvm_nested_guest *nested = vcpu->arch.nested;
2682 	struct kvm *kvm = vcpu->kvm;
2683 	int prev_cpu;
2684 
2685 	if (!cpu_has_feature(CPU_FTR_HVMODE))
2686 		return;
2687 
2688 	if (nested)
2689 		prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
2690 	else
2691 		prev_cpu = vcpu->arch.prev_cpu;
2692 
2693 	/*
2694 	 * With radix, the guest can do TLB invalidations itself,
2695 	 * and it could choose to use the local form (tlbiel) if
2696 	 * it is invalidating a translation that has only ever been
2697 	 * used on one vcpu.  However, that doesn't mean it has
2698 	 * only ever been used on one physical cpu, since vcpus
2699 	 * can move around between pcpus.  To cope with this, when
2700 	 * a vcpu moves from one pcpu to another, we need to tell
2701 	 * any vcpus running on the same core as this vcpu previously
2702 	 * ran to flush the TLB.  The TLB is shared between threads,
2703 	 * so we use a single bit in .need_tlb_flush for all 4 threads.
2704 	 */
2705 	if (prev_cpu != pcpu) {
2706 		if (prev_cpu >= 0 &&
2707 		    cpu_first_thread_sibling(prev_cpu) !=
2708 		    cpu_first_thread_sibling(pcpu))
2709 			radix_flush_cpu(kvm, prev_cpu, vcpu);
2710 		if (nested)
2711 			nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
2712 		else
2713 			vcpu->arch.prev_cpu = pcpu;
2714 	}
2715 }
2716 
2717 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
2718 {
2719 	int cpu;
2720 	struct paca_struct *tpaca;
2721 	struct kvm *kvm = vc->kvm;
2722 
2723 	cpu = vc->pcpu;
2724 	if (vcpu) {
2725 		if (vcpu->arch.timer_running) {
2726 			hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2727 			vcpu->arch.timer_running = 0;
2728 		}
2729 		cpu += vcpu->arch.ptid;
2730 		vcpu->cpu = vc->pcpu;
2731 		vcpu->arch.thread_cpu = cpu;
2732 		cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
2733 	}
2734 	tpaca = paca_ptrs[cpu];
2735 	tpaca->kvm_hstate.kvm_vcpu = vcpu;
2736 	tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
2737 	tpaca->kvm_hstate.fake_suspend = 0;
2738 	/* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */
2739 	smp_wmb();
2740 	tpaca->kvm_hstate.kvm_vcore = vc;
2741 	if (cpu != smp_processor_id())
2742 		kvmppc_ipi_thread(cpu);
2743 }
2744 
2745 static void kvmppc_wait_for_nap(int n_threads)
2746 {
2747 	int cpu = smp_processor_id();
2748 	int i, loops;
2749 
2750 	if (n_threads <= 1)
2751 		return;
2752 	for (loops = 0; loops < 1000000; ++loops) {
2753 		/*
2754 		 * Check if all threads are finished.
2755 		 * We set the vcore pointer when starting a thread
2756 		 * and the thread clears it when finished, so we look
2757 		 * for any threads that still have a non-NULL vcore ptr.
2758 		 */
2759 		for (i = 1; i < n_threads; ++i)
2760 			if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2761 				break;
2762 		if (i == n_threads) {
2763 			HMT_medium();
2764 			return;
2765 		}
2766 		HMT_low();
2767 	}
2768 	HMT_medium();
2769 	for (i = 1; i < n_threads; ++i)
2770 		if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2771 			pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
2772 }
2773 
2774 /*
2775  * Check that we are on thread 0 and that any other threads in
2776  * this core are off-line.  Then grab the threads so they can't
2777  * enter the kernel.
2778  */
2779 static int on_primary_thread(void)
2780 {
2781 	int cpu = smp_processor_id();
2782 	int thr;
2783 
2784 	/* Are we on a primary subcore? */
2785 	if (cpu_thread_in_subcore(cpu))
2786 		return 0;
2787 
2788 	thr = 0;
2789 	while (++thr < threads_per_subcore)
2790 		if (cpu_online(cpu + thr))
2791 			return 0;
2792 
2793 	/* Grab all hw threads so they can't go into the kernel */
2794 	for (thr = 1; thr < threads_per_subcore; ++thr) {
2795 		if (kvmppc_grab_hwthread(cpu + thr)) {
2796 			/* Couldn't grab one; let the others go */
2797 			do {
2798 				kvmppc_release_hwthread(cpu + thr);
2799 			} while (--thr > 0);
2800 			return 0;
2801 		}
2802 	}
2803 	return 1;
2804 }
2805 
2806 /*
2807  * A list of virtual cores for each physical CPU.
2808  * These are vcores that could run but their runner VCPU tasks are
2809  * (or may be) preempted.
2810  */
2811 struct preempted_vcore_list {
2812 	struct list_head	list;
2813 	spinlock_t		lock;
2814 };
2815 
2816 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
2817 
2818 static void init_vcore_lists(void)
2819 {
2820 	int cpu;
2821 
2822 	for_each_possible_cpu(cpu) {
2823 		struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
2824 		spin_lock_init(&lp->lock);
2825 		INIT_LIST_HEAD(&lp->list);
2826 	}
2827 }
2828 
2829 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
2830 {
2831 	struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2832 
2833 	vc->vcore_state = VCORE_PREEMPT;
2834 	vc->pcpu = smp_processor_id();
2835 	if (vc->num_threads < threads_per_vcore(vc->kvm)) {
2836 		spin_lock(&lp->lock);
2837 		list_add_tail(&vc->preempt_list, &lp->list);
2838 		spin_unlock(&lp->lock);
2839 	}
2840 
2841 	/* Start accumulating stolen time */
2842 	kvmppc_core_start_stolen(vc);
2843 }
2844 
2845 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
2846 {
2847 	struct preempted_vcore_list *lp;
2848 
2849 	kvmppc_core_end_stolen(vc);
2850 	if (!list_empty(&vc->preempt_list)) {
2851 		lp = &per_cpu(preempted_vcores, vc->pcpu);
2852 		spin_lock(&lp->lock);
2853 		list_del_init(&vc->preempt_list);
2854 		spin_unlock(&lp->lock);
2855 	}
2856 	vc->vcore_state = VCORE_INACTIVE;
2857 }
2858 
2859 /*
2860  * This stores information about the virtual cores currently
2861  * assigned to a physical core.
2862  */
2863 struct core_info {
2864 	int		n_subcores;
2865 	int		max_subcore_threads;
2866 	int		total_threads;
2867 	int		subcore_threads[MAX_SUBCORES];
2868 	struct kvmppc_vcore *vc[MAX_SUBCORES];
2869 };
2870 
2871 /*
2872  * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7
2873  * respectively in 2-way micro-threading (split-core) mode on POWER8.
2874  */
2875 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
2876 
2877 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
2878 {
2879 	memset(cip, 0, sizeof(*cip));
2880 	cip->n_subcores = 1;
2881 	cip->max_subcore_threads = vc->num_threads;
2882 	cip->total_threads = vc->num_threads;
2883 	cip->subcore_threads[0] = vc->num_threads;
2884 	cip->vc[0] = vc;
2885 }
2886 
2887 static bool subcore_config_ok(int n_subcores, int n_threads)
2888 {
2889 	/*
2890 	 * POWER9 "SMT4" cores are permanently in what is effectively a 4-way
2891 	 * split-core mode, with one thread per subcore.
2892 	 */
2893 	if (cpu_has_feature(CPU_FTR_ARCH_300))
2894 		return n_subcores <= 4 && n_threads == 1;
2895 
2896 	/* On POWER8, can only dynamically split if unsplit to begin with */
2897 	if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
2898 		return false;
2899 	if (n_subcores > MAX_SUBCORES)
2900 		return false;
2901 	if (n_subcores > 1) {
2902 		if (!(dynamic_mt_modes & 2))
2903 			n_subcores = 4;
2904 		if (n_subcores > 2 && !(dynamic_mt_modes & 4))
2905 			return false;
2906 	}
2907 
2908 	return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
2909 }
2910 
2911 static void init_vcore_to_run(struct kvmppc_vcore *vc)
2912 {
2913 	vc->entry_exit_map = 0;
2914 	vc->in_guest = 0;
2915 	vc->napping_threads = 0;
2916 	vc->conferring_threads = 0;
2917 	vc->tb_offset_applied = 0;
2918 }
2919 
2920 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
2921 {
2922 	int n_threads = vc->num_threads;
2923 	int sub;
2924 
2925 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2926 		return false;
2927 
2928 	/* In one_vm_per_core mode, require all vcores to be from the same vm */
2929 	if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
2930 		return false;
2931 
2932 	if (n_threads < cip->max_subcore_threads)
2933 		n_threads = cip->max_subcore_threads;
2934 	if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
2935 		return false;
2936 	cip->max_subcore_threads = n_threads;
2937 
2938 	sub = cip->n_subcores;
2939 	++cip->n_subcores;
2940 	cip->total_threads += vc->num_threads;
2941 	cip->subcore_threads[sub] = vc->num_threads;
2942 	cip->vc[sub] = vc;
2943 	init_vcore_to_run(vc);
2944 	list_del_init(&vc->preempt_list);
2945 
2946 	return true;
2947 }
2948 
2949 /*
2950  * Work out whether it is possible to piggyback the execution of
2951  * vcore *pvc onto the execution of the other vcores described in *cip.
2952  */
2953 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
2954 			  int target_threads)
2955 {
2956 	if (cip->total_threads + pvc->num_threads > target_threads)
2957 		return false;
2958 
2959 	return can_dynamic_split(pvc, cip);
2960 }
2961 
2962 static void prepare_threads(struct kvmppc_vcore *vc)
2963 {
2964 	int i;
2965 	struct kvm_vcpu *vcpu;
2966 
2967 	for_each_runnable_thread(i, vcpu, vc) {
2968 		if (signal_pending(vcpu->arch.run_task))
2969 			vcpu->arch.ret = -EINTR;
2970 		else if (no_mixing_hpt_and_radix &&
2971 			 kvm_is_radix(vc->kvm) != radix_enabled())
2972 			vcpu->arch.ret = -EINVAL;
2973 		else if (vcpu->arch.vpa.update_pending ||
2974 			 vcpu->arch.slb_shadow.update_pending ||
2975 			 vcpu->arch.dtl.update_pending)
2976 			vcpu->arch.ret = RESUME_GUEST;
2977 		else
2978 			continue;
2979 		kvmppc_remove_runnable(vc, vcpu);
2980 		wake_up(&vcpu->arch.cpu_run);
2981 	}
2982 }
2983 
2984 static void collect_piggybacks(struct core_info *cip, int target_threads)
2985 {
2986 	struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2987 	struct kvmppc_vcore *pvc, *vcnext;
2988 
2989 	spin_lock(&lp->lock);
2990 	list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
2991 		if (!spin_trylock(&pvc->lock))
2992 			continue;
2993 		prepare_threads(pvc);
2994 		if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
2995 			list_del_init(&pvc->preempt_list);
2996 			if (pvc->runner == NULL) {
2997 				pvc->vcore_state = VCORE_INACTIVE;
2998 				kvmppc_core_end_stolen(pvc);
2999 			}
3000 			spin_unlock(&pvc->lock);
3001 			continue;
3002 		}
3003 		if (!can_piggyback(pvc, cip, target_threads)) {
3004 			spin_unlock(&pvc->lock);
3005 			continue;
3006 		}
3007 		kvmppc_core_end_stolen(pvc);
3008 		pvc->vcore_state = VCORE_PIGGYBACK;
3009 		if (cip->total_threads >= target_threads)
3010 			break;
3011 	}
3012 	spin_unlock(&lp->lock);
3013 }
3014 
3015 static bool recheck_signals_and_mmu(struct core_info *cip)
3016 {
3017 	int sub, i;
3018 	struct kvm_vcpu *vcpu;
3019 	struct kvmppc_vcore *vc;
3020 
3021 	for (sub = 0; sub < cip->n_subcores; ++sub) {
3022 		vc = cip->vc[sub];
3023 		if (!vc->kvm->arch.mmu_ready)
3024 			return true;
3025 		for_each_runnable_thread(i, vcpu, vc)
3026 			if (signal_pending(vcpu->arch.run_task))
3027 				return true;
3028 	}
3029 	return false;
3030 }
3031 
3032 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
3033 {
3034 	int still_running = 0, i;
3035 	u64 now;
3036 	long ret;
3037 	struct kvm_vcpu *vcpu;
3038 
3039 	spin_lock(&vc->lock);
3040 	now = get_tb();
3041 	for_each_runnable_thread(i, vcpu, vc) {
3042 		/*
3043 		 * It's safe to unlock the vcore in the loop here, because
3044 		 * for_each_runnable_thread() is safe against removal of
3045 		 * the vcpu, and the vcore state is VCORE_EXITING here,
3046 		 * so any vcpus becoming runnable will have their arch.trap
3047 		 * set to zero and can't actually run in the guest.
3048 		 */
3049 		spin_unlock(&vc->lock);
3050 		/* cancel pending dec exception if dec is positive */
3051 		if (now < vcpu->arch.dec_expires &&
3052 		    kvmppc_core_pending_dec(vcpu))
3053 			kvmppc_core_dequeue_dec(vcpu);
3054 
3055 		trace_kvm_guest_exit(vcpu);
3056 
3057 		ret = RESUME_GUEST;
3058 		if (vcpu->arch.trap)
3059 			ret = kvmppc_handle_exit_hv(vcpu,
3060 						    vcpu->arch.run_task);
3061 
3062 		vcpu->arch.ret = ret;
3063 		vcpu->arch.trap = 0;
3064 
3065 		spin_lock(&vc->lock);
3066 		if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
3067 			if (vcpu->arch.pending_exceptions)
3068 				kvmppc_core_prepare_to_enter(vcpu);
3069 			if (vcpu->arch.ceded)
3070 				kvmppc_set_timer(vcpu);
3071 			else
3072 				++still_running;
3073 		} else {
3074 			kvmppc_remove_runnable(vc, vcpu);
3075 			wake_up(&vcpu->arch.cpu_run);
3076 		}
3077 	}
3078 	if (!is_master) {
3079 		if (still_running > 0) {
3080 			kvmppc_vcore_preempt(vc);
3081 		} else if (vc->runner) {
3082 			vc->vcore_state = VCORE_PREEMPT;
3083 			kvmppc_core_start_stolen(vc);
3084 		} else {
3085 			vc->vcore_state = VCORE_INACTIVE;
3086 		}
3087 		if (vc->n_runnable > 0 && vc->runner == NULL) {
3088 			/* make sure there's a candidate runner awake */
3089 			i = -1;
3090 			vcpu = next_runnable_thread(vc, &i);
3091 			wake_up(&vcpu->arch.cpu_run);
3092 		}
3093 	}
3094 	spin_unlock(&vc->lock);
3095 }
3096 
3097 /*
3098  * Clear core from the list of active host cores as we are about to
3099  * enter the guest. Only do this if it is the primary thread of the
3100  * core (not if a subcore) that is entering the guest.
3101  */
3102 static inline int kvmppc_clear_host_core(unsigned int cpu)
3103 {
3104 	int core;
3105 
3106 	if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3107 		return 0;
3108 	/*
3109 	 * Memory barrier can be omitted here as we will do a smp_wmb()
3110 	 * later in kvmppc_start_thread and we need ensure that state is
3111 	 * visible to other CPUs only after we enter guest.
3112 	 */
3113 	core = cpu >> threads_shift;
3114 	kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
3115 	return 0;
3116 }
3117 
3118 /*
3119  * Advertise this core as an active host core since we exited the guest
3120  * Only need to do this if it is the primary thread of the core that is
3121  * exiting.
3122  */
3123 static inline int kvmppc_set_host_core(unsigned int cpu)
3124 {
3125 	int core;
3126 
3127 	if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3128 		return 0;
3129 
3130 	/*
3131 	 * Memory barrier can be omitted here because we do a spin_unlock
3132 	 * immediately after this which provides the memory barrier.
3133 	 */
3134 	core = cpu >> threads_shift;
3135 	kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
3136 	return 0;
3137 }
3138 
3139 static void set_irq_happened(int trap)
3140 {
3141 	switch (trap) {
3142 	case BOOK3S_INTERRUPT_EXTERNAL:
3143 		local_paca->irq_happened |= PACA_IRQ_EE;
3144 		break;
3145 	case BOOK3S_INTERRUPT_H_DOORBELL:
3146 		local_paca->irq_happened |= PACA_IRQ_DBELL;
3147 		break;
3148 	case BOOK3S_INTERRUPT_HMI:
3149 		local_paca->irq_happened |= PACA_IRQ_HMI;
3150 		break;
3151 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
3152 		replay_system_reset();
3153 		break;
3154 	}
3155 }
3156 
3157 /*
3158  * Run a set of guest threads on a physical core.
3159  * Called with vc->lock held.
3160  */
3161 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
3162 {
3163 	struct kvm_vcpu *vcpu;
3164 	int i;
3165 	int srcu_idx;
3166 	struct core_info core_info;
3167 	struct kvmppc_vcore *pvc;
3168 	struct kvm_split_mode split_info, *sip;
3169 	int split, subcore_size, active;
3170 	int sub;
3171 	bool thr0_done;
3172 	unsigned long cmd_bit, stat_bit;
3173 	int pcpu, thr;
3174 	int target_threads;
3175 	int controlled_threads;
3176 	int trap;
3177 	bool is_power8;
3178 
3179 	/*
3180 	 * Remove from the list any threads that have a signal pending
3181 	 * or need a VPA update done
3182 	 */
3183 	prepare_threads(vc);
3184 
3185 	/* if the runner is no longer runnable, let the caller pick a new one */
3186 	if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
3187 		return;
3188 
3189 	/*
3190 	 * Initialize *vc.
3191 	 */
3192 	init_vcore_to_run(vc);
3193 	vc->preempt_tb = TB_NIL;
3194 
3195 	/*
3196 	 * Number of threads that we will be controlling: the same as
3197 	 * the number of threads per subcore, except on POWER9,
3198 	 * where it's 1 because the threads are (mostly) independent.
3199 	 */
3200 	controlled_threads = threads_per_vcore(vc->kvm);
3201 
3202 	/*
3203 	 * Make sure we are running on primary threads, and that secondary
3204 	 * threads are offline.  Also check if the number of threads in this
3205 	 * guest are greater than the current system threads per guest.
3206 	 * On POWER9, we need to be not in independent-threads mode if
3207 	 * this is a HPT guest on a radix host machine where the
3208 	 * CPU threads may not be in different MMU modes.
3209 	 */
3210 	if ((controlled_threads > 1) &&
3211 	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
3212 		for_each_runnable_thread(i, vcpu, vc) {
3213 			vcpu->arch.ret = -EBUSY;
3214 			kvmppc_remove_runnable(vc, vcpu);
3215 			wake_up(&vcpu->arch.cpu_run);
3216 		}
3217 		goto out;
3218 	}
3219 
3220 	/*
3221 	 * See if we could run any other vcores on the physical core
3222 	 * along with this one.
3223 	 */
3224 	init_core_info(&core_info, vc);
3225 	pcpu = smp_processor_id();
3226 	target_threads = controlled_threads;
3227 	if (target_smt_mode && target_smt_mode < target_threads)
3228 		target_threads = target_smt_mode;
3229 	if (vc->num_threads < target_threads)
3230 		collect_piggybacks(&core_info, target_threads);
3231 
3232 	/*
3233 	 * On radix, arrange for TLB flushing if necessary.
3234 	 * This has to be done before disabling interrupts since
3235 	 * it uses smp_call_function().
3236 	 */
3237 	pcpu = smp_processor_id();
3238 	if (kvm_is_radix(vc->kvm)) {
3239 		for (sub = 0; sub < core_info.n_subcores; ++sub)
3240 			for_each_runnable_thread(i, vcpu, core_info.vc[sub])
3241 				kvmppc_prepare_radix_vcpu(vcpu, pcpu);
3242 	}
3243 
3244 	/*
3245 	 * Hard-disable interrupts, and check resched flag and signals.
3246 	 * If we need to reschedule or deliver a signal, clean up
3247 	 * and return without going into the guest(s).
3248 	 * If the mmu_ready flag has been cleared, don't go into the
3249 	 * guest because that means a HPT resize operation is in progress.
3250 	 */
3251 	local_irq_disable();
3252 	hard_irq_disable();
3253 	if (lazy_irq_pending() || need_resched() ||
3254 	    recheck_signals_and_mmu(&core_info)) {
3255 		local_irq_enable();
3256 		vc->vcore_state = VCORE_INACTIVE;
3257 		/* Unlock all except the primary vcore */
3258 		for (sub = 1; sub < core_info.n_subcores; ++sub) {
3259 			pvc = core_info.vc[sub];
3260 			/* Put back on to the preempted vcores list */
3261 			kvmppc_vcore_preempt(pvc);
3262 			spin_unlock(&pvc->lock);
3263 		}
3264 		for (i = 0; i < controlled_threads; ++i)
3265 			kvmppc_release_hwthread(pcpu + i);
3266 		return;
3267 	}
3268 
3269 	kvmppc_clear_host_core(pcpu);
3270 
3271 	/* Decide on micro-threading (split-core) mode */
3272 	subcore_size = threads_per_subcore;
3273 	cmd_bit = stat_bit = 0;
3274 	split = core_info.n_subcores;
3275 	sip = NULL;
3276 	is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
3277 		&& !cpu_has_feature(CPU_FTR_ARCH_300);
3278 
3279 	if (split > 1) {
3280 		sip = &split_info;
3281 		memset(&split_info, 0, sizeof(split_info));
3282 		for (sub = 0; sub < core_info.n_subcores; ++sub)
3283 			split_info.vc[sub] = core_info.vc[sub];
3284 
3285 		if (is_power8) {
3286 			if (split == 2 && (dynamic_mt_modes & 2)) {
3287 				cmd_bit = HID0_POWER8_1TO2LPAR;
3288 				stat_bit = HID0_POWER8_2LPARMODE;
3289 			} else {
3290 				split = 4;
3291 				cmd_bit = HID0_POWER8_1TO4LPAR;
3292 				stat_bit = HID0_POWER8_4LPARMODE;
3293 			}
3294 			subcore_size = MAX_SMT_THREADS / split;
3295 			split_info.rpr = mfspr(SPRN_RPR);
3296 			split_info.pmmar = mfspr(SPRN_PMMAR);
3297 			split_info.ldbar = mfspr(SPRN_LDBAR);
3298 			split_info.subcore_size = subcore_size;
3299 		} else {
3300 			split_info.subcore_size = 1;
3301 		}
3302 
3303 		/* order writes to split_info before kvm_split_mode pointer */
3304 		smp_wmb();
3305 	}
3306 
3307 	for (thr = 0; thr < controlled_threads; ++thr) {
3308 		struct paca_struct *paca = paca_ptrs[pcpu + thr];
3309 
3310 		paca->kvm_hstate.napping = 0;
3311 		paca->kvm_hstate.kvm_split_mode = sip;
3312 	}
3313 
3314 	/* Initiate micro-threading (split-core) on POWER8 if required */
3315 	if (cmd_bit) {
3316 		unsigned long hid0 = mfspr(SPRN_HID0);
3317 
3318 		hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
3319 		mb();
3320 		mtspr(SPRN_HID0, hid0);
3321 		isync();
3322 		for (;;) {
3323 			hid0 = mfspr(SPRN_HID0);
3324 			if (hid0 & stat_bit)
3325 				break;
3326 			cpu_relax();
3327 		}
3328 	}
3329 
3330 	/*
3331 	 * On POWER8, set RWMR register.
3332 	 * Since it only affects PURR and SPURR, it doesn't affect
3333 	 * the host, so we don't save/restore the host value.
3334 	 */
3335 	if (is_power8) {
3336 		unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
3337 		int n_online = atomic_read(&vc->online_count);
3338 
3339 		/*
3340 		 * Use the 8-thread value if we're doing split-core
3341 		 * or if the vcore's online count looks bogus.
3342 		 */
3343 		if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
3344 		    n_online >= 1 && n_online <= MAX_SMT_THREADS)
3345 			rwmr_val = p8_rwmr_values[n_online];
3346 		mtspr(SPRN_RWMR, rwmr_val);
3347 	}
3348 
3349 	/* Start all the threads */
3350 	active = 0;
3351 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
3352 		thr = is_power8 ? subcore_thread_map[sub] : sub;
3353 		thr0_done = false;
3354 		active |= 1 << thr;
3355 		pvc = core_info.vc[sub];
3356 		pvc->pcpu = pcpu + thr;
3357 		for_each_runnable_thread(i, vcpu, pvc) {
3358 			kvmppc_start_thread(vcpu, pvc);
3359 			kvmppc_create_dtl_entry(vcpu, pvc);
3360 			trace_kvm_guest_enter(vcpu);
3361 			if (!vcpu->arch.ptid)
3362 				thr0_done = true;
3363 			active |= 1 << (thr + vcpu->arch.ptid);
3364 		}
3365 		/*
3366 		 * We need to start the first thread of each subcore
3367 		 * even if it doesn't have a vcpu.
3368 		 */
3369 		if (!thr0_done)
3370 			kvmppc_start_thread(NULL, pvc);
3371 	}
3372 
3373 	/*
3374 	 * Ensure that split_info.do_nap is set after setting
3375 	 * the vcore pointer in the PACA of the secondaries.
3376 	 */
3377 	smp_mb();
3378 
3379 	/*
3380 	 * When doing micro-threading, poke the inactive threads as well.
3381 	 * This gets them to the nap instruction after kvm_do_nap,
3382 	 * which reduces the time taken to unsplit later.
3383 	 */
3384 	if (cmd_bit) {
3385 		split_info.do_nap = 1;	/* ask secondaries to nap when done */
3386 		for (thr = 1; thr < threads_per_subcore; ++thr)
3387 			if (!(active & (1 << thr)))
3388 				kvmppc_ipi_thread(pcpu + thr);
3389 	}
3390 
3391 	vc->vcore_state = VCORE_RUNNING;
3392 	preempt_disable();
3393 
3394 	trace_kvmppc_run_core(vc, 0);
3395 
3396 	for (sub = 0; sub < core_info.n_subcores; ++sub)
3397 		spin_unlock(&core_info.vc[sub]->lock);
3398 
3399 	guest_enter_irqoff();
3400 
3401 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
3402 
3403 	this_cpu_disable_ftrace();
3404 
3405 	/*
3406 	 * Interrupts will be enabled once we get into the guest,
3407 	 * so tell lockdep that we're about to enable interrupts.
3408 	 */
3409 	trace_hardirqs_on();
3410 
3411 	trap = __kvmppc_vcore_entry();
3412 
3413 	trace_hardirqs_off();
3414 
3415 	this_cpu_enable_ftrace();
3416 
3417 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
3418 
3419 	set_irq_happened(trap);
3420 
3421 	spin_lock(&vc->lock);
3422 	/* prevent other vcpu threads from doing kvmppc_start_thread() now */
3423 	vc->vcore_state = VCORE_EXITING;
3424 
3425 	/* wait for secondary threads to finish writing their state to memory */
3426 	kvmppc_wait_for_nap(controlled_threads);
3427 
3428 	/* Return to whole-core mode if we split the core earlier */
3429 	if (cmd_bit) {
3430 		unsigned long hid0 = mfspr(SPRN_HID0);
3431 		unsigned long loops = 0;
3432 
3433 		hid0 &= ~HID0_POWER8_DYNLPARDIS;
3434 		stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
3435 		mb();
3436 		mtspr(SPRN_HID0, hid0);
3437 		isync();
3438 		for (;;) {
3439 			hid0 = mfspr(SPRN_HID0);
3440 			if (!(hid0 & stat_bit))
3441 				break;
3442 			cpu_relax();
3443 			++loops;
3444 		}
3445 		split_info.do_nap = 0;
3446 	}
3447 
3448 	kvmppc_set_host_core(pcpu);
3449 
3450 	guest_exit_irqoff();
3451 
3452 	local_irq_enable();
3453 
3454 	/* Let secondaries go back to the offline loop */
3455 	for (i = 0; i < controlled_threads; ++i) {
3456 		kvmppc_release_hwthread(pcpu + i);
3457 		if (sip && sip->napped[i])
3458 			kvmppc_ipi_thread(pcpu + i);
3459 		cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
3460 	}
3461 
3462 	spin_unlock(&vc->lock);
3463 
3464 	/* make sure updates to secondary vcpu structs are visible now */
3465 	smp_mb();
3466 
3467 	preempt_enable();
3468 
3469 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
3470 		pvc = core_info.vc[sub];
3471 		post_guest_process(pvc, pvc == vc);
3472 	}
3473 
3474 	spin_lock(&vc->lock);
3475 
3476  out:
3477 	vc->vcore_state = VCORE_INACTIVE;
3478 	trace_kvmppc_run_core(vc, 1);
3479 }
3480 
3481 /*
3482  * Load up hypervisor-mode registers on P9.
3483  */
3484 static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit,
3485 				     unsigned long lpcr)
3486 {
3487 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
3488 	s64 hdec;
3489 	u64 tb, purr, spurr;
3490 	int trap;
3491 	unsigned long host_hfscr = mfspr(SPRN_HFSCR);
3492 	unsigned long host_ciabr = mfspr(SPRN_CIABR);
3493 	unsigned long host_dawr0 = mfspr(SPRN_DAWR0);
3494 	unsigned long host_dawrx0 = mfspr(SPRN_DAWRX0);
3495 	unsigned long host_psscr = mfspr(SPRN_PSSCR);
3496 	unsigned long host_pidr = mfspr(SPRN_PID);
3497 	unsigned long host_dawr1 = 0;
3498 	unsigned long host_dawrx1 = 0;
3499 
3500 	if (cpu_has_feature(CPU_FTR_DAWR1)) {
3501 		host_dawr1 = mfspr(SPRN_DAWR1);
3502 		host_dawrx1 = mfspr(SPRN_DAWRX1);
3503 	}
3504 
3505 	/*
3506 	 * P8 and P9 suppress the HDEC exception when LPCR[HDICE] = 0,
3507 	 * so set HDICE before writing HDEC.
3508 	 */
3509 	mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr | LPCR_HDICE);
3510 	isync();
3511 
3512 	hdec = time_limit - mftb();
3513 	if (hdec < 0) {
3514 		mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr);
3515 		isync();
3516 		return BOOK3S_INTERRUPT_HV_DECREMENTER;
3517 	}
3518 	mtspr(SPRN_HDEC, hdec);
3519 
3520 	if (vc->tb_offset) {
3521 		u64 new_tb = mftb() + vc->tb_offset;
3522 		mtspr(SPRN_TBU40, new_tb);
3523 		tb = mftb();
3524 		if ((tb & 0xffffff) < (new_tb & 0xffffff))
3525 			mtspr(SPRN_TBU40, new_tb + 0x1000000);
3526 		vc->tb_offset_applied = vc->tb_offset;
3527 	}
3528 
3529 	if (vc->pcr)
3530 		mtspr(SPRN_PCR, vc->pcr | PCR_MASK);
3531 	mtspr(SPRN_DPDES, vc->dpdes);
3532 	mtspr(SPRN_VTB, vc->vtb);
3533 
3534 	local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
3535 	local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
3536 	mtspr(SPRN_PURR, vcpu->arch.purr);
3537 	mtspr(SPRN_SPURR, vcpu->arch.spurr);
3538 
3539 	if (dawr_enabled()) {
3540 		mtspr(SPRN_DAWR0, vcpu->arch.dawr0);
3541 		mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0);
3542 		if (cpu_has_feature(CPU_FTR_DAWR1)) {
3543 			mtspr(SPRN_DAWR1, vcpu->arch.dawr1);
3544 			mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1);
3545 		}
3546 	}
3547 	mtspr(SPRN_CIABR, vcpu->arch.ciabr);
3548 	mtspr(SPRN_IC, vcpu->arch.ic);
3549 	mtspr(SPRN_PID, vcpu->arch.pid);
3550 
3551 	mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
3552 	      (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3553 
3554 	mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
3555 
3556 	mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
3557 	mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
3558 	mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
3559 	mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
3560 
3561 	mtspr(SPRN_AMOR, ~0UL);
3562 
3563 	mtspr(SPRN_LPCR, lpcr);
3564 	isync();
3565 
3566 	kvmppc_xive_push_vcpu(vcpu);
3567 
3568 	mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
3569 	mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
3570 
3571 	trap = __kvmhv_vcpu_entry_p9(vcpu);
3572 
3573 	/* Advance host PURR/SPURR by the amount used by guest */
3574 	purr = mfspr(SPRN_PURR);
3575 	spurr = mfspr(SPRN_SPURR);
3576 	mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr +
3577 	      purr - vcpu->arch.purr);
3578 	mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr +
3579 	      spurr - vcpu->arch.spurr);
3580 	vcpu->arch.purr = purr;
3581 	vcpu->arch.spurr = spurr;
3582 
3583 	vcpu->arch.ic = mfspr(SPRN_IC);
3584 	vcpu->arch.pid = mfspr(SPRN_PID);
3585 	vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS;
3586 
3587 	vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
3588 	vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
3589 	vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
3590 	vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
3591 
3592 	/* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */
3593 	mtspr(SPRN_PSSCR, host_psscr |
3594 	      (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3595 	mtspr(SPRN_HFSCR, host_hfscr);
3596 	mtspr(SPRN_CIABR, host_ciabr);
3597 	mtspr(SPRN_DAWR0, host_dawr0);
3598 	mtspr(SPRN_DAWRX0, host_dawrx0);
3599 	if (cpu_has_feature(CPU_FTR_DAWR1)) {
3600 		mtspr(SPRN_DAWR1, host_dawr1);
3601 		mtspr(SPRN_DAWRX1, host_dawrx1);
3602 	}
3603 	mtspr(SPRN_PID, host_pidr);
3604 
3605 	/*
3606 	 * Since this is radix, do a eieio; tlbsync; ptesync sequence in
3607 	 * case we interrupted the guest between a tlbie and a ptesync.
3608 	 */
3609 	asm volatile("eieio; tlbsync; ptesync");
3610 
3611 	/*
3612 	 * cp_abort is required if the processor supports local copy-paste
3613 	 * to clear the copy buffer that was under control of the guest.
3614 	 */
3615 	if (cpu_has_feature(CPU_FTR_ARCH_31))
3616 		asm volatile(PPC_CP_ABORT);
3617 
3618 	mtspr(SPRN_LPID, vcpu->kvm->arch.host_lpid);	/* restore host LPID */
3619 	isync();
3620 
3621 	vc->dpdes = mfspr(SPRN_DPDES);
3622 	vc->vtb = mfspr(SPRN_VTB);
3623 	mtspr(SPRN_DPDES, 0);
3624 	if (vc->pcr)
3625 		mtspr(SPRN_PCR, PCR_MASK);
3626 
3627 	if (vc->tb_offset_applied) {
3628 		u64 new_tb = mftb() - vc->tb_offset_applied;
3629 		mtspr(SPRN_TBU40, new_tb);
3630 		tb = mftb();
3631 		if ((tb & 0xffffff) < (new_tb & 0xffffff))
3632 			mtspr(SPRN_TBU40, new_tb + 0x1000000);
3633 		vc->tb_offset_applied = 0;
3634 	}
3635 
3636 	mtspr(SPRN_HDEC, 0x7fffffff);
3637 	mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr);
3638 
3639 	return trap;
3640 }
3641 
3642 /*
3643  * Virtual-mode guest entry for POWER9 and later when the host and
3644  * guest are both using the radix MMU.  The LPIDR has already been set.
3645  */
3646 static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
3647 			 unsigned long lpcr)
3648 {
3649 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
3650 	unsigned long host_dscr = mfspr(SPRN_DSCR);
3651 	unsigned long host_tidr = mfspr(SPRN_TIDR);
3652 	unsigned long host_iamr = mfspr(SPRN_IAMR);
3653 	unsigned long host_amr = mfspr(SPRN_AMR);
3654 	unsigned long host_fscr = mfspr(SPRN_FSCR);
3655 	s64 dec;
3656 	u64 tb;
3657 	int trap, save_pmu;
3658 
3659 	dec = mfspr(SPRN_DEC);
3660 	tb = mftb();
3661 	if (dec < 0)
3662 		return BOOK3S_INTERRUPT_HV_DECREMENTER;
3663 	local_paca->kvm_hstate.dec_expires = dec + tb;
3664 	if (local_paca->kvm_hstate.dec_expires < time_limit)
3665 		time_limit = local_paca->kvm_hstate.dec_expires;
3666 
3667 	vcpu->arch.ceded = 0;
3668 
3669 	kvmhv_save_host_pmu();		/* saves it to PACA kvm_hstate */
3670 
3671 	kvmppc_subcore_enter_guest();
3672 
3673 	vc->entry_exit_map = 1;
3674 	vc->in_guest = 1;
3675 
3676 	if (vcpu->arch.vpa.pinned_addr) {
3677 		struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3678 		u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3679 		lp->yield_count = cpu_to_be32(yield_count);
3680 		vcpu->arch.vpa.dirty = 1;
3681 	}
3682 
3683 	if (cpu_has_feature(CPU_FTR_TM) ||
3684 	    cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3685 		kvmppc_restore_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3686 
3687 	kvmhv_load_guest_pmu(vcpu);
3688 
3689 	msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3690 	load_fp_state(&vcpu->arch.fp);
3691 #ifdef CONFIG_ALTIVEC
3692 	load_vr_state(&vcpu->arch.vr);
3693 #endif
3694 	mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
3695 
3696 	mtspr(SPRN_DSCR, vcpu->arch.dscr);
3697 	mtspr(SPRN_IAMR, vcpu->arch.iamr);
3698 	mtspr(SPRN_PSPB, vcpu->arch.pspb);
3699 	mtspr(SPRN_FSCR, vcpu->arch.fscr);
3700 	mtspr(SPRN_TAR, vcpu->arch.tar);
3701 	mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
3702 	mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
3703 	mtspr(SPRN_BESCR, vcpu->arch.bescr);
3704 	mtspr(SPRN_WORT, vcpu->arch.wort);
3705 	mtspr(SPRN_TIDR, vcpu->arch.tid);
3706 	mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
3707 	mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
3708 	mtspr(SPRN_AMR, vcpu->arch.amr);
3709 	mtspr(SPRN_UAMOR, vcpu->arch.uamor);
3710 
3711 	if (!(vcpu->arch.ctrl & 1))
3712 		mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1);
3713 
3714 	mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb());
3715 
3716 	if (kvmhv_on_pseries()) {
3717 		/*
3718 		 * We need to save and restore the guest visible part of the
3719 		 * psscr (i.e. using SPRN_PSSCR_PR) since the hypervisor
3720 		 * doesn't do this for us. Note only required if pseries since
3721 		 * this is done in kvmhv_load_hv_regs_and_go() below otherwise.
3722 		 */
3723 		unsigned long host_psscr;
3724 		/* call our hypervisor to load up HV regs and go */
3725 		struct hv_guest_state hvregs;
3726 
3727 		host_psscr = mfspr(SPRN_PSSCR_PR);
3728 		mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
3729 		kvmhv_save_hv_regs(vcpu, &hvregs);
3730 		hvregs.lpcr = lpcr;
3731 		vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
3732 		hvregs.version = HV_GUEST_STATE_VERSION;
3733 		if (vcpu->arch.nested) {
3734 			hvregs.lpid = vcpu->arch.nested->shadow_lpid;
3735 			hvregs.vcpu_token = vcpu->arch.nested_vcpu_id;
3736 		} else {
3737 			hvregs.lpid = vcpu->kvm->arch.lpid;
3738 			hvregs.vcpu_token = vcpu->vcpu_id;
3739 		}
3740 		hvregs.hdec_expiry = time_limit;
3741 		trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs),
3742 					  __pa(&vcpu->arch.regs));
3743 		kvmhv_restore_hv_return_state(vcpu, &hvregs);
3744 		vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
3745 		vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
3746 		vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
3747 		vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
3748 		mtspr(SPRN_PSSCR_PR, host_psscr);
3749 
3750 		/* H_CEDE has to be handled now, not later */
3751 		if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested &&
3752 		    kvmppc_get_gpr(vcpu, 3) == H_CEDE) {
3753 			kvmppc_nested_cede(vcpu);
3754 			kvmppc_set_gpr(vcpu, 3, 0);
3755 			trap = 0;
3756 		}
3757 	} else {
3758 		trap = kvmhv_load_hv_regs_and_go(vcpu, time_limit, lpcr);
3759 	}
3760 
3761 	vcpu->arch.slb_max = 0;
3762 	dec = mfspr(SPRN_DEC);
3763 	if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */
3764 		dec = (s32) dec;
3765 	tb = mftb();
3766 	vcpu->arch.dec_expires = dec + tb;
3767 	vcpu->cpu = -1;
3768 	vcpu->arch.thread_cpu = -1;
3769 	/* Save guest CTRL register, set runlatch to 1 */
3770 	vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
3771 	if (!(vcpu->arch.ctrl & 1))
3772 		mtspr(SPRN_CTRLT, vcpu->arch.ctrl | 1);
3773 
3774 	vcpu->arch.iamr = mfspr(SPRN_IAMR);
3775 	vcpu->arch.pspb = mfspr(SPRN_PSPB);
3776 	vcpu->arch.fscr = mfspr(SPRN_FSCR);
3777 	vcpu->arch.tar = mfspr(SPRN_TAR);
3778 	vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
3779 	vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
3780 	vcpu->arch.bescr = mfspr(SPRN_BESCR);
3781 	vcpu->arch.wort = mfspr(SPRN_WORT);
3782 	vcpu->arch.tid = mfspr(SPRN_TIDR);
3783 	vcpu->arch.amr = mfspr(SPRN_AMR);
3784 	vcpu->arch.uamor = mfspr(SPRN_UAMOR);
3785 	vcpu->arch.dscr = mfspr(SPRN_DSCR);
3786 
3787 	mtspr(SPRN_PSPB, 0);
3788 	mtspr(SPRN_WORT, 0);
3789 	mtspr(SPRN_UAMOR, 0);
3790 	mtspr(SPRN_DSCR, host_dscr);
3791 	mtspr(SPRN_TIDR, host_tidr);
3792 	mtspr(SPRN_IAMR, host_iamr);
3793 
3794 	if (host_amr != vcpu->arch.amr)
3795 		mtspr(SPRN_AMR, host_amr);
3796 
3797 	if (host_fscr != vcpu->arch.fscr)
3798 		mtspr(SPRN_FSCR, host_fscr);
3799 
3800 	msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3801 	store_fp_state(&vcpu->arch.fp);
3802 #ifdef CONFIG_ALTIVEC
3803 	store_vr_state(&vcpu->arch.vr);
3804 #endif
3805 	vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
3806 
3807 	if (cpu_has_feature(CPU_FTR_TM) ||
3808 	    cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3809 		kvmppc_save_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3810 
3811 	save_pmu = 1;
3812 	if (vcpu->arch.vpa.pinned_addr) {
3813 		struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3814 		u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3815 		lp->yield_count = cpu_to_be32(yield_count);
3816 		vcpu->arch.vpa.dirty = 1;
3817 		save_pmu = lp->pmcregs_in_use;
3818 	}
3819 	/* Must save pmu if this guest is capable of running nested guests */
3820 	save_pmu |= nesting_enabled(vcpu->kvm);
3821 
3822 	kvmhv_save_guest_pmu(vcpu, save_pmu);
3823 
3824 	vc->entry_exit_map = 0x101;
3825 	vc->in_guest = 0;
3826 
3827 	mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb());
3828 	mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
3829 
3830 	kvmhv_load_host_pmu();
3831 
3832 	kvmppc_subcore_exit_guest();
3833 
3834 	return trap;
3835 }
3836 
3837 /*
3838  * Wait for some other vcpu thread to execute us, and
3839  * wake us up when we need to handle something in the host.
3840  */
3841 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
3842 				 struct kvm_vcpu *vcpu, int wait_state)
3843 {
3844 	DEFINE_WAIT(wait);
3845 
3846 	prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
3847 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
3848 		spin_unlock(&vc->lock);
3849 		schedule();
3850 		spin_lock(&vc->lock);
3851 	}
3852 	finish_wait(&vcpu->arch.cpu_run, &wait);
3853 }
3854 
3855 static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
3856 {
3857 	if (!halt_poll_ns_grow)
3858 		return;
3859 
3860 	vc->halt_poll_ns *= halt_poll_ns_grow;
3861 	if (vc->halt_poll_ns < halt_poll_ns_grow_start)
3862 		vc->halt_poll_ns = halt_poll_ns_grow_start;
3863 }
3864 
3865 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
3866 {
3867 	if (halt_poll_ns_shrink == 0)
3868 		vc->halt_poll_ns = 0;
3869 	else
3870 		vc->halt_poll_ns /= halt_poll_ns_shrink;
3871 }
3872 
3873 #ifdef CONFIG_KVM_XICS
3874 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3875 {
3876 	if (!xics_on_xive())
3877 		return false;
3878 	return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
3879 		vcpu->arch.xive_saved_state.cppr;
3880 }
3881 #else
3882 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3883 {
3884 	return false;
3885 }
3886 #endif /* CONFIG_KVM_XICS */
3887 
3888 static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
3889 {
3890 	if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
3891 	    kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
3892 		return true;
3893 
3894 	return false;
3895 }
3896 
3897 /*
3898  * Check to see if any of the runnable vcpus on the vcore have pending
3899  * exceptions or are no longer ceded
3900  */
3901 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
3902 {
3903 	struct kvm_vcpu *vcpu;
3904 	int i;
3905 
3906 	for_each_runnable_thread(i, vcpu, vc) {
3907 		if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
3908 			return 1;
3909 	}
3910 
3911 	return 0;
3912 }
3913 
3914 /*
3915  * All the vcpus in this vcore are idle, so wait for a decrementer
3916  * or external interrupt to one of the vcpus.  vc->lock is held.
3917  */
3918 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
3919 {
3920 	ktime_t cur, start_poll, start_wait;
3921 	int do_sleep = 1;
3922 	u64 block_ns;
3923 
3924 	/* Poll for pending exceptions and ceded state */
3925 	cur = start_poll = ktime_get();
3926 	if (vc->halt_poll_ns) {
3927 		ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
3928 		++vc->runner->stat.halt_attempted_poll;
3929 
3930 		vc->vcore_state = VCORE_POLLING;
3931 		spin_unlock(&vc->lock);
3932 
3933 		do {
3934 			if (kvmppc_vcore_check_block(vc)) {
3935 				do_sleep = 0;
3936 				break;
3937 			}
3938 			cur = ktime_get();
3939 		} while (kvm_vcpu_can_poll(cur, stop));
3940 
3941 		spin_lock(&vc->lock);
3942 		vc->vcore_state = VCORE_INACTIVE;
3943 
3944 		if (!do_sleep) {
3945 			++vc->runner->stat.halt_successful_poll;
3946 			goto out;
3947 		}
3948 	}
3949 
3950 	prepare_to_rcuwait(&vc->wait);
3951 	set_current_state(TASK_INTERRUPTIBLE);
3952 	if (kvmppc_vcore_check_block(vc)) {
3953 		finish_rcuwait(&vc->wait);
3954 		do_sleep = 0;
3955 		/* If we polled, count this as a successful poll */
3956 		if (vc->halt_poll_ns)
3957 			++vc->runner->stat.halt_successful_poll;
3958 		goto out;
3959 	}
3960 
3961 	start_wait = ktime_get();
3962 
3963 	vc->vcore_state = VCORE_SLEEPING;
3964 	trace_kvmppc_vcore_blocked(vc, 0);
3965 	spin_unlock(&vc->lock);
3966 	schedule();
3967 	finish_rcuwait(&vc->wait);
3968 	spin_lock(&vc->lock);
3969 	vc->vcore_state = VCORE_INACTIVE;
3970 	trace_kvmppc_vcore_blocked(vc, 1);
3971 	++vc->runner->stat.halt_successful_wait;
3972 
3973 	cur = ktime_get();
3974 
3975 out:
3976 	block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
3977 
3978 	/* Attribute wait time */
3979 	if (do_sleep) {
3980 		vc->runner->stat.halt_wait_ns +=
3981 			ktime_to_ns(cur) - ktime_to_ns(start_wait);
3982 		/* Attribute failed poll time */
3983 		if (vc->halt_poll_ns)
3984 			vc->runner->stat.halt_poll_fail_ns +=
3985 				ktime_to_ns(start_wait) -
3986 				ktime_to_ns(start_poll);
3987 	} else {
3988 		/* Attribute successful poll time */
3989 		if (vc->halt_poll_ns)
3990 			vc->runner->stat.halt_poll_success_ns +=
3991 				ktime_to_ns(cur) -
3992 				ktime_to_ns(start_poll);
3993 	}
3994 
3995 	/* Adjust poll time */
3996 	if (halt_poll_ns) {
3997 		if (block_ns <= vc->halt_poll_ns)
3998 			;
3999 		/* We slept and blocked for longer than the max halt time */
4000 		else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
4001 			shrink_halt_poll_ns(vc);
4002 		/* We slept and our poll time is too small */
4003 		else if (vc->halt_poll_ns < halt_poll_ns &&
4004 				block_ns < halt_poll_ns)
4005 			grow_halt_poll_ns(vc);
4006 		if (vc->halt_poll_ns > halt_poll_ns)
4007 			vc->halt_poll_ns = halt_poll_ns;
4008 	} else
4009 		vc->halt_poll_ns = 0;
4010 
4011 	trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
4012 }
4013 
4014 /*
4015  * This never fails for a radix guest, as none of the operations it does
4016  * for a radix guest can fail or have a way to report failure.
4017  * kvmhv_run_single_vcpu() relies on this fact.
4018  */
4019 static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
4020 {
4021 	int r = 0;
4022 	struct kvm *kvm = vcpu->kvm;
4023 
4024 	mutex_lock(&kvm->arch.mmu_setup_lock);
4025 	if (!kvm->arch.mmu_ready) {
4026 		if (!kvm_is_radix(kvm))
4027 			r = kvmppc_hv_setup_htab_rma(vcpu);
4028 		if (!r) {
4029 			if (cpu_has_feature(CPU_FTR_ARCH_300))
4030 				kvmppc_setup_partition_table(kvm);
4031 			kvm->arch.mmu_ready = 1;
4032 		}
4033 	}
4034 	mutex_unlock(&kvm->arch.mmu_setup_lock);
4035 	return r;
4036 }
4037 
4038 static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu)
4039 {
4040 	struct kvm_run *run = vcpu->run;
4041 	int n_ceded, i, r;
4042 	struct kvmppc_vcore *vc;
4043 	struct kvm_vcpu *v;
4044 
4045 	trace_kvmppc_run_vcpu_enter(vcpu);
4046 
4047 	run->exit_reason = 0;
4048 	vcpu->arch.ret = RESUME_GUEST;
4049 	vcpu->arch.trap = 0;
4050 	kvmppc_update_vpas(vcpu);
4051 
4052 	/*
4053 	 * Synchronize with other threads in this virtual core
4054 	 */
4055 	vc = vcpu->arch.vcore;
4056 	spin_lock(&vc->lock);
4057 	vcpu->arch.ceded = 0;
4058 	vcpu->arch.run_task = current;
4059 	vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4060 	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4061 	vcpu->arch.busy_preempt = TB_NIL;
4062 	WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
4063 	++vc->n_runnable;
4064 
4065 	/*
4066 	 * This happens the first time this is called for a vcpu.
4067 	 * If the vcore is already running, we may be able to start
4068 	 * this thread straight away and have it join in.
4069 	 */
4070 	if (!signal_pending(current)) {
4071 		if ((vc->vcore_state == VCORE_PIGGYBACK ||
4072 		     vc->vcore_state == VCORE_RUNNING) &&
4073 			   !VCORE_IS_EXITING(vc)) {
4074 			kvmppc_create_dtl_entry(vcpu, vc);
4075 			kvmppc_start_thread(vcpu, vc);
4076 			trace_kvm_guest_enter(vcpu);
4077 		} else if (vc->vcore_state == VCORE_SLEEPING) {
4078 		        rcuwait_wake_up(&vc->wait);
4079 		}
4080 
4081 	}
4082 
4083 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4084 	       !signal_pending(current)) {
4085 		/* See if the MMU is ready to go */
4086 		if (!vcpu->kvm->arch.mmu_ready) {
4087 			spin_unlock(&vc->lock);
4088 			r = kvmhv_setup_mmu(vcpu);
4089 			spin_lock(&vc->lock);
4090 			if (r) {
4091 				run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4092 				run->fail_entry.
4093 					hardware_entry_failure_reason = 0;
4094 				vcpu->arch.ret = r;
4095 				break;
4096 			}
4097 		}
4098 
4099 		if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4100 			kvmppc_vcore_end_preempt(vc);
4101 
4102 		if (vc->vcore_state != VCORE_INACTIVE) {
4103 			kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
4104 			continue;
4105 		}
4106 		for_each_runnable_thread(i, v, vc) {
4107 			kvmppc_core_prepare_to_enter(v);
4108 			if (signal_pending(v->arch.run_task)) {
4109 				kvmppc_remove_runnable(vc, v);
4110 				v->stat.signal_exits++;
4111 				v->run->exit_reason = KVM_EXIT_INTR;
4112 				v->arch.ret = -EINTR;
4113 				wake_up(&v->arch.cpu_run);
4114 			}
4115 		}
4116 		if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
4117 			break;
4118 		n_ceded = 0;
4119 		for_each_runnable_thread(i, v, vc) {
4120 			if (!kvmppc_vcpu_woken(v))
4121 				n_ceded += v->arch.ceded;
4122 			else
4123 				v->arch.ceded = 0;
4124 		}
4125 		vc->runner = vcpu;
4126 		if (n_ceded == vc->n_runnable) {
4127 			kvmppc_vcore_blocked(vc);
4128 		} else if (need_resched()) {
4129 			kvmppc_vcore_preempt(vc);
4130 			/* Let something else run */
4131 			cond_resched_lock(&vc->lock);
4132 			if (vc->vcore_state == VCORE_PREEMPT)
4133 				kvmppc_vcore_end_preempt(vc);
4134 		} else {
4135 			kvmppc_run_core(vc);
4136 		}
4137 		vc->runner = NULL;
4138 	}
4139 
4140 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4141 	       (vc->vcore_state == VCORE_RUNNING ||
4142 		vc->vcore_state == VCORE_EXITING ||
4143 		vc->vcore_state == VCORE_PIGGYBACK))
4144 		kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
4145 
4146 	if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4147 		kvmppc_vcore_end_preempt(vc);
4148 
4149 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4150 		kvmppc_remove_runnable(vc, vcpu);
4151 		vcpu->stat.signal_exits++;
4152 		run->exit_reason = KVM_EXIT_INTR;
4153 		vcpu->arch.ret = -EINTR;
4154 	}
4155 
4156 	if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
4157 		/* Wake up some vcpu to run the core */
4158 		i = -1;
4159 		v = next_runnable_thread(vc, &i);
4160 		wake_up(&v->arch.cpu_run);
4161 	}
4162 
4163 	trace_kvmppc_run_vcpu_exit(vcpu);
4164 	spin_unlock(&vc->lock);
4165 	return vcpu->arch.ret;
4166 }
4167 
4168 int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
4169 			  unsigned long lpcr)
4170 {
4171 	struct kvm_run *run = vcpu->run;
4172 	int trap, r, pcpu;
4173 	int srcu_idx, lpid;
4174 	struct kvmppc_vcore *vc;
4175 	struct kvm *kvm = vcpu->kvm;
4176 	struct kvm_nested_guest *nested = vcpu->arch.nested;
4177 
4178 	trace_kvmppc_run_vcpu_enter(vcpu);
4179 
4180 	run->exit_reason = 0;
4181 	vcpu->arch.ret = RESUME_GUEST;
4182 	vcpu->arch.trap = 0;
4183 
4184 	vc = vcpu->arch.vcore;
4185 	vcpu->arch.ceded = 0;
4186 	vcpu->arch.run_task = current;
4187 	vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4188 	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4189 	vcpu->arch.busy_preempt = TB_NIL;
4190 	vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
4191 	vc->runnable_threads[0] = vcpu;
4192 	vc->n_runnable = 1;
4193 	vc->runner = vcpu;
4194 
4195 	/* See if the MMU is ready to go */
4196 	if (!kvm->arch.mmu_ready)
4197 		kvmhv_setup_mmu(vcpu);
4198 
4199 	if (need_resched())
4200 		cond_resched();
4201 
4202 	kvmppc_update_vpas(vcpu);
4203 
4204 	init_vcore_to_run(vc);
4205 	vc->preempt_tb = TB_NIL;
4206 
4207 	preempt_disable();
4208 	pcpu = smp_processor_id();
4209 	vc->pcpu = pcpu;
4210 	kvmppc_prepare_radix_vcpu(vcpu, pcpu);
4211 
4212 	local_irq_disable();
4213 	hard_irq_disable();
4214 	if (signal_pending(current))
4215 		goto sigpend;
4216 	if (lazy_irq_pending() || need_resched() || !kvm->arch.mmu_ready)
4217 		goto out;
4218 
4219 	if (!nested) {
4220 		kvmppc_core_prepare_to_enter(vcpu);
4221 		if (vcpu->arch.doorbell_request) {
4222 			vc->dpdes = 1;
4223 			smp_wmb();
4224 			vcpu->arch.doorbell_request = 0;
4225 		}
4226 		if (test_bit(BOOK3S_IRQPRIO_EXTERNAL,
4227 			     &vcpu->arch.pending_exceptions))
4228 			lpcr |= LPCR_MER;
4229 	} else if (vcpu->arch.pending_exceptions ||
4230 		   vcpu->arch.doorbell_request ||
4231 		   xive_interrupt_pending(vcpu)) {
4232 		vcpu->arch.ret = RESUME_HOST;
4233 		goto out;
4234 	}
4235 
4236 	kvmppc_clear_host_core(pcpu);
4237 
4238 	local_paca->kvm_hstate.napping = 0;
4239 	local_paca->kvm_hstate.kvm_split_mode = NULL;
4240 	kvmppc_start_thread(vcpu, vc);
4241 	kvmppc_create_dtl_entry(vcpu, vc);
4242 	trace_kvm_guest_enter(vcpu);
4243 
4244 	vc->vcore_state = VCORE_RUNNING;
4245 	trace_kvmppc_run_core(vc, 0);
4246 
4247 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
4248 		lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
4249 		mtspr(SPRN_LPID, lpid);
4250 		isync();
4251 		kvmppc_check_need_tlb_flush(kvm, pcpu, nested);
4252 	}
4253 
4254 	guest_enter_irqoff();
4255 
4256 	srcu_idx = srcu_read_lock(&kvm->srcu);
4257 
4258 	this_cpu_disable_ftrace();
4259 
4260 	/* Tell lockdep that we're about to enable interrupts */
4261 	trace_hardirqs_on();
4262 
4263 	trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr);
4264 	vcpu->arch.trap = trap;
4265 
4266 	trace_hardirqs_off();
4267 
4268 	this_cpu_enable_ftrace();
4269 
4270 	srcu_read_unlock(&kvm->srcu, srcu_idx);
4271 
4272 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
4273 		mtspr(SPRN_LPID, kvm->arch.host_lpid);
4274 		isync();
4275 	}
4276 
4277 	set_irq_happened(trap);
4278 
4279 	kvmppc_set_host_core(pcpu);
4280 
4281 	guest_exit_irqoff();
4282 
4283 	local_irq_enable();
4284 
4285 	cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest);
4286 
4287 	preempt_enable();
4288 
4289 	/*
4290 	 * cancel pending decrementer exception if DEC is now positive, or if
4291 	 * entering a nested guest in which case the decrementer is now owned
4292 	 * by L2 and the L1 decrementer is provided in hdec_expires
4293 	 */
4294 	if (kvmppc_core_pending_dec(vcpu) &&
4295 			((get_tb() < vcpu->arch.dec_expires) ||
4296 			 (trap == BOOK3S_INTERRUPT_SYSCALL &&
4297 			  kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED)))
4298 		kvmppc_core_dequeue_dec(vcpu);
4299 
4300 	trace_kvm_guest_exit(vcpu);
4301 	r = RESUME_GUEST;
4302 	if (trap) {
4303 		if (!nested)
4304 			r = kvmppc_handle_exit_hv(vcpu, current);
4305 		else
4306 			r = kvmppc_handle_nested_exit(vcpu);
4307 	}
4308 	vcpu->arch.ret = r;
4309 
4310 	if (is_kvmppc_resume_guest(r) && vcpu->arch.ceded &&
4311 	    !kvmppc_vcpu_woken(vcpu)) {
4312 		kvmppc_set_timer(vcpu);
4313 		while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) {
4314 			if (signal_pending(current)) {
4315 				vcpu->stat.signal_exits++;
4316 				run->exit_reason = KVM_EXIT_INTR;
4317 				vcpu->arch.ret = -EINTR;
4318 				break;
4319 			}
4320 			spin_lock(&vc->lock);
4321 			kvmppc_vcore_blocked(vc);
4322 			spin_unlock(&vc->lock);
4323 		}
4324 	}
4325 	vcpu->arch.ceded = 0;
4326 
4327 	vc->vcore_state = VCORE_INACTIVE;
4328 	trace_kvmppc_run_core(vc, 1);
4329 
4330  done:
4331 	kvmppc_remove_runnable(vc, vcpu);
4332 	trace_kvmppc_run_vcpu_exit(vcpu);
4333 
4334 	return vcpu->arch.ret;
4335 
4336  sigpend:
4337 	vcpu->stat.signal_exits++;
4338 	run->exit_reason = KVM_EXIT_INTR;
4339 	vcpu->arch.ret = -EINTR;
4340  out:
4341 	local_irq_enable();
4342 	preempt_enable();
4343 	goto done;
4344 }
4345 
4346 static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu)
4347 {
4348 	struct kvm_run *run = vcpu->run;
4349 	int r;
4350 	int srcu_idx;
4351 	unsigned long ebb_regs[3] = {};	/* shut up GCC */
4352 	unsigned long user_tar = 0;
4353 	unsigned int user_vrsave;
4354 	struct kvm *kvm;
4355 
4356 	if (!vcpu->arch.sane) {
4357 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4358 		return -EINVAL;
4359 	}
4360 
4361 	/*
4362 	 * Don't allow entry with a suspended transaction, because
4363 	 * the guest entry/exit code will lose it.
4364 	 * If the guest has TM enabled, save away their TM-related SPRs
4365 	 * (they will get restored by the TM unavailable interrupt).
4366 	 */
4367 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
4368 	if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
4369 	    (current->thread.regs->msr & MSR_TM)) {
4370 		if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
4371 			run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4372 			run->fail_entry.hardware_entry_failure_reason = 0;
4373 			return -EINVAL;
4374 		}
4375 		/* Enable TM so we can read the TM SPRs */
4376 		mtmsr(mfmsr() | MSR_TM);
4377 		current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
4378 		current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
4379 		current->thread.tm_texasr = mfspr(SPRN_TEXASR);
4380 		current->thread.regs->msr &= ~MSR_TM;
4381 	}
4382 #endif
4383 
4384 	/*
4385 	 * Force online to 1 for the sake of old userspace which doesn't
4386 	 * set it.
4387 	 */
4388 	if (!vcpu->arch.online) {
4389 		atomic_inc(&vcpu->arch.vcore->online_count);
4390 		vcpu->arch.online = 1;
4391 	}
4392 
4393 	kvmppc_core_prepare_to_enter(vcpu);
4394 
4395 	/* No need to go into the guest when all we'll do is come back out */
4396 	if (signal_pending(current)) {
4397 		run->exit_reason = KVM_EXIT_INTR;
4398 		return -EINTR;
4399 	}
4400 
4401 	kvm = vcpu->kvm;
4402 	atomic_inc(&kvm->arch.vcpus_running);
4403 	/* Order vcpus_running vs. mmu_ready, see kvmppc_alloc_reset_hpt */
4404 	smp_mb();
4405 
4406 	flush_all_to_thread(current);
4407 
4408 	/* Save userspace EBB and other register values */
4409 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4410 		ebb_regs[0] = mfspr(SPRN_EBBHR);
4411 		ebb_regs[1] = mfspr(SPRN_EBBRR);
4412 		ebb_regs[2] = mfspr(SPRN_BESCR);
4413 		user_tar = mfspr(SPRN_TAR);
4414 	}
4415 	user_vrsave = mfspr(SPRN_VRSAVE);
4416 
4417 	vcpu->arch.waitp = &vcpu->arch.vcore->wait;
4418 	vcpu->arch.pgdir = kvm->mm->pgd;
4419 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4420 
4421 	do {
4422 		/*
4423 		 * The TLB prefetch bug fixup is only in the kvmppc_run_vcpu
4424 		 * path, which also handles hash and dependent threads mode.
4425 		 */
4426 		if (kvm->arch.threads_indep && kvm_is_radix(kvm) &&
4427 		    !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
4428 			r = kvmhv_run_single_vcpu(vcpu, ~(u64)0,
4429 						  vcpu->arch.vcore->lpcr);
4430 		else
4431 			r = kvmppc_run_vcpu(vcpu);
4432 
4433 		if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
4434 		    !(vcpu->arch.shregs.msr & MSR_PR)) {
4435 			trace_kvm_hcall_enter(vcpu);
4436 			r = kvmppc_pseries_do_hcall(vcpu);
4437 			trace_kvm_hcall_exit(vcpu, r);
4438 			kvmppc_core_prepare_to_enter(vcpu);
4439 		} else if (r == RESUME_PAGE_FAULT) {
4440 			srcu_idx = srcu_read_lock(&kvm->srcu);
4441 			r = kvmppc_book3s_hv_page_fault(vcpu,
4442 				vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
4443 			srcu_read_unlock(&kvm->srcu, srcu_idx);
4444 		} else if (r == RESUME_PASSTHROUGH) {
4445 			if (WARN_ON(xics_on_xive()))
4446 				r = H_SUCCESS;
4447 			else
4448 				r = kvmppc_xics_rm_complete(vcpu, 0);
4449 		}
4450 	} while (is_kvmppc_resume_guest(r));
4451 
4452 	/* Restore userspace EBB and other register values */
4453 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4454 		mtspr(SPRN_EBBHR, ebb_regs[0]);
4455 		mtspr(SPRN_EBBRR, ebb_regs[1]);
4456 		mtspr(SPRN_BESCR, ebb_regs[2]);
4457 		mtspr(SPRN_TAR, user_tar);
4458 		mtspr(SPRN_FSCR, current->thread.fscr);
4459 	}
4460 	mtspr(SPRN_VRSAVE, user_vrsave);
4461 
4462 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
4463 	atomic_dec(&kvm->arch.vcpus_running);
4464 	return r;
4465 }
4466 
4467 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
4468 				     int shift, int sllp)
4469 {
4470 	(*sps)->page_shift = shift;
4471 	(*sps)->slb_enc = sllp;
4472 	(*sps)->enc[0].page_shift = shift;
4473 	(*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
4474 	/*
4475 	 * Add 16MB MPSS support (may get filtered out by userspace)
4476 	 */
4477 	if (shift != 24) {
4478 		int penc = kvmppc_pgsize_lp_encoding(shift, 24);
4479 		if (penc != -1) {
4480 			(*sps)->enc[1].page_shift = 24;
4481 			(*sps)->enc[1].pte_enc = penc;
4482 		}
4483 	}
4484 	(*sps)++;
4485 }
4486 
4487 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
4488 					 struct kvm_ppc_smmu_info *info)
4489 {
4490 	struct kvm_ppc_one_seg_page_size *sps;
4491 
4492 	/*
4493 	 * POWER7, POWER8 and POWER9 all support 32 storage keys for data.
4494 	 * POWER7 doesn't support keys for instruction accesses,
4495 	 * POWER8 and POWER9 do.
4496 	 */
4497 	info->data_keys = 32;
4498 	info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
4499 
4500 	/* POWER7, 8 and 9 all have 1T segments and 32-entry SLB */
4501 	info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
4502 	info->slb_size = 32;
4503 
4504 	/* We only support these sizes for now, and no muti-size segments */
4505 	sps = &info->sps[0];
4506 	kvmppc_add_seg_page_size(&sps, 12, 0);
4507 	kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
4508 	kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
4509 
4510 	/* If running as a nested hypervisor, we don't support HPT guests */
4511 	if (kvmhv_on_pseries())
4512 		info->flags |= KVM_PPC_NO_HASH;
4513 
4514 	return 0;
4515 }
4516 
4517 /*
4518  * Get (and clear) the dirty memory log for a memory slot.
4519  */
4520 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
4521 					 struct kvm_dirty_log *log)
4522 {
4523 	struct kvm_memslots *slots;
4524 	struct kvm_memory_slot *memslot;
4525 	int i, r;
4526 	unsigned long n;
4527 	unsigned long *buf, *p;
4528 	struct kvm_vcpu *vcpu;
4529 
4530 	mutex_lock(&kvm->slots_lock);
4531 
4532 	r = -EINVAL;
4533 	if (log->slot >= KVM_USER_MEM_SLOTS)
4534 		goto out;
4535 
4536 	slots = kvm_memslots(kvm);
4537 	memslot = id_to_memslot(slots, log->slot);
4538 	r = -ENOENT;
4539 	if (!memslot || !memslot->dirty_bitmap)
4540 		goto out;
4541 
4542 	/*
4543 	 * Use second half of bitmap area because both HPT and radix
4544 	 * accumulate bits in the first half.
4545 	 */
4546 	n = kvm_dirty_bitmap_bytes(memslot);
4547 	buf = memslot->dirty_bitmap + n / sizeof(long);
4548 	memset(buf, 0, n);
4549 
4550 	if (kvm_is_radix(kvm))
4551 		r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
4552 	else
4553 		r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
4554 	if (r)
4555 		goto out;
4556 
4557 	/*
4558 	 * We accumulate dirty bits in the first half of the
4559 	 * memslot's dirty_bitmap area, for when pages are paged
4560 	 * out or modified by the host directly.  Pick up these
4561 	 * bits and add them to the map.
4562 	 */
4563 	p = memslot->dirty_bitmap;
4564 	for (i = 0; i < n / sizeof(long); ++i)
4565 		buf[i] |= xchg(&p[i], 0);
4566 
4567 	/* Harvest dirty bits from VPA and DTL updates */
4568 	/* Note: we never modify the SLB shadow buffer areas */
4569 	kvm_for_each_vcpu(i, vcpu, kvm) {
4570 		spin_lock(&vcpu->arch.vpa_update_lock);
4571 		kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
4572 		kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
4573 		spin_unlock(&vcpu->arch.vpa_update_lock);
4574 	}
4575 
4576 	r = -EFAULT;
4577 	if (copy_to_user(log->dirty_bitmap, buf, n))
4578 		goto out;
4579 
4580 	r = 0;
4581 out:
4582 	mutex_unlock(&kvm->slots_lock);
4583 	return r;
4584 }
4585 
4586 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *slot)
4587 {
4588 	vfree(slot->arch.rmap);
4589 	slot->arch.rmap = NULL;
4590 }
4591 
4592 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
4593 					struct kvm_memory_slot *slot,
4594 					const struct kvm_userspace_memory_region *mem,
4595 					enum kvm_mr_change change)
4596 {
4597 	unsigned long npages = mem->memory_size >> PAGE_SHIFT;
4598 
4599 	if (change == KVM_MR_CREATE) {
4600 		slot->arch.rmap = vzalloc(array_size(npages,
4601 					  sizeof(*slot->arch.rmap)));
4602 		if (!slot->arch.rmap)
4603 			return -ENOMEM;
4604 	}
4605 
4606 	return 0;
4607 }
4608 
4609 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
4610 				const struct kvm_userspace_memory_region *mem,
4611 				const struct kvm_memory_slot *old,
4612 				const struct kvm_memory_slot *new,
4613 				enum kvm_mr_change change)
4614 {
4615 	unsigned long npages = mem->memory_size >> PAGE_SHIFT;
4616 
4617 	/*
4618 	 * If we are making a new memslot, it might make
4619 	 * some address that was previously cached as emulated
4620 	 * MMIO be no longer emulated MMIO, so invalidate
4621 	 * all the caches of emulated MMIO translations.
4622 	 */
4623 	if (npages)
4624 		atomic64_inc(&kvm->arch.mmio_update);
4625 
4626 	/*
4627 	 * For change == KVM_MR_MOVE or KVM_MR_DELETE, higher levels
4628 	 * have already called kvm_arch_flush_shadow_memslot() to
4629 	 * flush shadow mappings.  For KVM_MR_CREATE we have no
4630 	 * previous mappings.  So the only case to handle is
4631 	 * KVM_MR_FLAGS_ONLY when the KVM_MEM_LOG_DIRTY_PAGES bit
4632 	 * has been changed.
4633 	 * For radix guests, we flush on setting KVM_MEM_LOG_DIRTY_PAGES
4634 	 * to get rid of any THP PTEs in the partition-scoped page tables
4635 	 * so we can track dirtiness at the page level; we flush when
4636 	 * clearing KVM_MEM_LOG_DIRTY_PAGES so that we can go back to
4637 	 * using THP PTEs.
4638 	 */
4639 	if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
4640 	    ((new->flags ^ old->flags) & KVM_MEM_LOG_DIRTY_PAGES))
4641 		kvmppc_radix_flush_memslot(kvm, old);
4642 	/*
4643 	 * If UV hasn't yet called H_SVM_INIT_START, don't register memslots.
4644 	 */
4645 	if (!kvm->arch.secure_guest)
4646 		return;
4647 
4648 	switch (change) {
4649 	case KVM_MR_CREATE:
4650 		/*
4651 		 * @TODO kvmppc_uvmem_memslot_create() can fail and
4652 		 * return error. Fix this.
4653 		 */
4654 		kvmppc_uvmem_memslot_create(kvm, new);
4655 		break;
4656 	case KVM_MR_DELETE:
4657 		kvmppc_uvmem_memslot_delete(kvm, old);
4658 		break;
4659 	default:
4660 		/* TODO: Handle KVM_MR_MOVE */
4661 		break;
4662 	}
4663 }
4664 
4665 /*
4666  * Update LPCR values in kvm->arch and in vcores.
4667  * Caller must hold kvm->arch.mmu_setup_lock (for mutual exclusion
4668  * of kvm->arch.lpcr update).
4669  */
4670 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
4671 {
4672 	long int i;
4673 	u32 cores_done = 0;
4674 
4675 	if ((kvm->arch.lpcr & mask) == lpcr)
4676 		return;
4677 
4678 	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
4679 
4680 	for (i = 0; i < KVM_MAX_VCORES; ++i) {
4681 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];
4682 		if (!vc)
4683 			continue;
4684 
4685 		spin_lock(&vc->lock);
4686 		vc->lpcr = (vc->lpcr & ~mask) | lpcr;
4687 		verify_lpcr(kvm, vc->lpcr);
4688 		spin_unlock(&vc->lock);
4689 		if (++cores_done >= kvm->arch.online_vcores)
4690 			break;
4691 	}
4692 }
4693 
4694 void kvmppc_setup_partition_table(struct kvm *kvm)
4695 {
4696 	unsigned long dw0, dw1;
4697 
4698 	if (!kvm_is_radix(kvm)) {
4699 		/* PS field - page size for VRMA */
4700 		dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
4701 			((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
4702 		/* HTABSIZE and HTABORG fields */
4703 		dw0 |= kvm->arch.sdr1;
4704 
4705 		/* Second dword as set by userspace */
4706 		dw1 = kvm->arch.process_table;
4707 	} else {
4708 		dw0 = PATB_HR | radix__get_tree_size() |
4709 			__pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
4710 		dw1 = PATB_GR | kvm->arch.process_table;
4711 	}
4712 	kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
4713 }
4714 
4715 /*
4716  * Set up HPT (hashed page table) and RMA (real-mode area).
4717  * Must be called with kvm->arch.mmu_setup_lock held.
4718  */
4719 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
4720 {
4721 	int err = 0;
4722 	struct kvm *kvm = vcpu->kvm;
4723 	unsigned long hva;
4724 	struct kvm_memory_slot *memslot;
4725 	struct vm_area_struct *vma;
4726 	unsigned long lpcr = 0, senc;
4727 	unsigned long psize, porder;
4728 	int srcu_idx;
4729 
4730 	/* Allocate hashed page table (if not done already) and reset it */
4731 	if (!kvm->arch.hpt.virt) {
4732 		int order = KVM_DEFAULT_HPT_ORDER;
4733 		struct kvm_hpt_info info;
4734 
4735 		err = kvmppc_allocate_hpt(&info, order);
4736 		/* If we get here, it means userspace didn't specify a
4737 		 * size explicitly.  So, try successively smaller
4738 		 * sizes if the default failed. */
4739 		while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
4740 			err  = kvmppc_allocate_hpt(&info, order);
4741 
4742 		if (err < 0) {
4743 			pr_err("KVM: Couldn't alloc HPT\n");
4744 			goto out;
4745 		}
4746 
4747 		kvmppc_set_hpt(kvm, &info);
4748 	}
4749 
4750 	/* Look up the memslot for guest physical address 0 */
4751 	srcu_idx = srcu_read_lock(&kvm->srcu);
4752 	memslot = gfn_to_memslot(kvm, 0);
4753 
4754 	/* We must have some memory at 0 by now */
4755 	err = -EINVAL;
4756 	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
4757 		goto out_srcu;
4758 
4759 	/* Look up the VMA for the start of this memory slot */
4760 	hva = memslot->userspace_addr;
4761 	mmap_read_lock(kvm->mm);
4762 	vma = find_vma(kvm->mm, hva);
4763 	if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
4764 		goto up_out;
4765 
4766 	psize = vma_kernel_pagesize(vma);
4767 
4768 	mmap_read_unlock(kvm->mm);
4769 
4770 	/* We can handle 4k, 64k or 16M pages in the VRMA */
4771 	if (psize >= 0x1000000)
4772 		psize = 0x1000000;
4773 	else if (psize >= 0x10000)
4774 		psize = 0x10000;
4775 	else
4776 		psize = 0x1000;
4777 	porder = __ilog2(psize);
4778 
4779 	senc = slb_pgsize_encoding(psize);
4780 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
4781 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
4782 	/* Create HPTEs in the hash page table for the VRMA */
4783 	kvmppc_map_vrma(vcpu, memslot, porder);
4784 
4785 	/* Update VRMASD field in the LPCR */
4786 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
4787 		/* the -4 is to account for senc values starting at 0x10 */
4788 		lpcr = senc << (LPCR_VRMASD_SH - 4);
4789 		kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
4790 	}
4791 
4792 	/* Order updates to kvm->arch.lpcr etc. vs. mmu_ready */
4793 	smp_wmb();
4794 	err = 0;
4795  out_srcu:
4796 	srcu_read_unlock(&kvm->srcu, srcu_idx);
4797  out:
4798 	return err;
4799 
4800  up_out:
4801 	mmap_read_unlock(kvm->mm);
4802 	goto out_srcu;
4803 }
4804 
4805 /*
4806  * Must be called with kvm->arch.mmu_setup_lock held and
4807  * mmu_ready = 0 and no vcpus running.
4808  */
4809 int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
4810 {
4811 	if (nesting_enabled(kvm))
4812 		kvmhv_release_all_nested(kvm);
4813 	kvmppc_rmap_reset(kvm);
4814 	kvm->arch.process_table = 0;
4815 	/* Mutual exclusion with kvm_unmap_gfn_range etc. */
4816 	spin_lock(&kvm->mmu_lock);
4817 	kvm->arch.radix = 0;
4818 	spin_unlock(&kvm->mmu_lock);
4819 	kvmppc_free_radix(kvm);
4820 	kvmppc_update_lpcr(kvm, LPCR_VPM1,
4821 			   LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
4822 	return 0;
4823 }
4824 
4825 /*
4826  * Must be called with kvm->arch.mmu_setup_lock held and
4827  * mmu_ready = 0 and no vcpus running.
4828  */
4829 int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
4830 {
4831 	int err;
4832 
4833 	err = kvmppc_init_vm_radix(kvm);
4834 	if (err)
4835 		return err;
4836 	kvmppc_rmap_reset(kvm);
4837 	/* Mutual exclusion with kvm_unmap_gfn_range etc. */
4838 	spin_lock(&kvm->mmu_lock);
4839 	kvm->arch.radix = 1;
4840 	spin_unlock(&kvm->mmu_lock);
4841 	kvmppc_free_hpt(&kvm->arch.hpt);
4842 	kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR,
4843 			   LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
4844 	return 0;
4845 }
4846 
4847 #ifdef CONFIG_KVM_XICS
4848 /*
4849  * Allocate a per-core structure for managing state about which cores are
4850  * running in the host versus the guest and for exchanging data between
4851  * real mode KVM and CPU running in the host.
4852  * This is only done for the first VM.
4853  * The allocated structure stays even if all VMs have stopped.
4854  * It is only freed when the kvm-hv module is unloaded.
4855  * It's OK for this routine to fail, we just don't support host
4856  * core operations like redirecting H_IPI wakeups.
4857  */
4858 void kvmppc_alloc_host_rm_ops(void)
4859 {
4860 	struct kvmppc_host_rm_ops *ops;
4861 	unsigned long l_ops;
4862 	int cpu, core;
4863 	int size;
4864 
4865 	/* Not the first time here ? */
4866 	if (kvmppc_host_rm_ops_hv != NULL)
4867 		return;
4868 
4869 	ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
4870 	if (!ops)
4871 		return;
4872 
4873 	size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
4874 	ops->rm_core = kzalloc(size, GFP_KERNEL);
4875 
4876 	if (!ops->rm_core) {
4877 		kfree(ops);
4878 		return;
4879 	}
4880 
4881 	cpus_read_lock();
4882 
4883 	for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
4884 		if (!cpu_online(cpu))
4885 			continue;
4886 
4887 		core = cpu >> threads_shift;
4888 		ops->rm_core[core].rm_state.in_host = 1;
4889 	}
4890 
4891 	ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
4892 
4893 	/*
4894 	 * Make the contents of the kvmppc_host_rm_ops structure visible
4895 	 * to other CPUs before we assign it to the global variable.
4896 	 * Do an atomic assignment (no locks used here), but if someone
4897 	 * beats us to it, just free our copy and return.
4898 	 */
4899 	smp_wmb();
4900 	l_ops = (unsigned long) ops;
4901 
4902 	if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
4903 		cpus_read_unlock();
4904 		kfree(ops->rm_core);
4905 		kfree(ops);
4906 		return;
4907 	}
4908 
4909 	cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
4910 					     "ppc/kvm_book3s:prepare",
4911 					     kvmppc_set_host_core,
4912 					     kvmppc_clear_host_core);
4913 	cpus_read_unlock();
4914 }
4915 
4916 void kvmppc_free_host_rm_ops(void)
4917 {
4918 	if (kvmppc_host_rm_ops_hv) {
4919 		cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
4920 		kfree(kvmppc_host_rm_ops_hv->rm_core);
4921 		kfree(kvmppc_host_rm_ops_hv);
4922 		kvmppc_host_rm_ops_hv = NULL;
4923 	}
4924 }
4925 #endif
4926 
4927 static int kvmppc_core_init_vm_hv(struct kvm *kvm)
4928 {
4929 	unsigned long lpcr, lpid;
4930 	char buf[32];
4931 	int ret;
4932 
4933 	mutex_init(&kvm->arch.uvmem_lock);
4934 	INIT_LIST_HEAD(&kvm->arch.uvmem_pfns);
4935 	mutex_init(&kvm->arch.mmu_setup_lock);
4936 
4937 	/* Allocate the guest's logical partition ID */
4938 
4939 	lpid = kvmppc_alloc_lpid();
4940 	if ((long)lpid < 0)
4941 		return -ENOMEM;
4942 	kvm->arch.lpid = lpid;
4943 
4944 	kvmppc_alloc_host_rm_ops();
4945 
4946 	kvmhv_vm_nested_init(kvm);
4947 
4948 	/*
4949 	 * Since we don't flush the TLB when tearing down a VM,
4950 	 * and this lpid might have previously been used,
4951 	 * make sure we flush on each core before running the new VM.
4952 	 * On POWER9, the tlbie in mmu_partition_table_set_entry()
4953 	 * does this flush for us.
4954 	 */
4955 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
4956 		cpumask_setall(&kvm->arch.need_tlb_flush);
4957 
4958 	/* Start out with the default set of hcalls enabled */
4959 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
4960 	       sizeof(kvm->arch.enabled_hcalls));
4961 
4962 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
4963 		kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
4964 
4965 	/* Init LPCR for virtual RMA mode */
4966 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
4967 		kvm->arch.host_lpid = mfspr(SPRN_LPID);
4968 		kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
4969 		lpcr &= LPCR_PECE | LPCR_LPES;
4970 	} else {
4971 		lpcr = 0;
4972 	}
4973 	lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
4974 		LPCR_VPM0 | LPCR_VPM1;
4975 	kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
4976 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
4977 	/* On POWER8 turn on online bit to enable PURR/SPURR */
4978 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
4979 		lpcr |= LPCR_ONL;
4980 	/*
4981 	 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed)
4982 	 * Set HVICE bit to enable hypervisor virtualization interrupts.
4983 	 * Set HEIC to prevent OS interrupts to go to hypervisor (should
4984 	 * be unnecessary but better safe than sorry in case we re-enable
4985 	 * EE in HV mode with this LPCR still set)
4986 	 */
4987 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4988 		lpcr &= ~LPCR_VPM0;
4989 		lpcr |= LPCR_HVICE | LPCR_HEIC;
4990 
4991 		/*
4992 		 * If xive is enabled, we route 0x500 interrupts directly
4993 		 * to the guest.
4994 		 */
4995 		if (xics_on_xive())
4996 			lpcr |= LPCR_LPES;
4997 	}
4998 
4999 	/*
5000 	 * If the host uses radix, the guest starts out as radix.
5001 	 */
5002 	if (radix_enabled()) {
5003 		kvm->arch.radix = 1;
5004 		kvm->arch.mmu_ready = 1;
5005 		lpcr &= ~LPCR_VPM1;
5006 		lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
5007 		ret = kvmppc_init_vm_radix(kvm);
5008 		if (ret) {
5009 			kvmppc_free_lpid(kvm->arch.lpid);
5010 			return ret;
5011 		}
5012 		kvmppc_setup_partition_table(kvm);
5013 	}
5014 
5015 	verify_lpcr(kvm, lpcr);
5016 	kvm->arch.lpcr = lpcr;
5017 
5018 	/* Initialization for future HPT resizes */
5019 	kvm->arch.resize_hpt = NULL;
5020 
5021 	/*
5022 	 * Work out how many sets the TLB has, for the use of
5023 	 * the TLB invalidation loop in book3s_hv_rmhandlers.S.
5024 	 */
5025 	if (cpu_has_feature(CPU_FTR_ARCH_31)) {
5026 		/*
5027 		 * P10 will flush all the congruence class with a single tlbiel
5028 		 */
5029 		kvm->arch.tlb_sets = 1;
5030 	} else if (radix_enabled())
5031 		kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;	/* 128 */
5032 	else if (cpu_has_feature(CPU_FTR_ARCH_300))
5033 		kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;	/* 256 */
5034 	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
5035 		kvm->arch.tlb_sets = POWER8_TLB_SETS;		/* 512 */
5036 	else
5037 		kvm->arch.tlb_sets = POWER7_TLB_SETS;		/* 128 */
5038 
5039 	/*
5040 	 * Track that we now have a HV mode VM active. This blocks secondary
5041 	 * CPU threads from coming online.
5042 	 * On POWER9, we only need to do this if the "indep_threads_mode"
5043 	 * module parameter has been set to N.
5044 	 */
5045 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5046 		if (!indep_threads_mode && !cpu_has_feature(CPU_FTR_HVMODE)) {
5047 			pr_warn("KVM: Ignoring indep_threads_mode=N in nested hypervisor\n");
5048 			kvm->arch.threads_indep = true;
5049 		} else {
5050 			kvm->arch.threads_indep = indep_threads_mode;
5051 		}
5052 	}
5053 	if (!kvm->arch.threads_indep)
5054 		kvm_hv_vm_activated();
5055 
5056 	/*
5057 	 * Initialize smt_mode depending on processor.
5058 	 * POWER8 and earlier have to use "strict" threading, where
5059 	 * all vCPUs in a vcore have to run on the same (sub)core,
5060 	 * whereas on POWER9 the threads can each run a different
5061 	 * guest.
5062 	 */
5063 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5064 		kvm->arch.smt_mode = threads_per_subcore;
5065 	else
5066 		kvm->arch.smt_mode = 1;
5067 	kvm->arch.emul_smt_mode = 1;
5068 
5069 	/*
5070 	 * Create a debugfs directory for the VM
5071 	 */
5072 	snprintf(buf, sizeof(buf), "vm%d", current->pid);
5073 	kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
5074 	kvmppc_mmu_debugfs_init(kvm);
5075 	if (radix_enabled())
5076 		kvmhv_radix_debugfs_init(kvm);
5077 
5078 	return 0;
5079 }
5080 
5081 static void kvmppc_free_vcores(struct kvm *kvm)
5082 {
5083 	long int i;
5084 
5085 	for (i = 0; i < KVM_MAX_VCORES; ++i)
5086 		kfree(kvm->arch.vcores[i]);
5087 	kvm->arch.online_vcores = 0;
5088 }
5089 
5090 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
5091 {
5092 	debugfs_remove_recursive(kvm->arch.debugfs_dir);
5093 
5094 	if (!kvm->arch.threads_indep)
5095 		kvm_hv_vm_deactivated();
5096 
5097 	kvmppc_free_vcores(kvm);
5098 
5099 
5100 	if (kvm_is_radix(kvm))
5101 		kvmppc_free_radix(kvm);
5102 	else
5103 		kvmppc_free_hpt(&kvm->arch.hpt);
5104 
5105 	/* Perform global invalidation and return lpid to the pool */
5106 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5107 		if (nesting_enabled(kvm))
5108 			kvmhv_release_all_nested(kvm);
5109 		kvm->arch.process_table = 0;
5110 		if (kvm->arch.secure_guest)
5111 			uv_svm_terminate(kvm->arch.lpid);
5112 		kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
5113 	}
5114 
5115 	kvmppc_free_lpid(kvm->arch.lpid);
5116 
5117 	kvmppc_free_pimap(kvm);
5118 }
5119 
5120 /* We don't need to emulate any privileged instructions or dcbz */
5121 static int kvmppc_core_emulate_op_hv(struct kvm_vcpu *vcpu,
5122 				     unsigned int inst, int *advance)
5123 {
5124 	return EMULATE_FAIL;
5125 }
5126 
5127 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
5128 					ulong spr_val)
5129 {
5130 	return EMULATE_FAIL;
5131 }
5132 
5133 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
5134 					ulong *spr_val)
5135 {
5136 	return EMULATE_FAIL;
5137 }
5138 
5139 static int kvmppc_core_check_processor_compat_hv(void)
5140 {
5141 	if (cpu_has_feature(CPU_FTR_HVMODE) &&
5142 	    cpu_has_feature(CPU_FTR_ARCH_206))
5143 		return 0;
5144 
5145 	/* POWER9 in radix mode is capable of being a nested hypervisor. */
5146 	if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
5147 		return 0;
5148 
5149 	return -EIO;
5150 }
5151 
5152 #ifdef CONFIG_KVM_XICS
5153 
5154 void kvmppc_free_pimap(struct kvm *kvm)
5155 {
5156 	kfree(kvm->arch.pimap);
5157 }
5158 
5159 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
5160 {
5161 	return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
5162 }
5163 
5164 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5165 {
5166 	struct irq_desc *desc;
5167 	struct kvmppc_irq_map *irq_map;
5168 	struct kvmppc_passthru_irqmap *pimap;
5169 	struct irq_chip *chip;
5170 	int i, rc = 0;
5171 
5172 	if (!kvm_irq_bypass)
5173 		return 1;
5174 
5175 	desc = irq_to_desc(host_irq);
5176 	if (!desc)
5177 		return -EIO;
5178 
5179 	mutex_lock(&kvm->lock);
5180 
5181 	pimap = kvm->arch.pimap;
5182 	if (pimap == NULL) {
5183 		/* First call, allocate structure to hold IRQ map */
5184 		pimap = kvmppc_alloc_pimap();
5185 		if (pimap == NULL) {
5186 			mutex_unlock(&kvm->lock);
5187 			return -ENOMEM;
5188 		}
5189 		kvm->arch.pimap = pimap;
5190 	}
5191 
5192 	/*
5193 	 * For now, we only support interrupts for which the EOI operation
5194 	 * is an OPAL call followed by a write to XIRR, since that's
5195 	 * what our real-mode EOI code does, or a XIVE interrupt
5196 	 */
5197 	chip = irq_data_get_irq_chip(&desc->irq_data);
5198 	if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) {
5199 		pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
5200 			host_irq, guest_gsi);
5201 		mutex_unlock(&kvm->lock);
5202 		return -ENOENT;
5203 	}
5204 
5205 	/*
5206 	 * See if we already have an entry for this guest IRQ number.
5207 	 * If it's mapped to a hardware IRQ number, that's an error,
5208 	 * otherwise re-use this entry.
5209 	 */
5210 	for (i = 0; i < pimap->n_mapped; i++) {
5211 		if (guest_gsi == pimap->mapped[i].v_hwirq) {
5212 			if (pimap->mapped[i].r_hwirq) {
5213 				mutex_unlock(&kvm->lock);
5214 				return -EINVAL;
5215 			}
5216 			break;
5217 		}
5218 	}
5219 
5220 	if (i == KVMPPC_PIRQ_MAPPED) {
5221 		mutex_unlock(&kvm->lock);
5222 		return -EAGAIN;		/* table is full */
5223 	}
5224 
5225 	irq_map = &pimap->mapped[i];
5226 
5227 	irq_map->v_hwirq = guest_gsi;
5228 	irq_map->desc = desc;
5229 
5230 	/*
5231 	 * Order the above two stores before the next to serialize with
5232 	 * the KVM real mode handler.
5233 	 */
5234 	smp_wmb();
5235 	irq_map->r_hwirq = desc->irq_data.hwirq;
5236 
5237 	if (i == pimap->n_mapped)
5238 		pimap->n_mapped++;
5239 
5240 	if (xics_on_xive())
5241 		rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc);
5242 	else
5243 		kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq);
5244 	if (rc)
5245 		irq_map->r_hwirq = 0;
5246 
5247 	mutex_unlock(&kvm->lock);
5248 
5249 	return 0;
5250 }
5251 
5252 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5253 {
5254 	struct irq_desc *desc;
5255 	struct kvmppc_passthru_irqmap *pimap;
5256 	int i, rc = 0;
5257 
5258 	if (!kvm_irq_bypass)
5259 		return 0;
5260 
5261 	desc = irq_to_desc(host_irq);
5262 	if (!desc)
5263 		return -EIO;
5264 
5265 	mutex_lock(&kvm->lock);
5266 	if (!kvm->arch.pimap)
5267 		goto unlock;
5268 
5269 	pimap = kvm->arch.pimap;
5270 
5271 	for (i = 0; i < pimap->n_mapped; i++) {
5272 		if (guest_gsi == pimap->mapped[i].v_hwirq)
5273 			break;
5274 	}
5275 
5276 	if (i == pimap->n_mapped) {
5277 		mutex_unlock(&kvm->lock);
5278 		return -ENODEV;
5279 	}
5280 
5281 	if (xics_on_xive())
5282 		rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc);
5283 	else
5284 		kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
5285 
5286 	/* invalidate the entry (what do do on error from the above ?) */
5287 	pimap->mapped[i].r_hwirq = 0;
5288 
5289 	/*
5290 	 * We don't free this structure even when the count goes to
5291 	 * zero. The structure is freed when we destroy the VM.
5292 	 */
5293  unlock:
5294 	mutex_unlock(&kvm->lock);
5295 	return rc;
5296 }
5297 
5298 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
5299 					     struct irq_bypass_producer *prod)
5300 {
5301 	int ret = 0;
5302 	struct kvm_kernel_irqfd *irqfd =
5303 		container_of(cons, struct kvm_kernel_irqfd, consumer);
5304 
5305 	irqfd->producer = prod;
5306 
5307 	ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5308 	if (ret)
5309 		pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
5310 			prod->irq, irqfd->gsi, ret);
5311 
5312 	return ret;
5313 }
5314 
5315 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
5316 					      struct irq_bypass_producer *prod)
5317 {
5318 	int ret;
5319 	struct kvm_kernel_irqfd *irqfd =
5320 		container_of(cons, struct kvm_kernel_irqfd, consumer);
5321 
5322 	irqfd->producer = NULL;
5323 
5324 	/*
5325 	 * When producer of consumer is unregistered, we change back to
5326 	 * default external interrupt handling mode - KVM real mode
5327 	 * will switch back to host.
5328 	 */
5329 	ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5330 	if (ret)
5331 		pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
5332 			prod->irq, irqfd->gsi, ret);
5333 }
5334 #endif
5335 
5336 static long kvm_arch_vm_ioctl_hv(struct file *filp,
5337 				 unsigned int ioctl, unsigned long arg)
5338 {
5339 	struct kvm *kvm __maybe_unused = filp->private_data;
5340 	void __user *argp = (void __user *)arg;
5341 	long r;
5342 
5343 	switch (ioctl) {
5344 
5345 	case KVM_PPC_ALLOCATE_HTAB: {
5346 		u32 htab_order;
5347 
5348 		/* If we're a nested hypervisor, we currently only support radix */
5349 		if (kvmhv_on_pseries()) {
5350 			r = -EOPNOTSUPP;
5351 			break;
5352 		}
5353 
5354 		r = -EFAULT;
5355 		if (get_user(htab_order, (u32 __user *)argp))
5356 			break;
5357 		r = kvmppc_alloc_reset_hpt(kvm, htab_order);
5358 		if (r)
5359 			break;
5360 		r = 0;
5361 		break;
5362 	}
5363 
5364 	case KVM_PPC_GET_HTAB_FD: {
5365 		struct kvm_get_htab_fd ghf;
5366 
5367 		r = -EFAULT;
5368 		if (copy_from_user(&ghf, argp, sizeof(ghf)))
5369 			break;
5370 		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
5371 		break;
5372 	}
5373 
5374 	case KVM_PPC_RESIZE_HPT_PREPARE: {
5375 		struct kvm_ppc_resize_hpt rhpt;
5376 
5377 		r = -EFAULT;
5378 		if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5379 			break;
5380 
5381 		r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
5382 		break;
5383 	}
5384 
5385 	case KVM_PPC_RESIZE_HPT_COMMIT: {
5386 		struct kvm_ppc_resize_hpt rhpt;
5387 
5388 		r = -EFAULT;
5389 		if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5390 			break;
5391 
5392 		r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
5393 		break;
5394 	}
5395 
5396 	default:
5397 		r = -ENOTTY;
5398 	}
5399 
5400 	return r;
5401 }
5402 
5403 /*
5404  * List of hcall numbers to enable by default.
5405  * For compatibility with old userspace, we enable by default
5406  * all hcalls that were implemented before the hcall-enabling
5407  * facility was added.  Note this list should not include H_RTAS.
5408  */
5409 static unsigned int default_hcall_list[] = {
5410 	H_REMOVE,
5411 	H_ENTER,
5412 	H_READ,
5413 	H_PROTECT,
5414 	H_BULK_REMOVE,
5415 #ifdef CONFIG_SPAPR_TCE_IOMMU
5416 	H_GET_TCE,
5417 	H_PUT_TCE,
5418 #endif
5419 	H_SET_DABR,
5420 	H_SET_XDABR,
5421 	H_CEDE,
5422 	H_PROD,
5423 	H_CONFER,
5424 	H_REGISTER_VPA,
5425 #ifdef CONFIG_KVM_XICS
5426 	H_EOI,
5427 	H_CPPR,
5428 	H_IPI,
5429 	H_IPOLL,
5430 	H_XIRR,
5431 	H_XIRR_X,
5432 #endif
5433 	0
5434 };
5435 
5436 static void init_default_hcalls(void)
5437 {
5438 	int i;
5439 	unsigned int hcall;
5440 
5441 	for (i = 0; default_hcall_list[i]; ++i) {
5442 		hcall = default_hcall_list[i];
5443 		WARN_ON(!kvmppc_hcall_impl_hv(hcall));
5444 		__set_bit(hcall / 4, default_enabled_hcalls);
5445 	}
5446 }
5447 
5448 static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
5449 {
5450 	unsigned long lpcr;
5451 	int radix;
5452 	int err;
5453 
5454 	/* If not on a POWER9, reject it */
5455 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5456 		return -ENODEV;
5457 
5458 	/* If any unknown flags set, reject it */
5459 	if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
5460 		return -EINVAL;
5461 
5462 	/* GR (guest radix) bit in process_table field must match */
5463 	radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
5464 	if (!!(cfg->process_table & PATB_GR) != radix)
5465 		return -EINVAL;
5466 
5467 	/* Process table size field must be reasonable, i.e. <= 24 */
5468 	if ((cfg->process_table & PRTS_MASK) > 24)
5469 		return -EINVAL;
5470 
5471 	/* We can change a guest to/from radix now, if the host is radix */
5472 	if (radix && !radix_enabled())
5473 		return -EINVAL;
5474 
5475 	/* If we're a nested hypervisor, we currently only support radix */
5476 	if (kvmhv_on_pseries() && !radix)
5477 		return -EINVAL;
5478 
5479 	mutex_lock(&kvm->arch.mmu_setup_lock);
5480 	if (radix != kvm_is_radix(kvm)) {
5481 		if (kvm->arch.mmu_ready) {
5482 			kvm->arch.mmu_ready = 0;
5483 			/* order mmu_ready vs. vcpus_running */
5484 			smp_mb();
5485 			if (atomic_read(&kvm->arch.vcpus_running)) {
5486 				kvm->arch.mmu_ready = 1;
5487 				err = -EBUSY;
5488 				goto out_unlock;
5489 			}
5490 		}
5491 		if (radix)
5492 			err = kvmppc_switch_mmu_to_radix(kvm);
5493 		else
5494 			err = kvmppc_switch_mmu_to_hpt(kvm);
5495 		if (err)
5496 			goto out_unlock;
5497 	}
5498 
5499 	kvm->arch.process_table = cfg->process_table;
5500 	kvmppc_setup_partition_table(kvm);
5501 
5502 	lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
5503 	kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
5504 	err = 0;
5505 
5506  out_unlock:
5507 	mutex_unlock(&kvm->arch.mmu_setup_lock);
5508 	return err;
5509 }
5510 
5511 static int kvmhv_enable_nested(struct kvm *kvm)
5512 {
5513 	if (!nested)
5514 		return -EPERM;
5515 	if (!cpu_has_feature(CPU_FTR_ARCH_300) || no_mixing_hpt_and_radix)
5516 		return -ENODEV;
5517 
5518 	/* kvm == NULL means the caller is testing if the capability exists */
5519 	if (kvm)
5520 		kvm->arch.nested_enable = true;
5521 	return 0;
5522 }
5523 
5524 static int kvmhv_load_from_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5525 				 int size)
5526 {
5527 	int rc = -EINVAL;
5528 
5529 	if (kvmhv_vcpu_is_radix(vcpu)) {
5530 		rc = kvmhv_copy_from_guest_radix(vcpu, *eaddr, ptr, size);
5531 
5532 		if (rc > 0)
5533 			rc = -EINVAL;
5534 	}
5535 
5536 	/* For now quadrants are the only way to access nested guest memory */
5537 	if (rc && vcpu->arch.nested)
5538 		rc = -EAGAIN;
5539 
5540 	return rc;
5541 }
5542 
5543 static int kvmhv_store_to_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5544 				int size)
5545 {
5546 	int rc = -EINVAL;
5547 
5548 	if (kvmhv_vcpu_is_radix(vcpu)) {
5549 		rc = kvmhv_copy_to_guest_radix(vcpu, *eaddr, ptr, size);
5550 
5551 		if (rc > 0)
5552 			rc = -EINVAL;
5553 	}
5554 
5555 	/* For now quadrants are the only way to access nested guest memory */
5556 	if (rc && vcpu->arch.nested)
5557 		rc = -EAGAIN;
5558 
5559 	return rc;
5560 }
5561 
5562 static void unpin_vpa_reset(struct kvm *kvm, struct kvmppc_vpa *vpa)
5563 {
5564 	unpin_vpa(kvm, vpa);
5565 	vpa->gpa = 0;
5566 	vpa->pinned_addr = NULL;
5567 	vpa->dirty = false;
5568 	vpa->update_pending = 0;
5569 }
5570 
5571 /*
5572  * Enable a guest to become a secure VM, or test whether
5573  * that could be enabled.
5574  * Called when the KVM_CAP_PPC_SECURE_GUEST capability is
5575  * tested (kvm == NULL) or enabled (kvm != NULL).
5576  */
5577 static int kvmhv_enable_svm(struct kvm *kvm)
5578 {
5579 	if (!kvmppc_uvmem_available())
5580 		return -EINVAL;
5581 	if (kvm)
5582 		kvm->arch.svm_enabled = 1;
5583 	return 0;
5584 }
5585 
5586 /*
5587  *  IOCTL handler to turn off secure mode of guest
5588  *
5589  * - Release all device pages
5590  * - Issue ucall to terminate the guest on the UV side
5591  * - Unpin the VPA pages.
5592  * - Reinit the partition scoped page tables
5593  */
5594 static int kvmhv_svm_off(struct kvm *kvm)
5595 {
5596 	struct kvm_vcpu *vcpu;
5597 	int mmu_was_ready;
5598 	int srcu_idx;
5599 	int ret = 0;
5600 	int i;
5601 
5602 	if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
5603 		return ret;
5604 
5605 	mutex_lock(&kvm->arch.mmu_setup_lock);
5606 	mmu_was_ready = kvm->arch.mmu_ready;
5607 	if (kvm->arch.mmu_ready) {
5608 		kvm->arch.mmu_ready = 0;
5609 		/* order mmu_ready vs. vcpus_running */
5610 		smp_mb();
5611 		if (atomic_read(&kvm->arch.vcpus_running)) {
5612 			kvm->arch.mmu_ready = 1;
5613 			ret = -EBUSY;
5614 			goto out;
5615 		}
5616 	}
5617 
5618 	srcu_idx = srcu_read_lock(&kvm->srcu);
5619 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5620 		struct kvm_memory_slot *memslot;
5621 		struct kvm_memslots *slots = __kvm_memslots(kvm, i);
5622 
5623 		if (!slots)
5624 			continue;
5625 
5626 		kvm_for_each_memslot(memslot, slots) {
5627 			kvmppc_uvmem_drop_pages(memslot, kvm, true);
5628 			uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
5629 		}
5630 	}
5631 	srcu_read_unlock(&kvm->srcu, srcu_idx);
5632 
5633 	ret = uv_svm_terminate(kvm->arch.lpid);
5634 	if (ret != U_SUCCESS) {
5635 		ret = -EINVAL;
5636 		goto out;
5637 	}
5638 
5639 	/*
5640 	 * When secure guest is reset, all the guest pages are sent
5641 	 * to UV via UV_PAGE_IN before the non-boot vcpus get a
5642 	 * chance to run and unpin their VPA pages. Unpinning of all
5643 	 * VPA pages is done here explicitly so that VPA pages
5644 	 * can be migrated to the secure side.
5645 	 *
5646 	 * This is required to for the secure SMP guest to reboot
5647 	 * correctly.
5648 	 */
5649 	kvm_for_each_vcpu(i, vcpu, kvm) {
5650 		spin_lock(&vcpu->arch.vpa_update_lock);
5651 		unpin_vpa_reset(kvm, &vcpu->arch.dtl);
5652 		unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
5653 		unpin_vpa_reset(kvm, &vcpu->arch.vpa);
5654 		spin_unlock(&vcpu->arch.vpa_update_lock);
5655 	}
5656 
5657 	kvmppc_setup_partition_table(kvm);
5658 	kvm->arch.secure_guest = 0;
5659 	kvm->arch.mmu_ready = mmu_was_ready;
5660 out:
5661 	mutex_unlock(&kvm->arch.mmu_setup_lock);
5662 	return ret;
5663 }
5664 
5665 static int kvmhv_enable_dawr1(struct kvm *kvm)
5666 {
5667 	if (!cpu_has_feature(CPU_FTR_DAWR1))
5668 		return -ENODEV;
5669 
5670 	/* kvm == NULL means the caller is testing if the capability exists */
5671 	if (kvm)
5672 		kvm->arch.dawr1_enabled = true;
5673 	return 0;
5674 }
5675 
5676 static bool kvmppc_hash_v3_possible(void)
5677 {
5678 	if (radix_enabled() && no_mixing_hpt_and_radix)
5679 		return false;
5680 
5681 	return cpu_has_feature(CPU_FTR_ARCH_300) &&
5682 		cpu_has_feature(CPU_FTR_HVMODE);
5683 }
5684 
5685 static struct kvmppc_ops kvm_ops_hv = {
5686 	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
5687 	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
5688 	.get_one_reg = kvmppc_get_one_reg_hv,
5689 	.set_one_reg = kvmppc_set_one_reg_hv,
5690 	.vcpu_load   = kvmppc_core_vcpu_load_hv,
5691 	.vcpu_put    = kvmppc_core_vcpu_put_hv,
5692 	.inject_interrupt = kvmppc_inject_interrupt_hv,
5693 	.set_msr     = kvmppc_set_msr_hv,
5694 	.vcpu_run    = kvmppc_vcpu_run_hv,
5695 	.vcpu_create = kvmppc_core_vcpu_create_hv,
5696 	.vcpu_free   = kvmppc_core_vcpu_free_hv,
5697 	.check_requests = kvmppc_core_check_requests_hv,
5698 	.get_dirty_log  = kvm_vm_ioctl_get_dirty_log_hv,
5699 	.flush_memslot  = kvmppc_core_flush_memslot_hv,
5700 	.prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
5701 	.commit_memory_region  = kvmppc_core_commit_memory_region_hv,
5702 	.unmap_gfn_range = kvm_unmap_gfn_range_hv,
5703 	.age_gfn = kvm_age_gfn_hv,
5704 	.test_age_gfn = kvm_test_age_gfn_hv,
5705 	.set_spte_gfn = kvm_set_spte_gfn_hv,
5706 	.free_memslot = kvmppc_core_free_memslot_hv,
5707 	.init_vm =  kvmppc_core_init_vm_hv,
5708 	.destroy_vm = kvmppc_core_destroy_vm_hv,
5709 	.get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
5710 	.emulate_op = kvmppc_core_emulate_op_hv,
5711 	.emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
5712 	.emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
5713 	.fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
5714 	.arch_vm_ioctl  = kvm_arch_vm_ioctl_hv,
5715 	.hcall_implemented = kvmppc_hcall_impl_hv,
5716 #ifdef CONFIG_KVM_XICS
5717 	.irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
5718 	.irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
5719 #endif
5720 	.configure_mmu = kvmhv_configure_mmu,
5721 	.get_rmmu_info = kvmhv_get_rmmu_info,
5722 	.set_smt_mode = kvmhv_set_smt_mode,
5723 	.enable_nested = kvmhv_enable_nested,
5724 	.load_from_eaddr = kvmhv_load_from_eaddr,
5725 	.store_to_eaddr = kvmhv_store_to_eaddr,
5726 	.enable_svm = kvmhv_enable_svm,
5727 	.svm_off = kvmhv_svm_off,
5728 	.enable_dawr1 = kvmhv_enable_dawr1,
5729 	.hash_v3_possible = kvmppc_hash_v3_possible,
5730 };
5731 
5732 static int kvm_init_subcore_bitmap(void)
5733 {
5734 	int i, j;
5735 	int nr_cores = cpu_nr_cores();
5736 	struct sibling_subcore_state *sibling_subcore_state;
5737 
5738 	for (i = 0; i < nr_cores; i++) {
5739 		int first_cpu = i * threads_per_core;
5740 		int node = cpu_to_node(first_cpu);
5741 
5742 		/* Ignore if it is already allocated. */
5743 		if (paca_ptrs[first_cpu]->sibling_subcore_state)
5744 			continue;
5745 
5746 		sibling_subcore_state =
5747 			kzalloc_node(sizeof(struct sibling_subcore_state),
5748 							GFP_KERNEL, node);
5749 		if (!sibling_subcore_state)
5750 			return -ENOMEM;
5751 
5752 
5753 		for (j = 0; j < threads_per_core; j++) {
5754 			int cpu = first_cpu + j;
5755 
5756 			paca_ptrs[cpu]->sibling_subcore_state =
5757 						sibling_subcore_state;
5758 		}
5759 	}
5760 	return 0;
5761 }
5762 
5763 static int kvmppc_radix_possible(void)
5764 {
5765 	return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
5766 }
5767 
5768 static int kvmppc_book3s_init_hv(void)
5769 {
5770 	int r;
5771 
5772 	if (!tlbie_capable) {
5773 		pr_err("KVM-HV: Host does not support TLBIE\n");
5774 		return -ENODEV;
5775 	}
5776 
5777 	/*
5778 	 * FIXME!! Do we need to check on all cpus ?
5779 	 */
5780 	r = kvmppc_core_check_processor_compat_hv();
5781 	if (r < 0)
5782 		return -ENODEV;
5783 
5784 	r = kvmhv_nested_init();
5785 	if (r)
5786 		return r;
5787 
5788 	r = kvm_init_subcore_bitmap();
5789 	if (r)
5790 		return r;
5791 
5792 	/*
5793 	 * We need a way of accessing the XICS interrupt controller,
5794 	 * either directly, via paca_ptrs[cpu]->kvm_hstate.xics_phys, or
5795 	 * indirectly, via OPAL.
5796 	 */
5797 #ifdef CONFIG_SMP
5798 	if (!xics_on_xive() && !kvmhv_on_pseries() &&
5799 	    !local_paca->kvm_hstate.xics_phys) {
5800 		struct device_node *np;
5801 
5802 		np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
5803 		if (!np) {
5804 			pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
5805 			return -ENODEV;
5806 		}
5807 		/* presence of intc confirmed - node can be dropped again */
5808 		of_node_put(np);
5809 	}
5810 #endif
5811 
5812 	kvm_ops_hv.owner = THIS_MODULE;
5813 	kvmppc_hv_ops = &kvm_ops_hv;
5814 
5815 	init_default_hcalls();
5816 
5817 	init_vcore_lists();
5818 
5819 	r = kvmppc_mmu_hv_init();
5820 	if (r)
5821 		return r;
5822 
5823 	if (kvmppc_radix_possible())
5824 		r = kvmppc_radix_init();
5825 
5826 	/*
5827 	 * POWER9 chips before version 2.02 can't have some threads in
5828 	 * HPT mode and some in radix mode on the same core.
5829 	 */
5830 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5831 		unsigned int pvr = mfspr(SPRN_PVR);
5832 		if ((pvr >> 16) == PVR_POWER9 &&
5833 		    (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
5834 		     ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
5835 			no_mixing_hpt_and_radix = true;
5836 	}
5837 
5838 	r = kvmppc_uvmem_init();
5839 	if (r < 0)
5840 		pr_err("KVM-HV: kvmppc_uvmem_init failed %d\n", r);
5841 
5842 	return r;
5843 }
5844 
5845 static void kvmppc_book3s_exit_hv(void)
5846 {
5847 	kvmppc_uvmem_free();
5848 	kvmppc_free_host_rm_ops();
5849 	if (kvmppc_radix_possible())
5850 		kvmppc_radix_exit();
5851 	kvmppc_hv_ops = NULL;
5852 	kvmhv_nested_exit();
5853 }
5854 
5855 module_init(kvmppc_book3s_init_hv);
5856 module_exit(kvmppc_book3s_exit_hv);
5857 MODULE_LICENSE("GPL");
5858 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5859 MODULE_ALIAS("devname:kvm");
5860