xref: /openbmc/linux/arch/powerpc/kvm/book3s_hv.c (revision 6774def6)
1 /*
2  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4  *
5  * Authors:
6  *    Paul Mackerras <paulus@au1.ibm.com>
7  *    Alexander Graf <agraf@suse.de>
8  *    Kevin Wolf <mail@kevin-wolf.de>
9  *
10  * Description: KVM functions specific to running on Book 3S
11  * processors in hypervisor mode (specifically POWER7 and later).
12  *
13  * This file is derived from arch/powerpc/kvm/book3s.c,
14  * by Alexander Graf <agraf@suse.de>.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License, version 2, as
18  * published by the Free Software Foundation.
19  */
20 
21 #include <linux/kvm_host.h>
22 #include <linux/err.h>
23 #include <linux/slab.h>
24 #include <linux/preempt.h>
25 #include <linux/sched.h>
26 #include <linux/delay.h>
27 #include <linux/export.h>
28 #include <linux/fs.h>
29 #include <linux/anon_inodes.h>
30 #include <linux/cpumask.h>
31 #include <linux/spinlock.h>
32 #include <linux/page-flags.h>
33 #include <linux/srcu.h>
34 #include <linux/miscdevice.h>
35 
36 #include <asm/reg.h>
37 #include <asm/cputable.h>
38 #include <asm/cache.h>
39 #include <asm/cacheflush.h>
40 #include <asm/tlbflush.h>
41 #include <asm/uaccess.h>
42 #include <asm/io.h>
43 #include <asm/kvm_ppc.h>
44 #include <asm/kvm_book3s.h>
45 #include <asm/mmu_context.h>
46 #include <asm/lppaca.h>
47 #include <asm/processor.h>
48 #include <asm/cputhreads.h>
49 #include <asm/page.h>
50 #include <asm/hvcall.h>
51 #include <asm/switch_to.h>
52 #include <asm/smp.h>
53 #include <linux/gfp.h>
54 #include <linux/vmalloc.h>
55 #include <linux/highmem.h>
56 #include <linux/hugetlb.h>
57 #include <linux/module.h>
58 
59 #include "book3s.h"
60 
61 /* #define EXIT_DEBUG */
62 /* #define EXIT_DEBUG_SIMPLE */
63 /* #define EXIT_DEBUG_INT */
64 
65 /* Used to indicate that a guest page fault needs to be handled */
66 #define RESUME_PAGE_FAULT	(RESUME_GUEST | RESUME_FLAG_ARCH1)
67 
68 /* Used as a "null" value for timebase values */
69 #define TB_NIL	(~(u64)0)
70 
71 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
72 
73 #if defined(CONFIG_PPC_64K_PAGES)
74 #define MPP_BUFFER_ORDER	0
75 #elif defined(CONFIG_PPC_4K_PAGES)
76 #define MPP_BUFFER_ORDER	3
77 #endif
78 
79 
80 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
81 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
82 
83 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
84 {
85 	int me;
86 	int cpu = vcpu->cpu;
87 	wait_queue_head_t *wqp;
88 
89 	wqp = kvm_arch_vcpu_wq(vcpu);
90 	if (waitqueue_active(wqp)) {
91 		wake_up_interruptible(wqp);
92 		++vcpu->stat.halt_wakeup;
93 	}
94 
95 	me = get_cpu();
96 
97 	/* CPU points to the first thread of the core */
98 	if (cpu != me && cpu >= 0 && cpu < nr_cpu_ids) {
99 #ifdef CONFIG_PPC_ICP_NATIVE
100 		int real_cpu = cpu + vcpu->arch.ptid;
101 		if (paca[real_cpu].kvm_hstate.xics_phys)
102 			xics_wake_cpu(real_cpu);
103 		else
104 #endif
105 		if (cpu_online(cpu))
106 			smp_send_reschedule(cpu);
107 	}
108 	put_cpu();
109 }
110 
111 /*
112  * We use the vcpu_load/put functions to measure stolen time.
113  * Stolen time is counted as time when either the vcpu is able to
114  * run as part of a virtual core, but the task running the vcore
115  * is preempted or sleeping, or when the vcpu needs something done
116  * in the kernel by the task running the vcpu, but that task is
117  * preempted or sleeping.  Those two things have to be counted
118  * separately, since one of the vcpu tasks will take on the job
119  * of running the core, and the other vcpu tasks in the vcore will
120  * sleep waiting for it to do that, but that sleep shouldn't count
121  * as stolen time.
122  *
123  * Hence we accumulate stolen time when the vcpu can run as part of
124  * a vcore using vc->stolen_tb, and the stolen time when the vcpu
125  * needs its task to do other things in the kernel (for example,
126  * service a page fault) in busy_stolen.  We don't accumulate
127  * stolen time for a vcore when it is inactive, or for a vcpu
128  * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
129  * a misnomer; it means that the vcpu task is not executing in
130  * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
131  * the kernel.  We don't have any way of dividing up that time
132  * between time that the vcpu is genuinely stopped, time that
133  * the task is actively working on behalf of the vcpu, and time
134  * that the task is preempted, so we don't count any of it as
135  * stolen.
136  *
137  * Updates to busy_stolen are protected by arch.tbacct_lock;
138  * updates to vc->stolen_tb are protected by the arch.tbacct_lock
139  * of the vcpu that has taken responsibility for running the vcore
140  * (i.e. vc->runner).  The stolen times are measured in units of
141  * timebase ticks.  (Note that the != TB_NIL checks below are
142  * purely defensive; they should never fail.)
143  */
144 
145 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
146 {
147 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
148 	unsigned long flags;
149 
150 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
151 	if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE &&
152 	    vc->preempt_tb != TB_NIL) {
153 		vc->stolen_tb += mftb() - vc->preempt_tb;
154 		vc->preempt_tb = TB_NIL;
155 	}
156 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
157 	    vcpu->arch.busy_preempt != TB_NIL) {
158 		vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
159 		vcpu->arch.busy_preempt = TB_NIL;
160 	}
161 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
162 }
163 
164 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
165 {
166 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
167 	unsigned long flags;
168 
169 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
170 	if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE)
171 		vc->preempt_tb = mftb();
172 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
173 		vcpu->arch.busy_preempt = mftb();
174 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
175 }
176 
177 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
178 {
179 	vcpu->arch.shregs.msr = msr;
180 	kvmppc_end_cede(vcpu);
181 }
182 
183 void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
184 {
185 	vcpu->arch.pvr = pvr;
186 }
187 
188 int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
189 {
190 	unsigned long pcr = 0;
191 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
192 
193 	if (arch_compat) {
194 		if (!cpu_has_feature(CPU_FTR_ARCH_206))
195 			return -EINVAL;	/* 970 has no compat mode support */
196 
197 		switch (arch_compat) {
198 		case PVR_ARCH_205:
199 			/*
200 			 * If an arch bit is set in PCR, all the defined
201 			 * higher-order arch bits also have to be set.
202 			 */
203 			pcr = PCR_ARCH_206 | PCR_ARCH_205;
204 			break;
205 		case PVR_ARCH_206:
206 		case PVR_ARCH_206p:
207 			pcr = PCR_ARCH_206;
208 			break;
209 		case PVR_ARCH_207:
210 			break;
211 		default:
212 			return -EINVAL;
213 		}
214 
215 		if (!cpu_has_feature(CPU_FTR_ARCH_207S)) {
216 			/* POWER7 can't emulate POWER8 */
217 			if (!(pcr & PCR_ARCH_206))
218 				return -EINVAL;
219 			pcr &= ~PCR_ARCH_206;
220 		}
221 	}
222 
223 	spin_lock(&vc->lock);
224 	vc->arch_compat = arch_compat;
225 	vc->pcr = pcr;
226 	spin_unlock(&vc->lock);
227 
228 	return 0;
229 }
230 
231 void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
232 {
233 	int r;
234 
235 	pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
236 	pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
237 	       vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
238 	for (r = 0; r < 16; ++r)
239 		pr_err("r%2d = %.16lx  r%d = %.16lx\n",
240 		       r, kvmppc_get_gpr(vcpu, r),
241 		       r+16, kvmppc_get_gpr(vcpu, r+16));
242 	pr_err("ctr = %.16lx  lr  = %.16lx\n",
243 	       vcpu->arch.ctr, vcpu->arch.lr);
244 	pr_err("srr0 = %.16llx srr1 = %.16llx\n",
245 	       vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
246 	pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
247 	       vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
248 	pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
249 	       vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
250 	pr_err("cr = %.8x  xer = %.16lx  dsisr = %.8x\n",
251 	       vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
252 	pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
253 	pr_err("fault dar = %.16lx dsisr = %.8x\n",
254 	       vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
255 	pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
256 	for (r = 0; r < vcpu->arch.slb_max; ++r)
257 		pr_err("  ESID = %.16llx VSID = %.16llx\n",
258 		       vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
259 	pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
260 	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
261 	       vcpu->arch.last_inst);
262 }
263 
264 struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
265 {
266 	int r;
267 	struct kvm_vcpu *v, *ret = NULL;
268 
269 	mutex_lock(&kvm->lock);
270 	kvm_for_each_vcpu(r, v, kvm) {
271 		if (v->vcpu_id == id) {
272 			ret = v;
273 			break;
274 		}
275 	}
276 	mutex_unlock(&kvm->lock);
277 	return ret;
278 }
279 
280 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
281 {
282 	vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
283 	vpa->yield_count = cpu_to_be32(1);
284 }
285 
286 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
287 		   unsigned long addr, unsigned long len)
288 {
289 	/* check address is cacheline aligned */
290 	if (addr & (L1_CACHE_BYTES - 1))
291 		return -EINVAL;
292 	spin_lock(&vcpu->arch.vpa_update_lock);
293 	if (v->next_gpa != addr || v->len != len) {
294 		v->next_gpa = addr;
295 		v->len = addr ? len : 0;
296 		v->update_pending = 1;
297 	}
298 	spin_unlock(&vcpu->arch.vpa_update_lock);
299 	return 0;
300 }
301 
302 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
303 struct reg_vpa {
304 	u32 dummy;
305 	union {
306 		__be16 hword;
307 		__be32 word;
308 	} length;
309 };
310 
311 static int vpa_is_registered(struct kvmppc_vpa *vpap)
312 {
313 	if (vpap->update_pending)
314 		return vpap->next_gpa != 0;
315 	return vpap->pinned_addr != NULL;
316 }
317 
318 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
319 				       unsigned long flags,
320 				       unsigned long vcpuid, unsigned long vpa)
321 {
322 	struct kvm *kvm = vcpu->kvm;
323 	unsigned long len, nb;
324 	void *va;
325 	struct kvm_vcpu *tvcpu;
326 	int err;
327 	int subfunc;
328 	struct kvmppc_vpa *vpap;
329 
330 	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
331 	if (!tvcpu)
332 		return H_PARAMETER;
333 
334 	subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
335 	if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
336 	    subfunc == H_VPA_REG_SLB) {
337 		/* Registering new area - address must be cache-line aligned */
338 		if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
339 			return H_PARAMETER;
340 
341 		/* convert logical addr to kernel addr and read length */
342 		va = kvmppc_pin_guest_page(kvm, vpa, &nb);
343 		if (va == NULL)
344 			return H_PARAMETER;
345 		if (subfunc == H_VPA_REG_VPA)
346 			len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
347 		else
348 			len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
349 		kvmppc_unpin_guest_page(kvm, va, vpa, false);
350 
351 		/* Check length */
352 		if (len > nb || len < sizeof(struct reg_vpa))
353 			return H_PARAMETER;
354 	} else {
355 		vpa = 0;
356 		len = 0;
357 	}
358 
359 	err = H_PARAMETER;
360 	vpap = NULL;
361 	spin_lock(&tvcpu->arch.vpa_update_lock);
362 
363 	switch (subfunc) {
364 	case H_VPA_REG_VPA:		/* register VPA */
365 		if (len < sizeof(struct lppaca))
366 			break;
367 		vpap = &tvcpu->arch.vpa;
368 		err = 0;
369 		break;
370 
371 	case H_VPA_REG_DTL:		/* register DTL */
372 		if (len < sizeof(struct dtl_entry))
373 			break;
374 		len -= len % sizeof(struct dtl_entry);
375 
376 		/* Check that they have previously registered a VPA */
377 		err = H_RESOURCE;
378 		if (!vpa_is_registered(&tvcpu->arch.vpa))
379 			break;
380 
381 		vpap = &tvcpu->arch.dtl;
382 		err = 0;
383 		break;
384 
385 	case H_VPA_REG_SLB:		/* register SLB shadow buffer */
386 		/* Check that they have previously registered a VPA */
387 		err = H_RESOURCE;
388 		if (!vpa_is_registered(&tvcpu->arch.vpa))
389 			break;
390 
391 		vpap = &tvcpu->arch.slb_shadow;
392 		err = 0;
393 		break;
394 
395 	case H_VPA_DEREG_VPA:		/* deregister VPA */
396 		/* Check they don't still have a DTL or SLB buf registered */
397 		err = H_RESOURCE;
398 		if (vpa_is_registered(&tvcpu->arch.dtl) ||
399 		    vpa_is_registered(&tvcpu->arch.slb_shadow))
400 			break;
401 
402 		vpap = &tvcpu->arch.vpa;
403 		err = 0;
404 		break;
405 
406 	case H_VPA_DEREG_DTL:		/* deregister DTL */
407 		vpap = &tvcpu->arch.dtl;
408 		err = 0;
409 		break;
410 
411 	case H_VPA_DEREG_SLB:		/* deregister SLB shadow buffer */
412 		vpap = &tvcpu->arch.slb_shadow;
413 		err = 0;
414 		break;
415 	}
416 
417 	if (vpap) {
418 		vpap->next_gpa = vpa;
419 		vpap->len = len;
420 		vpap->update_pending = 1;
421 	}
422 
423 	spin_unlock(&tvcpu->arch.vpa_update_lock);
424 
425 	return err;
426 }
427 
428 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
429 {
430 	struct kvm *kvm = vcpu->kvm;
431 	void *va;
432 	unsigned long nb;
433 	unsigned long gpa;
434 
435 	/*
436 	 * We need to pin the page pointed to by vpap->next_gpa,
437 	 * but we can't call kvmppc_pin_guest_page under the lock
438 	 * as it does get_user_pages() and down_read().  So we
439 	 * have to drop the lock, pin the page, then get the lock
440 	 * again and check that a new area didn't get registered
441 	 * in the meantime.
442 	 */
443 	for (;;) {
444 		gpa = vpap->next_gpa;
445 		spin_unlock(&vcpu->arch.vpa_update_lock);
446 		va = NULL;
447 		nb = 0;
448 		if (gpa)
449 			va = kvmppc_pin_guest_page(kvm, gpa, &nb);
450 		spin_lock(&vcpu->arch.vpa_update_lock);
451 		if (gpa == vpap->next_gpa)
452 			break;
453 		/* sigh... unpin that one and try again */
454 		if (va)
455 			kvmppc_unpin_guest_page(kvm, va, gpa, false);
456 	}
457 
458 	vpap->update_pending = 0;
459 	if (va && nb < vpap->len) {
460 		/*
461 		 * If it's now too short, it must be that userspace
462 		 * has changed the mappings underlying guest memory,
463 		 * so unregister the region.
464 		 */
465 		kvmppc_unpin_guest_page(kvm, va, gpa, false);
466 		va = NULL;
467 	}
468 	if (vpap->pinned_addr)
469 		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
470 					vpap->dirty);
471 	vpap->gpa = gpa;
472 	vpap->pinned_addr = va;
473 	vpap->dirty = false;
474 	if (va)
475 		vpap->pinned_end = va + vpap->len;
476 }
477 
478 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
479 {
480 	if (!(vcpu->arch.vpa.update_pending ||
481 	      vcpu->arch.slb_shadow.update_pending ||
482 	      vcpu->arch.dtl.update_pending))
483 		return;
484 
485 	spin_lock(&vcpu->arch.vpa_update_lock);
486 	if (vcpu->arch.vpa.update_pending) {
487 		kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
488 		if (vcpu->arch.vpa.pinned_addr)
489 			init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
490 	}
491 	if (vcpu->arch.dtl.update_pending) {
492 		kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
493 		vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
494 		vcpu->arch.dtl_index = 0;
495 	}
496 	if (vcpu->arch.slb_shadow.update_pending)
497 		kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
498 	spin_unlock(&vcpu->arch.vpa_update_lock);
499 }
500 
501 /*
502  * Return the accumulated stolen time for the vcore up until `now'.
503  * The caller should hold the vcore lock.
504  */
505 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
506 {
507 	u64 p;
508 
509 	/*
510 	 * If we are the task running the vcore, then since we hold
511 	 * the vcore lock, we can't be preempted, so stolen_tb/preempt_tb
512 	 * can't be updated, so we don't need the tbacct_lock.
513 	 * If the vcore is inactive, it can't become active (since we
514 	 * hold the vcore lock), so the vcpu load/put functions won't
515 	 * update stolen_tb/preempt_tb, and we don't need tbacct_lock.
516 	 */
517 	if (vc->vcore_state != VCORE_INACTIVE &&
518 	    vc->runner->arch.run_task != current) {
519 		spin_lock_irq(&vc->runner->arch.tbacct_lock);
520 		p = vc->stolen_tb;
521 		if (vc->preempt_tb != TB_NIL)
522 			p += now - vc->preempt_tb;
523 		spin_unlock_irq(&vc->runner->arch.tbacct_lock);
524 	} else {
525 		p = vc->stolen_tb;
526 	}
527 	return p;
528 }
529 
530 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
531 				    struct kvmppc_vcore *vc)
532 {
533 	struct dtl_entry *dt;
534 	struct lppaca *vpa;
535 	unsigned long stolen;
536 	unsigned long core_stolen;
537 	u64 now;
538 
539 	dt = vcpu->arch.dtl_ptr;
540 	vpa = vcpu->arch.vpa.pinned_addr;
541 	now = mftb();
542 	core_stolen = vcore_stolen_time(vc, now);
543 	stolen = core_stolen - vcpu->arch.stolen_logged;
544 	vcpu->arch.stolen_logged = core_stolen;
545 	spin_lock_irq(&vcpu->arch.tbacct_lock);
546 	stolen += vcpu->arch.busy_stolen;
547 	vcpu->arch.busy_stolen = 0;
548 	spin_unlock_irq(&vcpu->arch.tbacct_lock);
549 	if (!dt || !vpa)
550 		return;
551 	memset(dt, 0, sizeof(struct dtl_entry));
552 	dt->dispatch_reason = 7;
553 	dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
554 	dt->timebase = cpu_to_be64(now + vc->tb_offset);
555 	dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
556 	dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
557 	dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
558 	++dt;
559 	if (dt == vcpu->arch.dtl.pinned_end)
560 		dt = vcpu->arch.dtl.pinned_addr;
561 	vcpu->arch.dtl_ptr = dt;
562 	/* order writing *dt vs. writing vpa->dtl_idx */
563 	smp_wmb();
564 	vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
565 	vcpu->arch.dtl.dirty = true;
566 }
567 
568 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
569 {
570 	if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
571 		return true;
572 	if ((!vcpu->arch.vcore->arch_compat) &&
573 	    cpu_has_feature(CPU_FTR_ARCH_207S))
574 		return true;
575 	return false;
576 }
577 
578 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
579 			     unsigned long resource, unsigned long value1,
580 			     unsigned long value2)
581 {
582 	switch (resource) {
583 	case H_SET_MODE_RESOURCE_SET_CIABR:
584 		if (!kvmppc_power8_compatible(vcpu))
585 			return H_P2;
586 		if (value2)
587 			return H_P4;
588 		if (mflags)
589 			return H_UNSUPPORTED_FLAG_START;
590 		/* Guests can't breakpoint the hypervisor */
591 		if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
592 			return H_P3;
593 		vcpu->arch.ciabr  = value1;
594 		return H_SUCCESS;
595 	case H_SET_MODE_RESOURCE_SET_DAWR:
596 		if (!kvmppc_power8_compatible(vcpu))
597 			return H_P2;
598 		if (mflags)
599 			return H_UNSUPPORTED_FLAG_START;
600 		if (value2 & DABRX_HYP)
601 			return H_P4;
602 		vcpu->arch.dawr  = value1;
603 		vcpu->arch.dawrx = value2;
604 		return H_SUCCESS;
605 	default:
606 		return H_TOO_HARD;
607 	}
608 }
609 
610 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
611 {
612 	unsigned long req = kvmppc_get_gpr(vcpu, 3);
613 	unsigned long target, ret = H_SUCCESS;
614 	struct kvm_vcpu *tvcpu;
615 	int idx, rc;
616 
617 	if (req <= MAX_HCALL_OPCODE &&
618 	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
619 		return RESUME_HOST;
620 
621 	switch (req) {
622 	case H_ENTER:
623 		idx = srcu_read_lock(&vcpu->kvm->srcu);
624 		ret = kvmppc_virtmode_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4),
625 					      kvmppc_get_gpr(vcpu, 5),
626 					      kvmppc_get_gpr(vcpu, 6),
627 					      kvmppc_get_gpr(vcpu, 7));
628 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
629 		break;
630 	case H_CEDE:
631 		break;
632 	case H_PROD:
633 		target = kvmppc_get_gpr(vcpu, 4);
634 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
635 		if (!tvcpu) {
636 			ret = H_PARAMETER;
637 			break;
638 		}
639 		tvcpu->arch.prodded = 1;
640 		smp_mb();
641 		if (vcpu->arch.ceded) {
642 			if (waitqueue_active(&vcpu->wq)) {
643 				wake_up_interruptible(&vcpu->wq);
644 				vcpu->stat.halt_wakeup++;
645 			}
646 		}
647 		break;
648 	case H_CONFER:
649 		target = kvmppc_get_gpr(vcpu, 4);
650 		if (target == -1)
651 			break;
652 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
653 		if (!tvcpu) {
654 			ret = H_PARAMETER;
655 			break;
656 		}
657 		kvm_vcpu_yield_to(tvcpu);
658 		break;
659 	case H_REGISTER_VPA:
660 		ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
661 					kvmppc_get_gpr(vcpu, 5),
662 					kvmppc_get_gpr(vcpu, 6));
663 		break;
664 	case H_RTAS:
665 		if (list_empty(&vcpu->kvm->arch.rtas_tokens))
666 			return RESUME_HOST;
667 
668 		idx = srcu_read_lock(&vcpu->kvm->srcu);
669 		rc = kvmppc_rtas_hcall(vcpu);
670 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
671 
672 		if (rc == -ENOENT)
673 			return RESUME_HOST;
674 		else if (rc == 0)
675 			break;
676 
677 		/* Send the error out to userspace via KVM_RUN */
678 		return rc;
679 	case H_SET_MODE:
680 		ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
681 					kvmppc_get_gpr(vcpu, 5),
682 					kvmppc_get_gpr(vcpu, 6),
683 					kvmppc_get_gpr(vcpu, 7));
684 		if (ret == H_TOO_HARD)
685 			return RESUME_HOST;
686 		break;
687 	case H_XIRR:
688 	case H_CPPR:
689 	case H_EOI:
690 	case H_IPI:
691 	case H_IPOLL:
692 	case H_XIRR_X:
693 		if (kvmppc_xics_enabled(vcpu)) {
694 			ret = kvmppc_xics_hcall(vcpu, req);
695 			break;
696 		} /* fallthrough */
697 	default:
698 		return RESUME_HOST;
699 	}
700 	kvmppc_set_gpr(vcpu, 3, ret);
701 	vcpu->arch.hcall_needed = 0;
702 	return RESUME_GUEST;
703 }
704 
705 static int kvmppc_hcall_impl_hv(unsigned long cmd)
706 {
707 	switch (cmd) {
708 	case H_CEDE:
709 	case H_PROD:
710 	case H_CONFER:
711 	case H_REGISTER_VPA:
712 	case H_SET_MODE:
713 #ifdef CONFIG_KVM_XICS
714 	case H_XIRR:
715 	case H_CPPR:
716 	case H_EOI:
717 	case H_IPI:
718 	case H_IPOLL:
719 	case H_XIRR_X:
720 #endif
721 		return 1;
722 	}
723 
724 	/* See if it's in the real-mode table */
725 	return kvmppc_hcall_impl_hv_realmode(cmd);
726 }
727 
728 static int kvmppc_emulate_debug_inst(struct kvm_run *run,
729 					struct kvm_vcpu *vcpu)
730 {
731 	u32 last_inst;
732 
733 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
734 					EMULATE_DONE) {
735 		/*
736 		 * Fetch failed, so return to guest and
737 		 * try executing it again.
738 		 */
739 		return RESUME_GUEST;
740 	}
741 
742 	if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
743 		run->exit_reason = KVM_EXIT_DEBUG;
744 		run->debug.arch.address = kvmppc_get_pc(vcpu);
745 		return RESUME_HOST;
746 	} else {
747 		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
748 		return RESUME_GUEST;
749 	}
750 }
751 
752 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
753 				 struct task_struct *tsk)
754 {
755 	int r = RESUME_HOST;
756 
757 	vcpu->stat.sum_exits++;
758 
759 	run->exit_reason = KVM_EXIT_UNKNOWN;
760 	run->ready_for_interrupt_injection = 1;
761 	switch (vcpu->arch.trap) {
762 	/* We're good on these - the host merely wanted to get our attention */
763 	case BOOK3S_INTERRUPT_HV_DECREMENTER:
764 		vcpu->stat.dec_exits++;
765 		r = RESUME_GUEST;
766 		break;
767 	case BOOK3S_INTERRUPT_EXTERNAL:
768 	case BOOK3S_INTERRUPT_H_DOORBELL:
769 		vcpu->stat.ext_intr_exits++;
770 		r = RESUME_GUEST;
771 		break;
772 	case BOOK3S_INTERRUPT_PERFMON:
773 		r = RESUME_GUEST;
774 		break;
775 	case BOOK3S_INTERRUPT_MACHINE_CHECK:
776 		/*
777 		 * Deliver a machine check interrupt to the guest.
778 		 * We have to do this, even if the host has handled the
779 		 * machine check, because machine checks use SRR0/1 and
780 		 * the interrupt might have trashed guest state in them.
781 		 */
782 		kvmppc_book3s_queue_irqprio(vcpu,
783 					    BOOK3S_INTERRUPT_MACHINE_CHECK);
784 		r = RESUME_GUEST;
785 		break;
786 	case BOOK3S_INTERRUPT_PROGRAM:
787 	{
788 		ulong flags;
789 		/*
790 		 * Normally program interrupts are delivered directly
791 		 * to the guest by the hardware, but we can get here
792 		 * as a result of a hypervisor emulation interrupt
793 		 * (e40) getting turned into a 700 by BML RTAS.
794 		 */
795 		flags = vcpu->arch.shregs.msr & 0x1f0000ull;
796 		kvmppc_core_queue_program(vcpu, flags);
797 		r = RESUME_GUEST;
798 		break;
799 	}
800 	case BOOK3S_INTERRUPT_SYSCALL:
801 	{
802 		/* hcall - punt to userspace */
803 		int i;
804 
805 		/* hypercall with MSR_PR has already been handled in rmode,
806 		 * and never reaches here.
807 		 */
808 
809 		run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
810 		for (i = 0; i < 9; ++i)
811 			run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
812 		run->exit_reason = KVM_EXIT_PAPR_HCALL;
813 		vcpu->arch.hcall_needed = 1;
814 		r = RESUME_HOST;
815 		break;
816 	}
817 	/*
818 	 * We get these next two if the guest accesses a page which it thinks
819 	 * it has mapped but which is not actually present, either because
820 	 * it is for an emulated I/O device or because the corresonding
821 	 * host page has been paged out.  Any other HDSI/HISI interrupts
822 	 * have been handled already.
823 	 */
824 	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
825 		r = RESUME_PAGE_FAULT;
826 		break;
827 	case BOOK3S_INTERRUPT_H_INST_STORAGE:
828 		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
829 		vcpu->arch.fault_dsisr = 0;
830 		r = RESUME_PAGE_FAULT;
831 		break;
832 	/*
833 	 * This occurs if the guest executes an illegal instruction.
834 	 * If the guest debug is disabled, generate a program interrupt
835 	 * to the guest. If guest debug is enabled, we need to check
836 	 * whether the instruction is a software breakpoint instruction.
837 	 * Accordingly return to Guest or Host.
838 	 */
839 	case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
840 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
841 			r = kvmppc_emulate_debug_inst(run, vcpu);
842 		} else {
843 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
844 			r = RESUME_GUEST;
845 		}
846 		break;
847 	/*
848 	 * This occurs if the guest (kernel or userspace), does something that
849 	 * is prohibited by HFSCR.  We just generate a program interrupt to
850 	 * the guest.
851 	 */
852 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
853 		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
854 		r = RESUME_GUEST;
855 		break;
856 	default:
857 		kvmppc_dump_regs(vcpu);
858 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
859 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
860 			vcpu->arch.shregs.msr);
861 		run->hw.hardware_exit_reason = vcpu->arch.trap;
862 		r = RESUME_HOST;
863 		break;
864 	}
865 
866 	return r;
867 }
868 
869 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
870 					    struct kvm_sregs *sregs)
871 {
872 	int i;
873 
874 	memset(sregs, 0, sizeof(struct kvm_sregs));
875 	sregs->pvr = vcpu->arch.pvr;
876 	for (i = 0; i < vcpu->arch.slb_max; i++) {
877 		sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
878 		sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
879 	}
880 
881 	return 0;
882 }
883 
884 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
885 					    struct kvm_sregs *sregs)
886 {
887 	int i, j;
888 
889 	/* Only accept the same PVR as the host's, since we can't spoof it */
890 	if (sregs->pvr != vcpu->arch.pvr)
891 		return -EINVAL;
892 
893 	j = 0;
894 	for (i = 0; i < vcpu->arch.slb_nr; i++) {
895 		if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
896 			vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
897 			vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
898 			++j;
899 		}
900 	}
901 	vcpu->arch.slb_max = j;
902 
903 	return 0;
904 }
905 
906 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
907 		bool preserve_top32)
908 {
909 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
910 	u64 mask;
911 
912 	spin_lock(&vc->lock);
913 	/*
914 	 * If ILE (interrupt little-endian) has changed, update the
915 	 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
916 	 */
917 	if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
918 		struct kvm *kvm = vcpu->kvm;
919 		struct kvm_vcpu *vcpu;
920 		int i;
921 
922 		mutex_lock(&kvm->lock);
923 		kvm_for_each_vcpu(i, vcpu, kvm) {
924 			if (vcpu->arch.vcore != vc)
925 				continue;
926 			if (new_lpcr & LPCR_ILE)
927 				vcpu->arch.intr_msr |= MSR_LE;
928 			else
929 				vcpu->arch.intr_msr &= ~MSR_LE;
930 		}
931 		mutex_unlock(&kvm->lock);
932 	}
933 
934 	/*
935 	 * Userspace can only modify DPFD (default prefetch depth),
936 	 * ILE (interrupt little-endian) and TC (translation control).
937 	 * On POWER8 userspace can also modify AIL (alt. interrupt loc.)
938 	 */
939 	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
940 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
941 		mask |= LPCR_AIL;
942 
943 	/* Broken 32-bit version of LPCR must not clear top bits */
944 	if (preserve_top32)
945 		mask &= 0xFFFFFFFF;
946 	vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
947 	spin_unlock(&vc->lock);
948 }
949 
950 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
951 				 union kvmppc_one_reg *val)
952 {
953 	int r = 0;
954 	long int i;
955 
956 	switch (id) {
957 	case KVM_REG_PPC_DEBUG_INST:
958 		*val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
959 		break;
960 	case KVM_REG_PPC_HIOR:
961 		*val = get_reg_val(id, 0);
962 		break;
963 	case KVM_REG_PPC_DABR:
964 		*val = get_reg_val(id, vcpu->arch.dabr);
965 		break;
966 	case KVM_REG_PPC_DABRX:
967 		*val = get_reg_val(id, vcpu->arch.dabrx);
968 		break;
969 	case KVM_REG_PPC_DSCR:
970 		*val = get_reg_val(id, vcpu->arch.dscr);
971 		break;
972 	case KVM_REG_PPC_PURR:
973 		*val = get_reg_val(id, vcpu->arch.purr);
974 		break;
975 	case KVM_REG_PPC_SPURR:
976 		*val = get_reg_val(id, vcpu->arch.spurr);
977 		break;
978 	case KVM_REG_PPC_AMR:
979 		*val = get_reg_val(id, vcpu->arch.amr);
980 		break;
981 	case KVM_REG_PPC_UAMOR:
982 		*val = get_reg_val(id, vcpu->arch.uamor);
983 		break;
984 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
985 		i = id - KVM_REG_PPC_MMCR0;
986 		*val = get_reg_val(id, vcpu->arch.mmcr[i]);
987 		break;
988 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
989 		i = id - KVM_REG_PPC_PMC1;
990 		*val = get_reg_val(id, vcpu->arch.pmc[i]);
991 		break;
992 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
993 		i = id - KVM_REG_PPC_SPMC1;
994 		*val = get_reg_val(id, vcpu->arch.spmc[i]);
995 		break;
996 	case KVM_REG_PPC_SIAR:
997 		*val = get_reg_val(id, vcpu->arch.siar);
998 		break;
999 	case KVM_REG_PPC_SDAR:
1000 		*val = get_reg_val(id, vcpu->arch.sdar);
1001 		break;
1002 	case KVM_REG_PPC_SIER:
1003 		*val = get_reg_val(id, vcpu->arch.sier);
1004 		break;
1005 	case KVM_REG_PPC_IAMR:
1006 		*val = get_reg_val(id, vcpu->arch.iamr);
1007 		break;
1008 	case KVM_REG_PPC_PSPB:
1009 		*val = get_reg_val(id, vcpu->arch.pspb);
1010 		break;
1011 	case KVM_REG_PPC_DPDES:
1012 		*val = get_reg_val(id, vcpu->arch.vcore->dpdes);
1013 		break;
1014 	case KVM_REG_PPC_DAWR:
1015 		*val = get_reg_val(id, vcpu->arch.dawr);
1016 		break;
1017 	case KVM_REG_PPC_DAWRX:
1018 		*val = get_reg_val(id, vcpu->arch.dawrx);
1019 		break;
1020 	case KVM_REG_PPC_CIABR:
1021 		*val = get_reg_val(id, vcpu->arch.ciabr);
1022 		break;
1023 	case KVM_REG_PPC_CSIGR:
1024 		*val = get_reg_val(id, vcpu->arch.csigr);
1025 		break;
1026 	case KVM_REG_PPC_TACR:
1027 		*val = get_reg_val(id, vcpu->arch.tacr);
1028 		break;
1029 	case KVM_REG_PPC_TCSCR:
1030 		*val = get_reg_val(id, vcpu->arch.tcscr);
1031 		break;
1032 	case KVM_REG_PPC_PID:
1033 		*val = get_reg_val(id, vcpu->arch.pid);
1034 		break;
1035 	case KVM_REG_PPC_ACOP:
1036 		*val = get_reg_val(id, vcpu->arch.acop);
1037 		break;
1038 	case KVM_REG_PPC_WORT:
1039 		*val = get_reg_val(id, vcpu->arch.wort);
1040 		break;
1041 	case KVM_REG_PPC_VPA_ADDR:
1042 		spin_lock(&vcpu->arch.vpa_update_lock);
1043 		*val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1044 		spin_unlock(&vcpu->arch.vpa_update_lock);
1045 		break;
1046 	case KVM_REG_PPC_VPA_SLB:
1047 		spin_lock(&vcpu->arch.vpa_update_lock);
1048 		val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1049 		val->vpaval.length = vcpu->arch.slb_shadow.len;
1050 		spin_unlock(&vcpu->arch.vpa_update_lock);
1051 		break;
1052 	case KVM_REG_PPC_VPA_DTL:
1053 		spin_lock(&vcpu->arch.vpa_update_lock);
1054 		val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1055 		val->vpaval.length = vcpu->arch.dtl.len;
1056 		spin_unlock(&vcpu->arch.vpa_update_lock);
1057 		break;
1058 	case KVM_REG_PPC_TB_OFFSET:
1059 		*val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1060 		break;
1061 	case KVM_REG_PPC_LPCR:
1062 	case KVM_REG_PPC_LPCR_64:
1063 		*val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1064 		break;
1065 	case KVM_REG_PPC_PPR:
1066 		*val = get_reg_val(id, vcpu->arch.ppr);
1067 		break;
1068 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1069 	case KVM_REG_PPC_TFHAR:
1070 		*val = get_reg_val(id, vcpu->arch.tfhar);
1071 		break;
1072 	case KVM_REG_PPC_TFIAR:
1073 		*val = get_reg_val(id, vcpu->arch.tfiar);
1074 		break;
1075 	case KVM_REG_PPC_TEXASR:
1076 		*val = get_reg_val(id, vcpu->arch.texasr);
1077 		break;
1078 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1079 		i = id - KVM_REG_PPC_TM_GPR0;
1080 		*val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1081 		break;
1082 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1083 	{
1084 		int j;
1085 		i = id - KVM_REG_PPC_TM_VSR0;
1086 		if (i < 32)
1087 			for (j = 0; j < TS_FPRWIDTH; j++)
1088 				val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1089 		else {
1090 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
1091 				val->vval = vcpu->arch.vr_tm.vr[i-32];
1092 			else
1093 				r = -ENXIO;
1094 		}
1095 		break;
1096 	}
1097 	case KVM_REG_PPC_TM_CR:
1098 		*val = get_reg_val(id, vcpu->arch.cr_tm);
1099 		break;
1100 	case KVM_REG_PPC_TM_LR:
1101 		*val = get_reg_val(id, vcpu->arch.lr_tm);
1102 		break;
1103 	case KVM_REG_PPC_TM_CTR:
1104 		*val = get_reg_val(id, vcpu->arch.ctr_tm);
1105 		break;
1106 	case KVM_REG_PPC_TM_FPSCR:
1107 		*val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1108 		break;
1109 	case KVM_REG_PPC_TM_AMR:
1110 		*val = get_reg_val(id, vcpu->arch.amr_tm);
1111 		break;
1112 	case KVM_REG_PPC_TM_PPR:
1113 		*val = get_reg_val(id, vcpu->arch.ppr_tm);
1114 		break;
1115 	case KVM_REG_PPC_TM_VRSAVE:
1116 		*val = get_reg_val(id, vcpu->arch.vrsave_tm);
1117 		break;
1118 	case KVM_REG_PPC_TM_VSCR:
1119 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
1120 			*val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1121 		else
1122 			r = -ENXIO;
1123 		break;
1124 	case KVM_REG_PPC_TM_DSCR:
1125 		*val = get_reg_val(id, vcpu->arch.dscr_tm);
1126 		break;
1127 	case KVM_REG_PPC_TM_TAR:
1128 		*val = get_reg_val(id, vcpu->arch.tar_tm);
1129 		break;
1130 #endif
1131 	case KVM_REG_PPC_ARCH_COMPAT:
1132 		*val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1133 		break;
1134 	default:
1135 		r = -EINVAL;
1136 		break;
1137 	}
1138 
1139 	return r;
1140 }
1141 
1142 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1143 				 union kvmppc_one_reg *val)
1144 {
1145 	int r = 0;
1146 	long int i;
1147 	unsigned long addr, len;
1148 
1149 	switch (id) {
1150 	case KVM_REG_PPC_HIOR:
1151 		/* Only allow this to be set to zero */
1152 		if (set_reg_val(id, *val))
1153 			r = -EINVAL;
1154 		break;
1155 	case KVM_REG_PPC_DABR:
1156 		vcpu->arch.dabr = set_reg_val(id, *val);
1157 		break;
1158 	case KVM_REG_PPC_DABRX:
1159 		vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1160 		break;
1161 	case KVM_REG_PPC_DSCR:
1162 		vcpu->arch.dscr = set_reg_val(id, *val);
1163 		break;
1164 	case KVM_REG_PPC_PURR:
1165 		vcpu->arch.purr = set_reg_val(id, *val);
1166 		break;
1167 	case KVM_REG_PPC_SPURR:
1168 		vcpu->arch.spurr = set_reg_val(id, *val);
1169 		break;
1170 	case KVM_REG_PPC_AMR:
1171 		vcpu->arch.amr = set_reg_val(id, *val);
1172 		break;
1173 	case KVM_REG_PPC_UAMOR:
1174 		vcpu->arch.uamor = set_reg_val(id, *val);
1175 		break;
1176 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1177 		i = id - KVM_REG_PPC_MMCR0;
1178 		vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1179 		break;
1180 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1181 		i = id - KVM_REG_PPC_PMC1;
1182 		vcpu->arch.pmc[i] = set_reg_val(id, *val);
1183 		break;
1184 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1185 		i = id - KVM_REG_PPC_SPMC1;
1186 		vcpu->arch.spmc[i] = set_reg_val(id, *val);
1187 		break;
1188 	case KVM_REG_PPC_SIAR:
1189 		vcpu->arch.siar = set_reg_val(id, *val);
1190 		break;
1191 	case KVM_REG_PPC_SDAR:
1192 		vcpu->arch.sdar = set_reg_val(id, *val);
1193 		break;
1194 	case KVM_REG_PPC_SIER:
1195 		vcpu->arch.sier = set_reg_val(id, *val);
1196 		break;
1197 	case KVM_REG_PPC_IAMR:
1198 		vcpu->arch.iamr = set_reg_val(id, *val);
1199 		break;
1200 	case KVM_REG_PPC_PSPB:
1201 		vcpu->arch.pspb = set_reg_val(id, *val);
1202 		break;
1203 	case KVM_REG_PPC_DPDES:
1204 		vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1205 		break;
1206 	case KVM_REG_PPC_DAWR:
1207 		vcpu->arch.dawr = set_reg_val(id, *val);
1208 		break;
1209 	case KVM_REG_PPC_DAWRX:
1210 		vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1211 		break;
1212 	case KVM_REG_PPC_CIABR:
1213 		vcpu->arch.ciabr = set_reg_val(id, *val);
1214 		/* Don't allow setting breakpoints in hypervisor code */
1215 		if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1216 			vcpu->arch.ciabr &= ~CIABR_PRIV;	/* disable */
1217 		break;
1218 	case KVM_REG_PPC_CSIGR:
1219 		vcpu->arch.csigr = set_reg_val(id, *val);
1220 		break;
1221 	case KVM_REG_PPC_TACR:
1222 		vcpu->arch.tacr = set_reg_val(id, *val);
1223 		break;
1224 	case KVM_REG_PPC_TCSCR:
1225 		vcpu->arch.tcscr = set_reg_val(id, *val);
1226 		break;
1227 	case KVM_REG_PPC_PID:
1228 		vcpu->arch.pid = set_reg_val(id, *val);
1229 		break;
1230 	case KVM_REG_PPC_ACOP:
1231 		vcpu->arch.acop = set_reg_val(id, *val);
1232 		break;
1233 	case KVM_REG_PPC_WORT:
1234 		vcpu->arch.wort = set_reg_val(id, *val);
1235 		break;
1236 	case KVM_REG_PPC_VPA_ADDR:
1237 		addr = set_reg_val(id, *val);
1238 		r = -EINVAL;
1239 		if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1240 			      vcpu->arch.dtl.next_gpa))
1241 			break;
1242 		r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1243 		break;
1244 	case KVM_REG_PPC_VPA_SLB:
1245 		addr = val->vpaval.addr;
1246 		len = val->vpaval.length;
1247 		r = -EINVAL;
1248 		if (addr && !vcpu->arch.vpa.next_gpa)
1249 			break;
1250 		r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1251 		break;
1252 	case KVM_REG_PPC_VPA_DTL:
1253 		addr = val->vpaval.addr;
1254 		len = val->vpaval.length;
1255 		r = -EINVAL;
1256 		if (addr && (len < sizeof(struct dtl_entry) ||
1257 			     !vcpu->arch.vpa.next_gpa))
1258 			break;
1259 		len -= len % sizeof(struct dtl_entry);
1260 		r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1261 		break;
1262 	case KVM_REG_PPC_TB_OFFSET:
1263 		/* round up to multiple of 2^24 */
1264 		vcpu->arch.vcore->tb_offset =
1265 			ALIGN(set_reg_val(id, *val), 1UL << 24);
1266 		break;
1267 	case KVM_REG_PPC_LPCR:
1268 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1269 		break;
1270 	case KVM_REG_PPC_LPCR_64:
1271 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1272 		break;
1273 	case KVM_REG_PPC_PPR:
1274 		vcpu->arch.ppr = set_reg_val(id, *val);
1275 		break;
1276 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1277 	case KVM_REG_PPC_TFHAR:
1278 		vcpu->arch.tfhar = set_reg_val(id, *val);
1279 		break;
1280 	case KVM_REG_PPC_TFIAR:
1281 		vcpu->arch.tfiar = set_reg_val(id, *val);
1282 		break;
1283 	case KVM_REG_PPC_TEXASR:
1284 		vcpu->arch.texasr = set_reg_val(id, *val);
1285 		break;
1286 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1287 		i = id - KVM_REG_PPC_TM_GPR0;
1288 		vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1289 		break;
1290 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1291 	{
1292 		int j;
1293 		i = id - KVM_REG_PPC_TM_VSR0;
1294 		if (i < 32)
1295 			for (j = 0; j < TS_FPRWIDTH; j++)
1296 				vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
1297 		else
1298 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
1299 				vcpu->arch.vr_tm.vr[i-32] = val->vval;
1300 			else
1301 				r = -ENXIO;
1302 		break;
1303 	}
1304 	case KVM_REG_PPC_TM_CR:
1305 		vcpu->arch.cr_tm = set_reg_val(id, *val);
1306 		break;
1307 	case KVM_REG_PPC_TM_LR:
1308 		vcpu->arch.lr_tm = set_reg_val(id, *val);
1309 		break;
1310 	case KVM_REG_PPC_TM_CTR:
1311 		vcpu->arch.ctr_tm = set_reg_val(id, *val);
1312 		break;
1313 	case KVM_REG_PPC_TM_FPSCR:
1314 		vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
1315 		break;
1316 	case KVM_REG_PPC_TM_AMR:
1317 		vcpu->arch.amr_tm = set_reg_val(id, *val);
1318 		break;
1319 	case KVM_REG_PPC_TM_PPR:
1320 		vcpu->arch.ppr_tm = set_reg_val(id, *val);
1321 		break;
1322 	case KVM_REG_PPC_TM_VRSAVE:
1323 		vcpu->arch.vrsave_tm = set_reg_val(id, *val);
1324 		break;
1325 	case KVM_REG_PPC_TM_VSCR:
1326 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
1327 			vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
1328 		else
1329 			r = - ENXIO;
1330 		break;
1331 	case KVM_REG_PPC_TM_DSCR:
1332 		vcpu->arch.dscr_tm = set_reg_val(id, *val);
1333 		break;
1334 	case KVM_REG_PPC_TM_TAR:
1335 		vcpu->arch.tar_tm = set_reg_val(id, *val);
1336 		break;
1337 #endif
1338 	case KVM_REG_PPC_ARCH_COMPAT:
1339 		r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
1340 		break;
1341 	default:
1342 		r = -EINVAL;
1343 		break;
1344 	}
1345 
1346 	return r;
1347 }
1348 
1349 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
1350 {
1351 	struct kvmppc_vcore *vcore;
1352 
1353 	vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
1354 
1355 	if (vcore == NULL)
1356 		return NULL;
1357 
1358 	INIT_LIST_HEAD(&vcore->runnable_threads);
1359 	spin_lock_init(&vcore->lock);
1360 	init_waitqueue_head(&vcore->wq);
1361 	vcore->preempt_tb = TB_NIL;
1362 	vcore->lpcr = kvm->arch.lpcr;
1363 	vcore->first_vcpuid = core * threads_per_subcore;
1364 	vcore->kvm = kvm;
1365 
1366 	vcore->mpp_buffer_is_valid = false;
1367 
1368 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
1369 		vcore->mpp_buffer = (void *)__get_free_pages(
1370 			GFP_KERNEL|__GFP_ZERO,
1371 			MPP_BUFFER_ORDER);
1372 
1373 	return vcore;
1374 }
1375 
1376 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
1377 						   unsigned int id)
1378 {
1379 	struct kvm_vcpu *vcpu;
1380 	int err = -EINVAL;
1381 	int core;
1382 	struct kvmppc_vcore *vcore;
1383 
1384 	core = id / threads_per_subcore;
1385 	if (core >= KVM_MAX_VCORES)
1386 		goto out;
1387 
1388 	err = -ENOMEM;
1389 	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1390 	if (!vcpu)
1391 		goto out;
1392 
1393 	err = kvm_vcpu_init(vcpu, kvm, id);
1394 	if (err)
1395 		goto free_vcpu;
1396 
1397 	vcpu->arch.shared = &vcpu->arch.shregs;
1398 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1399 	/*
1400 	 * The shared struct is never shared on HV,
1401 	 * so we can always use host endianness
1402 	 */
1403 #ifdef __BIG_ENDIAN__
1404 	vcpu->arch.shared_big_endian = true;
1405 #else
1406 	vcpu->arch.shared_big_endian = false;
1407 #endif
1408 #endif
1409 	vcpu->arch.mmcr[0] = MMCR0_FC;
1410 	vcpu->arch.ctrl = CTRL_RUNLATCH;
1411 	/* default to host PVR, since we can't spoof it */
1412 	kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
1413 	spin_lock_init(&vcpu->arch.vpa_update_lock);
1414 	spin_lock_init(&vcpu->arch.tbacct_lock);
1415 	vcpu->arch.busy_preempt = TB_NIL;
1416 	vcpu->arch.intr_msr = MSR_SF | MSR_ME;
1417 
1418 	kvmppc_mmu_book3s_hv_init(vcpu);
1419 
1420 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
1421 
1422 	init_waitqueue_head(&vcpu->arch.cpu_run);
1423 
1424 	mutex_lock(&kvm->lock);
1425 	vcore = kvm->arch.vcores[core];
1426 	if (!vcore) {
1427 		vcore = kvmppc_vcore_create(kvm, core);
1428 		kvm->arch.vcores[core] = vcore;
1429 		kvm->arch.online_vcores++;
1430 	}
1431 	mutex_unlock(&kvm->lock);
1432 
1433 	if (!vcore)
1434 		goto free_vcpu;
1435 
1436 	spin_lock(&vcore->lock);
1437 	++vcore->num_threads;
1438 	spin_unlock(&vcore->lock);
1439 	vcpu->arch.vcore = vcore;
1440 	vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
1441 
1442 	vcpu->arch.cpu_type = KVM_CPU_3S_64;
1443 	kvmppc_sanity_check(vcpu);
1444 
1445 	return vcpu;
1446 
1447 free_vcpu:
1448 	kmem_cache_free(kvm_vcpu_cache, vcpu);
1449 out:
1450 	return ERR_PTR(err);
1451 }
1452 
1453 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
1454 {
1455 	if (vpa->pinned_addr)
1456 		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
1457 					vpa->dirty);
1458 }
1459 
1460 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
1461 {
1462 	spin_lock(&vcpu->arch.vpa_update_lock);
1463 	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
1464 	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
1465 	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
1466 	spin_unlock(&vcpu->arch.vpa_update_lock);
1467 	kvm_vcpu_uninit(vcpu);
1468 	kmem_cache_free(kvm_vcpu_cache, vcpu);
1469 }
1470 
1471 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
1472 {
1473 	/* Indicate we want to get back into the guest */
1474 	return 1;
1475 }
1476 
1477 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
1478 {
1479 	unsigned long dec_nsec, now;
1480 
1481 	now = get_tb();
1482 	if (now > vcpu->arch.dec_expires) {
1483 		/* decrementer has already gone negative */
1484 		kvmppc_core_queue_dec(vcpu);
1485 		kvmppc_core_prepare_to_enter(vcpu);
1486 		return;
1487 	}
1488 	dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
1489 		   / tb_ticks_per_sec;
1490 	hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
1491 		      HRTIMER_MODE_REL);
1492 	vcpu->arch.timer_running = 1;
1493 }
1494 
1495 static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
1496 {
1497 	vcpu->arch.ceded = 0;
1498 	if (vcpu->arch.timer_running) {
1499 		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
1500 		vcpu->arch.timer_running = 0;
1501 	}
1502 }
1503 
1504 extern void __kvmppc_vcore_entry(void);
1505 
1506 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
1507 				   struct kvm_vcpu *vcpu)
1508 {
1509 	u64 now;
1510 
1511 	if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
1512 		return;
1513 	spin_lock_irq(&vcpu->arch.tbacct_lock);
1514 	now = mftb();
1515 	vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
1516 		vcpu->arch.stolen_logged;
1517 	vcpu->arch.busy_preempt = now;
1518 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
1519 	spin_unlock_irq(&vcpu->arch.tbacct_lock);
1520 	--vc->n_runnable;
1521 	list_del(&vcpu->arch.run_list);
1522 }
1523 
1524 static int kvmppc_grab_hwthread(int cpu)
1525 {
1526 	struct paca_struct *tpaca;
1527 	long timeout = 10000;
1528 
1529 	tpaca = &paca[cpu];
1530 
1531 	/* Ensure the thread won't go into the kernel if it wakes */
1532 	tpaca->kvm_hstate.hwthread_req = 1;
1533 	tpaca->kvm_hstate.kvm_vcpu = NULL;
1534 
1535 	/*
1536 	 * If the thread is already executing in the kernel (e.g. handling
1537 	 * a stray interrupt), wait for it to get back to nap mode.
1538 	 * The smp_mb() is to ensure that our setting of hwthread_req
1539 	 * is visible before we look at hwthread_state, so if this
1540 	 * races with the code at system_reset_pSeries and the thread
1541 	 * misses our setting of hwthread_req, we are sure to see its
1542 	 * setting of hwthread_state, and vice versa.
1543 	 */
1544 	smp_mb();
1545 	while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
1546 		if (--timeout <= 0) {
1547 			pr_err("KVM: couldn't grab cpu %d\n", cpu);
1548 			return -EBUSY;
1549 		}
1550 		udelay(1);
1551 	}
1552 	return 0;
1553 }
1554 
1555 static void kvmppc_release_hwthread(int cpu)
1556 {
1557 	struct paca_struct *tpaca;
1558 
1559 	tpaca = &paca[cpu];
1560 	tpaca->kvm_hstate.hwthread_req = 0;
1561 	tpaca->kvm_hstate.kvm_vcpu = NULL;
1562 }
1563 
1564 static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
1565 {
1566 	int cpu;
1567 	struct paca_struct *tpaca;
1568 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
1569 
1570 	if (vcpu->arch.timer_running) {
1571 		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
1572 		vcpu->arch.timer_running = 0;
1573 	}
1574 	cpu = vc->pcpu + vcpu->arch.ptid;
1575 	tpaca = &paca[cpu];
1576 	tpaca->kvm_hstate.kvm_vcpu = vcpu;
1577 	tpaca->kvm_hstate.kvm_vcore = vc;
1578 	tpaca->kvm_hstate.ptid = vcpu->arch.ptid;
1579 	vcpu->cpu = vc->pcpu;
1580 	smp_wmb();
1581 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
1582 	if (cpu != smp_processor_id()) {
1583 		xics_wake_cpu(cpu);
1584 		if (vcpu->arch.ptid)
1585 			++vc->n_woken;
1586 	}
1587 #endif
1588 }
1589 
1590 static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
1591 {
1592 	int i;
1593 
1594 	HMT_low();
1595 	i = 0;
1596 	while (vc->nap_count < vc->n_woken) {
1597 		if (++i >= 1000000) {
1598 			pr_err("kvmppc_wait_for_nap timeout %d %d\n",
1599 			       vc->nap_count, vc->n_woken);
1600 			break;
1601 		}
1602 		cpu_relax();
1603 	}
1604 	HMT_medium();
1605 }
1606 
1607 /*
1608  * Check that we are on thread 0 and that any other threads in
1609  * this core are off-line.  Then grab the threads so they can't
1610  * enter the kernel.
1611  */
1612 static int on_primary_thread(void)
1613 {
1614 	int cpu = smp_processor_id();
1615 	int thr;
1616 
1617 	/* Are we on a primary subcore? */
1618 	if (cpu_thread_in_subcore(cpu))
1619 		return 0;
1620 
1621 	thr = 0;
1622 	while (++thr < threads_per_subcore)
1623 		if (cpu_online(cpu + thr))
1624 			return 0;
1625 
1626 	/* Grab all hw threads so they can't go into the kernel */
1627 	for (thr = 1; thr < threads_per_subcore; ++thr) {
1628 		if (kvmppc_grab_hwthread(cpu + thr)) {
1629 			/* Couldn't grab one; let the others go */
1630 			do {
1631 				kvmppc_release_hwthread(cpu + thr);
1632 			} while (--thr > 0);
1633 			return 0;
1634 		}
1635 	}
1636 	return 1;
1637 }
1638 
1639 static void kvmppc_start_saving_l2_cache(struct kvmppc_vcore *vc)
1640 {
1641 	phys_addr_t phy_addr, mpp_addr;
1642 
1643 	phy_addr = (phys_addr_t)virt_to_phys(vc->mpp_buffer);
1644 	mpp_addr = phy_addr & PPC_MPPE_ADDRESS_MASK;
1645 
1646 	mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_ABORT);
1647 	logmpp(mpp_addr | PPC_LOGMPP_LOG_L2);
1648 
1649 	vc->mpp_buffer_is_valid = true;
1650 }
1651 
1652 static void kvmppc_start_restoring_l2_cache(const struct kvmppc_vcore *vc)
1653 {
1654 	phys_addr_t phy_addr, mpp_addr;
1655 
1656 	phy_addr = virt_to_phys(vc->mpp_buffer);
1657 	mpp_addr = phy_addr & PPC_MPPE_ADDRESS_MASK;
1658 
1659 	/* We must abort any in-progress save operations to ensure
1660 	 * the table is valid so that prefetch engine knows when to
1661 	 * stop prefetching. */
1662 	logmpp(mpp_addr | PPC_LOGMPP_LOG_ABORT);
1663 	mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_WHOLE_TABLE);
1664 }
1665 
1666 /*
1667  * Run a set of guest threads on a physical core.
1668  * Called with vc->lock held.
1669  */
1670 static void kvmppc_run_core(struct kvmppc_vcore *vc)
1671 {
1672 	struct kvm_vcpu *vcpu, *vnext;
1673 	long ret;
1674 	u64 now;
1675 	int i, need_vpa_update;
1676 	int srcu_idx;
1677 	struct kvm_vcpu *vcpus_to_update[threads_per_core];
1678 
1679 	/* don't start if any threads have a signal pending */
1680 	need_vpa_update = 0;
1681 	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1682 		if (signal_pending(vcpu->arch.run_task))
1683 			return;
1684 		if (vcpu->arch.vpa.update_pending ||
1685 		    vcpu->arch.slb_shadow.update_pending ||
1686 		    vcpu->arch.dtl.update_pending)
1687 			vcpus_to_update[need_vpa_update++] = vcpu;
1688 	}
1689 
1690 	/*
1691 	 * Initialize *vc, in particular vc->vcore_state, so we can
1692 	 * drop the vcore lock if necessary.
1693 	 */
1694 	vc->n_woken = 0;
1695 	vc->nap_count = 0;
1696 	vc->entry_exit_count = 0;
1697 	vc->vcore_state = VCORE_STARTING;
1698 	vc->in_guest = 0;
1699 	vc->napping_threads = 0;
1700 
1701 	/*
1702 	 * Updating any of the vpas requires calling kvmppc_pin_guest_page,
1703 	 * which can't be called with any spinlocks held.
1704 	 */
1705 	if (need_vpa_update) {
1706 		spin_unlock(&vc->lock);
1707 		for (i = 0; i < need_vpa_update; ++i)
1708 			kvmppc_update_vpas(vcpus_to_update[i]);
1709 		spin_lock(&vc->lock);
1710 	}
1711 
1712 	/*
1713 	 * Make sure we are running on primary threads, and that secondary
1714 	 * threads are offline.  Also check if the number of threads in this
1715 	 * guest are greater than the current system threads per guest.
1716 	 */
1717 	if ((threads_per_core > 1) &&
1718 	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
1719 		list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1720 			vcpu->arch.ret = -EBUSY;
1721 		goto out;
1722 	}
1723 
1724 
1725 	vc->pcpu = smp_processor_id();
1726 	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1727 		kvmppc_start_thread(vcpu);
1728 		kvmppc_create_dtl_entry(vcpu, vc);
1729 	}
1730 
1731 	/* Set this explicitly in case thread 0 doesn't have a vcpu */
1732 	get_paca()->kvm_hstate.kvm_vcore = vc;
1733 	get_paca()->kvm_hstate.ptid = 0;
1734 
1735 	vc->vcore_state = VCORE_RUNNING;
1736 	preempt_disable();
1737 	spin_unlock(&vc->lock);
1738 
1739 	kvm_guest_enter();
1740 
1741 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
1742 
1743 	if (vc->mpp_buffer_is_valid)
1744 		kvmppc_start_restoring_l2_cache(vc);
1745 
1746 	__kvmppc_vcore_entry();
1747 
1748 	spin_lock(&vc->lock);
1749 
1750 	if (vc->mpp_buffer)
1751 		kvmppc_start_saving_l2_cache(vc);
1752 
1753 	/* disable sending of IPIs on virtual external irqs */
1754 	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1755 		vcpu->cpu = -1;
1756 	/* wait for secondary threads to finish writing their state to memory */
1757 	if (vc->nap_count < vc->n_woken)
1758 		kvmppc_wait_for_nap(vc);
1759 	for (i = 0; i < threads_per_subcore; ++i)
1760 		kvmppc_release_hwthread(vc->pcpu + i);
1761 	/* prevent other vcpu threads from doing kvmppc_start_thread() now */
1762 	vc->vcore_state = VCORE_EXITING;
1763 	spin_unlock(&vc->lock);
1764 
1765 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
1766 
1767 	/* make sure updates to secondary vcpu structs are visible now */
1768 	smp_mb();
1769 	kvm_guest_exit();
1770 
1771 	preempt_enable();
1772 	cond_resched();
1773 
1774 	spin_lock(&vc->lock);
1775 	now = get_tb();
1776 	list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1777 		/* cancel pending dec exception if dec is positive */
1778 		if (now < vcpu->arch.dec_expires &&
1779 		    kvmppc_core_pending_dec(vcpu))
1780 			kvmppc_core_dequeue_dec(vcpu);
1781 
1782 		ret = RESUME_GUEST;
1783 		if (vcpu->arch.trap)
1784 			ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
1785 						    vcpu->arch.run_task);
1786 
1787 		vcpu->arch.ret = ret;
1788 		vcpu->arch.trap = 0;
1789 
1790 		if (vcpu->arch.ceded) {
1791 			if (!is_kvmppc_resume_guest(ret))
1792 				kvmppc_end_cede(vcpu);
1793 			else
1794 				kvmppc_set_timer(vcpu);
1795 		}
1796 	}
1797 
1798  out:
1799 	vc->vcore_state = VCORE_INACTIVE;
1800 	list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1801 				 arch.run_list) {
1802 		if (!is_kvmppc_resume_guest(vcpu->arch.ret)) {
1803 			kvmppc_remove_runnable(vc, vcpu);
1804 			wake_up(&vcpu->arch.cpu_run);
1805 		}
1806 	}
1807 }
1808 
1809 /*
1810  * Wait for some other vcpu thread to execute us, and
1811  * wake us up when we need to handle something in the host.
1812  */
1813 static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state)
1814 {
1815 	DEFINE_WAIT(wait);
1816 
1817 	prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
1818 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
1819 		schedule();
1820 	finish_wait(&vcpu->arch.cpu_run, &wait);
1821 }
1822 
1823 /*
1824  * All the vcpus in this vcore are idle, so wait for a decrementer
1825  * or external interrupt to one of the vcpus.  vc->lock is held.
1826  */
1827 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
1828 {
1829 	DEFINE_WAIT(wait);
1830 
1831 	prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
1832 	vc->vcore_state = VCORE_SLEEPING;
1833 	spin_unlock(&vc->lock);
1834 	schedule();
1835 	finish_wait(&vc->wq, &wait);
1836 	spin_lock(&vc->lock);
1837 	vc->vcore_state = VCORE_INACTIVE;
1838 }
1839 
1840 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1841 {
1842 	int n_ceded;
1843 	struct kvmppc_vcore *vc;
1844 	struct kvm_vcpu *v, *vn;
1845 
1846 	kvm_run->exit_reason = 0;
1847 	vcpu->arch.ret = RESUME_GUEST;
1848 	vcpu->arch.trap = 0;
1849 	kvmppc_update_vpas(vcpu);
1850 
1851 	/*
1852 	 * Synchronize with other threads in this virtual core
1853 	 */
1854 	vc = vcpu->arch.vcore;
1855 	spin_lock(&vc->lock);
1856 	vcpu->arch.ceded = 0;
1857 	vcpu->arch.run_task = current;
1858 	vcpu->arch.kvm_run = kvm_run;
1859 	vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
1860 	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
1861 	vcpu->arch.busy_preempt = TB_NIL;
1862 	list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
1863 	++vc->n_runnable;
1864 
1865 	/*
1866 	 * This happens the first time this is called for a vcpu.
1867 	 * If the vcore is already running, we may be able to start
1868 	 * this thread straight away and have it join in.
1869 	 */
1870 	if (!signal_pending(current)) {
1871 		if (vc->vcore_state == VCORE_RUNNING &&
1872 		    VCORE_EXIT_COUNT(vc) == 0) {
1873 			kvmppc_create_dtl_entry(vcpu, vc);
1874 			kvmppc_start_thread(vcpu);
1875 		} else if (vc->vcore_state == VCORE_SLEEPING) {
1876 			wake_up(&vc->wq);
1877 		}
1878 
1879 	}
1880 
1881 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1882 	       !signal_pending(current)) {
1883 		if (vc->vcore_state != VCORE_INACTIVE) {
1884 			spin_unlock(&vc->lock);
1885 			kvmppc_wait_for_exec(vcpu, TASK_INTERRUPTIBLE);
1886 			spin_lock(&vc->lock);
1887 			continue;
1888 		}
1889 		list_for_each_entry_safe(v, vn, &vc->runnable_threads,
1890 					 arch.run_list) {
1891 			kvmppc_core_prepare_to_enter(v);
1892 			if (signal_pending(v->arch.run_task)) {
1893 				kvmppc_remove_runnable(vc, v);
1894 				v->stat.signal_exits++;
1895 				v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
1896 				v->arch.ret = -EINTR;
1897 				wake_up(&v->arch.cpu_run);
1898 			}
1899 		}
1900 		if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
1901 			break;
1902 		vc->runner = vcpu;
1903 		n_ceded = 0;
1904 		list_for_each_entry(v, &vc->runnable_threads, arch.run_list) {
1905 			if (!v->arch.pending_exceptions)
1906 				n_ceded += v->arch.ceded;
1907 			else
1908 				v->arch.ceded = 0;
1909 		}
1910 		if (n_ceded == vc->n_runnable)
1911 			kvmppc_vcore_blocked(vc);
1912 		else
1913 			kvmppc_run_core(vc);
1914 		vc->runner = NULL;
1915 	}
1916 
1917 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1918 	       (vc->vcore_state == VCORE_RUNNING ||
1919 		vc->vcore_state == VCORE_EXITING)) {
1920 		spin_unlock(&vc->lock);
1921 		kvmppc_wait_for_exec(vcpu, TASK_UNINTERRUPTIBLE);
1922 		spin_lock(&vc->lock);
1923 	}
1924 
1925 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
1926 		kvmppc_remove_runnable(vc, vcpu);
1927 		vcpu->stat.signal_exits++;
1928 		kvm_run->exit_reason = KVM_EXIT_INTR;
1929 		vcpu->arch.ret = -EINTR;
1930 	}
1931 
1932 	if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
1933 		/* Wake up some vcpu to run the core */
1934 		v = list_first_entry(&vc->runnable_threads,
1935 				     struct kvm_vcpu, arch.run_list);
1936 		wake_up(&v->arch.cpu_run);
1937 	}
1938 
1939 	spin_unlock(&vc->lock);
1940 	return vcpu->arch.ret;
1941 }
1942 
1943 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
1944 {
1945 	int r;
1946 	int srcu_idx;
1947 
1948 	if (!vcpu->arch.sane) {
1949 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1950 		return -EINVAL;
1951 	}
1952 
1953 	kvmppc_core_prepare_to_enter(vcpu);
1954 
1955 	/* No need to go into the guest when all we'll do is come back out */
1956 	if (signal_pending(current)) {
1957 		run->exit_reason = KVM_EXIT_INTR;
1958 		return -EINTR;
1959 	}
1960 
1961 	atomic_inc(&vcpu->kvm->arch.vcpus_running);
1962 	/* Order vcpus_running vs. rma_setup_done, see kvmppc_alloc_reset_hpt */
1963 	smp_mb();
1964 
1965 	/* On the first time here, set up HTAB and VRMA or RMA */
1966 	if (!vcpu->kvm->arch.rma_setup_done) {
1967 		r = kvmppc_hv_setup_htab_rma(vcpu);
1968 		if (r)
1969 			goto out;
1970 	}
1971 
1972 	flush_fp_to_thread(current);
1973 	flush_altivec_to_thread(current);
1974 	flush_vsx_to_thread(current);
1975 	vcpu->arch.wqp = &vcpu->arch.vcore->wq;
1976 	vcpu->arch.pgdir = current->mm->pgd;
1977 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
1978 
1979 	do {
1980 		r = kvmppc_run_vcpu(run, vcpu);
1981 
1982 		if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
1983 		    !(vcpu->arch.shregs.msr & MSR_PR)) {
1984 			r = kvmppc_pseries_do_hcall(vcpu);
1985 			kvmppc_core_prepare_to_enter(vcpu);
1986 		} else if (r == RESUME_PAGE_FAULT) {
1987 			srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1988 			r = kvmppc_book3s_hv_page_fault(run, vcpu,
1989 				vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
1990 			srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1991 		}
1992 	} while (is_kvmppc_resume_guest(r));
1993 
1994  out:
1995 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
1996 	atomic_dec(&vcpu->kvm->arch.vcpus_running);
1997 	return r;
1998 }
1999 
2000 
2001 /* Work out RMLS (real mode limit selector) field value for a given RMA size.
2002    Assumes POWER7 or PPC970. */
2003 static inline int lpcr_rmls(unsigned long rma_size)
2004 {
2005 	switch (rma_size) {
2006 	case 32ul << 20:	/* 32 MB */
2007 		if (cpu_has_feature(CPU_FTR_ARCH_206))
2008 			return 8;	/* only supported on POWER7 */
2009 		return -1;
2010 	case 64ul << 20:	/* 64 MB */
2011 		return 3;
2012 	case 128ul << 20:	/* 128 MB */
2013 		return 7;
2014 	case 256ul << 20:	/* 256 MB */
2015 		return 4;
2016 	case 1ul << 30:		/* 1 GB */
2017 		return 2;
2018 	case 16ul << 30:	/* 16 GB */
2019 		return 1;
2020 	case 256ul << 30:	/* 256 GB */
2021 		return 0;
2022 	default:
2023 		return -1;
2024 	}
2025 }
2026 
2027 static int kvm_rma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2028 {
2029 	struct page *page;
2030 	struct kvm_rma_info *ri = vma->vm_file->private_data;
2031 
2032 	if (vmf->pgoff >= kvm_rma_pages)
2033 		return VM_FAULT_SIGBUS;
2034 
2035 	page = pfn_to_page(ri->base_pfn + vmf->pgoff);
2036 	get_page(page);
2037 	vmf->page = page;
2038 	return 0;
2039 }
2040 
2041 static const struct vm_operations_struct kvm_rma_vm_ops = {
2042 	.fault = kvm_rma_fault,
2043 };
2044 
2045 static int kvm_rma_mmap(struct file *file, struct vm_area_struct *vma)
2046 {
2047 	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
2048 	vma->vm_ops = &kvm_rma_vm_ops;
2049 	return 0;
2050 }
2051 
2052 static int kvm_rma_release(struct inode *inode, struct file *filp)
2053 {
2054 	struct kvm_rma_info *ri = filp->private_data;
2055 
2056 	kvm_release_rma(ri);
2057 	return 0;
2058 }
2059 
2060 static const struct file_operations kvm_rma_fops = {
2061 	.mmap           = kvm_rma_mmap,
2062 	.release	= kvm_rma_release,
2063 };
2064 
2065 static long kvm_vm_ioctl_allocate_rma(struct kvm *kvm,
2066 				      struct kvm_allocate_rma *ret)
2067 {
2068 	long fd;
2069 	struct kvm_rma_info *ri;
2070 	/*
2071 	 * Only do this on PPC970 in HV mode
2072 	 */
2073 	if (!cpu_has_feature(CPU_FTR_HVMODE) ||
2074 	    !cpu_has_feature(CPU_FTR_ARCH_201))
2075 		return -EINVAL;
2076 
2077 	if (!kvm_rma_pages)
2078 		return -EINVAL;
2079 
2080 	ri = kvm_alloc_rma();
2081 	if (!ri)
2082 		return -ENOMEM;
2083 
2084 	fd = anon_inode_getfd("kvm-rma", &kvm_rma_fops, ri, O_RDWR | O_CLOEXEC);
2085 	if (fd < 0)
2086 		kvm_release_rma(ri);
2087 
2088 	ret->rma_size = kvm_rma_pages << PAGE_SHIFT;
2089 	return fd;
2090 }
2091 
2092 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
2093 				     int linux_psize)
2094 {
2095 	struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];
2096 
2097 	if (!def->shift)
2098 		return;
2099 	(*sps)->page_shift = def->shift;
2100 	(*sps)->slb_enc = def->sllp;
2101 	(*sps)->enc[0].page_shift = def->shift;
2102 	(*sps)->enc[0].pte_enc = def->penc[linux_psize];
2103 	/*
2104 	 * Add 16MB MPSS support if host supports it
2105 	 */
2106 	if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) {
2107 		(*sps)->enc[1].page_shift = 24;
2108 		(*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M];
2109 	}
2110 	(*sps)++;
2111 }
2112 
2113 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
2114 					 struct kvm_ppc_smmu_info *info)
2115 {
2116 	struct kvm_ppc_one_seg_page_size *sps;
2117 
2118 	info->flags = KVM_PPC_PAGE_SIZES_REAL;
2119 	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
2120 		info->flags |= KVM_PPC_1T_SEGMENTS;
2121 	info->slb_size = mmu_slb_size;
2122 
2123 	/* We only support these sizes for now, and no muti-size segments */
2124 	sps = &info->sps[0];
2125 	kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K);
2126 	kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K);
2127 	kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M);
2128 
2129 	return 0;
2130 }
2131 
2132 /*
2133  * Get (and clear) the dirty memory log for a memory slot.
2134  */
2135 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
2136 					 struct kvm_dirty_log *log)
2137 {
2138 	struct kvm_memory_slot *memslot;
2139 	int r;
2140 	unsigned long n;
2141 
2142 	mutex_lock(&kvm->slots_lock);
2143 
2144 	r = -EINVAL;
2145 	if (log->slot >= KVM_USER_MEM_SLOTS)
2146 		goto out;
2147 
2148 	memslot = id_to_memslot(kvm->memslots, log->slot);
2149 	r = -ENOENT;
2150 	if (!memslot->dirty_bitmap)
2151 		goto out;
2152 
2153 	n = kvm_dirty_bitmap_bytes(memslot);
2154 	memset(memslot->dirty_bitmap, 0, n);
2155 
2156 	r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);
2157 	if (r)
2158 		goto out;
2159 
2160 	r = -EFAULT;
2161 	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
2162 		goto out;
2163 
2164 	r = 0;
2165 out:
2166 	mutex_unlock(&kvm->slots_lock);
2167 	return r;
2168 }
2169 
2170 static void unpin_slot(struct kvm_memory_slot *memslot)
2171 {
2172 	unsigned long *physp;
2173 	unsigned long j, npages, pfn;
2174 	struct page *page;
2175 
2176 	physp = memslot->arch.slot_phys;
2177 	npages = memslot->npages;
2178 	if (!physp)
2179 		return;
2180 	for (j = 0; j < npages; j++) {
2181 		if (!(physp[j] & KVMPPC_GOT_PAGE))
2182 			continue;
2183 		pfn = physp[j] >> PAGE_SHIFT;
2184 		page = pfn_to_page(pfn);
2185 		SetPageDirty(page);
2186 		put_page(page);
2187 	}
2188 }
2189 
2190 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
2191 					struct kvm_memory_slot *dont)
2192 {
2193 	if (!dont || free->arch.rmap != dont->arch.rmap) {
2194 		vfree(free->arch.rmap);
2195 		free->arch.rmap = NULL;
2196 	}
2197 	if (!dont || free->arch.slot_phys != dont->arch.slot_phys) {
2198 		unpin_slot(free);
2199 		vfree(free->arch.slot_phys);
2200 		free->arch.slot_phys = NULL;
2201 	}
2202 }
2203 
2204 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
2205 					 unsigned long npages)
2206 {
2207 	slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
2208 	if (!slot->arch.rmap)
2209 		return -ENOMEM;
2210 	slot->arch.slot_phys = NULL;
2211 
2212 	return 0;
2213 }
2214 
2215 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
2216 					struct kvm_memory_slot *memslot,
2217 					struct kvm_userspace_memory_region *mem)
2218 {
2219 	unsigned long *phys;
2220 
2221 	/* Allocate a slot_phys array if needed */
2222 	phys = memslot->arch.slot_phys;
2223 	if (!kvm->arch.using_mmu_notifiers && !phys && memslot->npages) {
2224 		phys = vzalloc(memslot->npages * sizeof(unsigned long));
2225 		if (!phys)
2226 			return -ENOMEM;
2227 		memslot->arch.slot_phys = phys;
2228 	}
2229 
2230 	return 0;
2231 }
2232 
2233 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
2234 				struct kvm_userspace_memory_region *mem,
2235 				const struct kvm_memory_slot *old)
2236 {
2237 	unsigned long npages = mem->memory_size >> PAGE_SHIFT;
2238 	struct kvm_memory_slot *memslot;
2239 
2240 	if (npages && old->npages) {
2241 		/*
2242 		 * If modifying a memslot, reset all the rmap dirty bits.
2243 		 * If this is a new memslot, we don't need to do anything
2244 		 * since the rmap array starts out as all zeroes,
2245 		 * i.e. no pages are dirty.
2246 		 */
2247 		memslot = id_to_memslot(kvm->memslots, mem->slot);
2248 		kvmppc_hv_get_dirty_log(kvm, memslot, NULL);
2249 	}
2250 }
2251 
2252 /*
2253  * Update LPCR values in kvm->arch and in vcores.
2254  * Caller must hold kvm->lock.
2255  */
2256 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
2257 {
2258 	long int i;
2259 	u32 cores_done = 0;
2260 
2261 	if ((kvm->arch.lpcr & mask) == lpcr)
2262 		return;
2263 
2264 	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
2265 
2266 	for (i = 0; i < KVM_MAX_VCORES; ++i) {
2267 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];
2268 		if (!vc)
2269 			continue;
2270 		spin_lock(&vc->lock);
2271 		vc->lpcr = (vc->lpcr & ~mask) | lpcr;
2272 		spin_unlock(&vc->lock);
2273 		if (++cores_done >= kvm->arch.online_vcores)
2274 			break;
2275 	}
2276 }
2277 
2278 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
2279 {
2280 	return;
2281 }
2282 
2283 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
2284 {
2285 	int err = 0;
2286 	struct kvm *kvm = vcpu->kvm;
2287 	struct kvm_rma_info *ri = NULL;
2288 	unsigned long hva;
2289 	struct kvm_memory_slot *memslot;
2290 	struct vm_area_struct *vma;
2291 	unsigned long lpcr = 0, senc;
2292 	unsigned long lpcr_mask = 0;
2293 	unsigned long psize, porder;
2294 	unsigned long rma_size;
2295 	unsigned long rmls;
2296 	unsigned long *physp;
2297 	unsigned long i, npages;
2298 	int srcu_idx;
2299 
2300 	mutex_lock(&kvm->lock);
2301 	if (kvm->arch.rma_setup_done)
2302 		goto out;	/* another vcpu beat us to it */
2303 
2304 	/* Allocate hashed page table (if not done already) and reset it */
2305 	if (!kvm->arch.hpt_virt) {
2306 		err = kvmppc_alloc_hpt(kvm, NULL);
2307 		if (err) {
2308 			pr_err("KVM: Couldn't alloc HPT\n");
2309 			goto out;
2310 		}
2311 	}
2312 
2313 	/* Look up the memslot for guest physical address 0 */
2314 	srcu_idx = srcu_read_lock(&kvm->srcu);
2315 	memslot = gfn_to_memslot(kvm, 0);
2316 
2317 	/* We must have some memory at 0 by now */
2318 	err = -EINVAL;
2319 	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
2320 		goto out_srcu;
2321 
2322 	/* Look up the VMA for the start of this memory slot */
2323 	hva = memslot->userspace_addr;
2324 	down_read(&current->mm->mmap_sem);
2325 	vma = find_vma(current->mm, hva);
2326 	if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
2327 		goto up_out;
2328 
2329 	psize = vma_kernel_pagesize(vma);
2330 	porder = __ilog2(psize);
2331 
2332 	/* Is this one of our preallocated RMAs? */
2333 	if (vma->vm_file && vma->vm_file->f_op == &kvm_rma_fops &&
2334 	    hva == vma->vm_start)
2335 		ri = vma->vm_file->private_data;
2336 
2337 	up_read(&current->mm->mmap_sem);
2338 
2339 	if (!ri) {
2340 		/* On POWER7, use VRMA; on PPC970, give up */
2341 		err = -EPERM;
2342 		if (cpu_has_feature(CPU_FTR_ARCH_201)) {
2343 			pr_err("KVM: CPU requires an RMO\n");
2344 			goto out_srcu;
2345 		}
2346 
2347 		/* We can handle 4k, 64k or 16M pages in the VRMA */
2348 		err = -EINVAL;
2349 		if (!(psize == 0x1000 || psize == 0x10000 ||
2350 		      psize == 0x1000000))
2351 			goto out_srcu;
2352 
2353 		/* Update VRMASD field in the LPCR */
2354 		senc = slb_pgsize_encoding(psize);
2355 		kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
2356 			(VRMA_VSID << SLB_VSID_SHIFT_1T);
2357 		lpcr_mask = LPCR_VRMASD;
2358 		/* the -4 is to account for senc values starting at 0x10 */
2359 		lpcr = senc << (LPCR_VRMASD_SH - 4);
2360 
2361 		/* Create HPTEs in the hash page table for the VRMA */
2362 		kvmppc_map_vrma(vcpu, memslot, porder);
2363 
2364 	} else {
2365 		/* Set up to use an RMO region */
2366 		rma_size = kvm_rma_pages;
2367 		if (rma_size > memslot->npages)
2368 			rma_size = memslot->npages;
2369 		rma_size <<= PAGE_SHIFT;
2370 		rmls = lpcr_rmls(rma_size);
2371 		err = -EINVAL;
2372 		if ((long)rmls < 0) {
2373 			pr_err("KVM: Can't use RMA of 0x%lx bytes\n", rma_size);
2374 			goto out_srcu;
2375 		}
2376 		atomic_inc(&ri->use_count);
2377 		kvm->arch.rma = ri;
2378 
2379 		/* Update LPCR and RMOR */
2380 		if (cpu_has_feature(CPU_FTR_ARCH_201)) {
2381 			/* PPC970; insert RMLS value (split field) in HID4 */
2382 			lpcr_mask = (1ul << HID4_RMLS0_SH) |
2383 				(3ul << HID4_RMLS2_SH) | HID4_RMOR;
2384 			lpcr = ((rmls >> 2) << HID4_RMLS0_SH) |
2385 				((rmls & 3) << HID4_RMLS2_SH);
2386 			/* RMOR is also in HID4 */
2387 			lpcr |= ((ri->base_pfn >> (26 - PAGE_SHIFT)) & 0xffff)
2388 				<< HID4_RMOR_SH;
2389 		} else {
2390 			/* POWER7 */
2391 			lpcr_mask = LPCR_VPM0 | LPCR_VRMA_L | LPCR_RMLS;
2392 			lpcr = rmls << LPCR_RMLS_SH;
2393 			kvm->arch.rmor = ri->base_pfn << PAGE_SHIFT;
2394 		}
2395 		pr_info("KVM: Using RMO at %lx size %lx (LPCR = %lx)\n",
2396 			ri->base_pfn << PAGE_SHIFT, rma_size, lpcr);
2397 
2398 		/* Initialize phys addrs of pages in RMO */
2399 		npages = kvm_rma_pages;
2400 		porder = __ilog2(npages);
2401 		physp = memslot->arch.slot_phys;
2402 		if (physp) {
2403 			if (npages > memslot->npages)
2404 				npages = memslot->npages;
2405 			spin_lock(&kvm->arch.slot_phys_lock);
2406 			for (i = 0; i < npages; ++i)
2407 				physp[i] = ((ri->base_pfn + i) << PAGE_SHIFT) +
2408 					porder;
2409 			spin_unlock(&kvm->arch.slot_phys_lock);
2410 		}
2411 	}
2412 
2413 	kvmppc_update_lpcr(kvm, lpcr, lpcr_mask);
2414 
2415 	/* Order updates to kvm->arch.lpcr etc. vs. rma_setup_done */
2416 	smp_wmb();
2417 	kvm->arch.rma_setup_done = 1;
2418 	err = 0;
2419  out_srcu:
2420 	srcu_read_unlock(&kvm->srcu, srcu_idx);
2421  out:
2422 	mutex_unlock(&kvm->lock);
2423 	return err;
2424 
2425  up_out:
2426 	up_read(&current->mm->mmap_sem);
2427 	goto out_srcu;
2428 }
2429 
2430 static int kvmppc_core_init_vm_hv(struct kvm *kvm)
2431 {
2432 	unsigned long lpcr, lpid;
2433 
2434 	/* Allocate the guest's logical partition ID */
2435 
2436 	lpid = kvmppc_alloc_lpid();
2437 	if ((long)lpid < 0)
2438 		return -ENOMEM;
2439 	kvm->arch.lpid = lpid;
2440 
2441 	/*
2442 	 * Since we don't flush the TLB when tearing down a VM,
2443 	 * and this lpid might have previously been used,
2444 	 * make sure we flush on each core before running the new VM.
2445 	 */
2446 	cpumask_setall(&kvm->arch.need_tlb_flush);
2447 
2448 	/* Start out with the default set of hcalls enabled */
2449 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
2450 	       sizeof(kvm->arch.enabled_hcalls));
2451 
2452 	kvm->arch.rma = NULL;
2453 
2454 	kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
2455 
2456 	if (cpu_has_feature(CPU_FTR_ARCH_201)) {
2457 		/* PPC970; HID4 is effectively the LPCR */
2458 		kvm->arch.host_lpid = 0;
2459 		kvm->arch.host_lpcr = lpcr = mfspr(SPRN_HID4);
2460 		lpcr &= ~((3 << HID4_LPID1_SH) | (0xful << HID4_LPID5_SH));
2461 		lpcr |= ((lpid >> 4) << HID4_LPID1_SH) |
2462 			((lpid & 0xf) << HID4_LPID5_SH);
2463 	} else {
2464 		/* POWER7; init LPCR for virtual RMA mode */
2465 		kvm->arch.host_lpid = mfspr(SPRN_LPID);
2466 		kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
2467 		lpcr &= LPCR_PECE | LPCR_LPES;
2468 		lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
2469 			LPCR_VPM0 | LPCR_VPM1;
2470 		kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
2471 			(VRMA_VSID << SLB_VSID_SHIFT_1T);
2472 		/* On POWER8 turn on online bit to enable PURR/SPURR */
2473 		if (cpu_has_feature(CPU_FTR_ARCH_207S))
2474 			lpcr |= LPCR_ONL;
2475 	}
2476 	kvm->arch.lpcr = lpcr;
2477 
2478 	kvm->arch.using_mmu_notifiers = !!cpu_has_feature(CPU_FTR_ARCH_206);
2479 	spin_lock_init(&kvm->arch.slot_phys_lock);
2480 
2481 	/*
2482 	 * Track that we now have a HV mode VM active. This blocks secondary
2483 	 * CPU threads from coming online.
2484 	 */
2485 	kvm_hv_vm_activated();
2486 
2487 	return 0;
2488 }
2489 
2490 static void kvmppc_free_vcores(struct kvm *kvm)
2491 {
2492 	long int i;
2493 
2494 	for (i = 0; i < KVM_MAX_VCORES; ++i) {
2495 		if (kvm->arch.vcores[i] && kvm->arch.vcores[i]->mpp_buffer) {
2496 			struct kvmppc_vcore *vc = kvm->arch.vcores[i];
2497 			free_pages((unsigned long)vc->mpp_buffer,
2498 				   MPP_BUFFER_ORDER);
2499 		}
2500 		kfree(kvm->arch.vcores[i]);
2501 	}
2502 	kvm->arch.online_vcores = 0;
2503 }
2504 
2505 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
2506 {
2507 	kvm_hv_vm_deactivated();
2508 
2509 	kvmppc_free_vcores(kvm);
2510 	if (kvm->arch.rma) {
2511 		kvm_release_rma(kvm->arch.rma);
2512 		kvm->arch.rma = NULL;
2513 	}
2514 
2515 	kvmppc_free_hpt(kvm);
2516 }
2517 
2518 /* We don't need to emulate any privileged instructions or dcbz */
2519 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
2520 				     unsigned int inst, int *advance)
2521 {
2522 	return EMULATE_FAIL;
2523 }
2524 
2525 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
2526 					ulong spr_val)
2527 {
2528 	return EMULATE_FAIL;
2529 }
2530 
2531 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
2532 					ulong *spr_val)
2533 {
2534 	return EMULATE_FAIL;
2535 }
2536 
2537 static int kvmppc_core_check_processor_compat_hv(void)
2538 {
2539 	if (!cpu_has_feature(CPU_FTR_HVMODE))
2540 		return -EIO;
2541 	return 0;
2542 }
2543 
2544 static long kvm_arch_vm_ioctl_hv(struct file *filp,
2545 				 unsigned int ioctl, unsigned long arg)
2546 {
2547 	struct kvm *kvm __maybe_unused = filp->private_data;
2548 	void __user *argp = (void __user *)arg;
2549 	long r;
2550 
2551 	switch (ioctl) {
2552 
2553 	case KVM_ALLOCATE_RMA: {
2554 		struct kvm_allocate_rma rma;
2555 		struct kvm *kvm = filp->private_data;
2556 
2557 		r = kvm_vm_ioctl_allocate_rma(kvm, &rma);
2558 		if (r >= 0 && copy_to_user(argp, &rma, sizeof(rma)))
2559 			r = -EFAULT;
2560 		break;
2561 	}
2562 
2563 	case KVM_PPC_ALLOCATE_HTAB: {
2564 		u32 htab_order;
2565 
2566 		r = -EFAULT;
2567 		if (get_user(htab_order, (u32 __user *)argp))
2568 			break;
2569 		r = kvmppc_alloc_reset_hpt(kvm, &htab_order);
2570 		if (r)
2571 			break;
2572 		r = -EFAULT;
2573 		if (put_user(htab_order, (u32 __user *)argp))
2574 			break;
2575 		r = 0;
2576 		break;
2577 	}
2578 
2579 	case KVM_PPC_GET_HTAB_FD: {
2580 		struct kvm_get_htab_fd ghf;
2581 
2582 		r = -EFAULT;
2583 		if (copy_from_user(&ghf, argp, sizeof(ghf)))
2584 			break;
2585 		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
2586 		break;
2587 	}
2588 
2589 	default:
2590 		r = -ENOTTY;
2591 	}
2592 
2593 	return r;
2594 }
2595 
2596 /*
2597  * List of hcall numbers to enable by default.
2598  * For compatibility with old userspace, we enable by default
2599  * all hcalls that were implemented before the hcall-enabling
2600  * facility was added.  Note this list should not include H_RTAS.
2601  */
2602 static unsigned int default_hcall_list[] = {
2603 	H_REMOVE,
2604 	H_ENTER,
2605 	H_READ,
2606 	H_PROTECT,
2607 	H_BULK_REMOVE,
2608 	H_GET_TCE,
2609 	H_PUT_TCE,
2610 	H_SET_DABR,
2611 	H_SET_XDABR,
2612 	H_CEDE,
2613 	H_PROD,
2614 	H_CONFER,
2615 	H_REGISTER_VPA,
2616 #ifdef CONFIG_KVM_XICS
2617 	H_EOI,
2618 	H_CPPR,
2619 	H_IPI,
2620 	H_IPOLL,
2621 	H_XIRR,
2622 	H_XIRR_X,
2623 #endif
2624 	0
2625 };
2626 
2627 static void init_default_hcalls(void)
2628 {
2629 	int i;
2630 	unsigned int hcall;
2631 
2632 	for (i = 0; default_hcall_list[i]; ++i) {
2633 		hcall = default_hcall_list[i];
2634 		WARN_ON(!kvmppc_hcall_impl_hv(hcall));
2635 		__set_bit(hcall / 4, default_enabled_hcalls);
2636 	}
2637 }
2638 
2639 static struct kvmppc_ops kvm_ops_hv = {
2640 	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
2641 	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
2642 	.get_one_reg = kvmppc_get_one_reg_hv,
2643 	.set_one_reg = kvmppc_set_one_reg_hv,
2644 	.vcpu_load   = kvmppc_core_vcpu_load_hv,
2645 	.vcpu_put    = kvmppc_core_vcpu_put_hv,
2646 	.set_msr     = kvmppc_set_msr_hv,
2647 	.vcpu_run    = kvmppc_vcpu_run_hv,
2648 	.vcpu_create = kvmppc_core_vcpu_create_hv,
2649 	.vcpu_free   = kvmppc_core_vcpu_free_hv,
2650 	.check_requests = kvmppc_core_check_requests_hv,
2651 	.get_dirty_log  = kvm_vm_ioctl_get_dirty_log_hv,
2652 	.flush_memslot  = kvmppc_core_flush_memslot_hv,
2653 	.prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
2654 	.commit_memory_region  = kvmppc_core_commit_memory_region_hv,
2655 	.unmap_hva = kvm_unmap_hva_hv,
2656 	.unmap_hva_range = kvm_unmap_hva_range_hv,
2657 	.age_hva  = kvm_age_hva_hv,
2658 	.test_age_hva = kvm_test_age_hva_hv,
2659 	.set_spte_hva = kvm_set_spte_hva_hv,
2660 	.mmu_destroy  = kvmppc_mmu_destroy_hv,
2661 	.free_memslot = kvmppc_core_free_memslot_hv,
2662 	.create_memslot = kvmppc_core_create_memslot_hv,
2663 	.init_vm =  kvmppc_core_init_vm_hv,
2664 	.destroy_vm = kvmppc_core_destroy_vm_hv,
2665 	.get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
2666 	.emulate_op = kvmppc_core_emulate_op_hv,
2667 	.emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
2668 	.emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
2669 	.fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
2670 	.arch_vm_ioctl  = kvm_arch_vm_ioctl_hv,
2671 	.hcall_implemented = kvmppc_hcall_impl_hv,
2672 };
2673 
2674 static int kvmppc_book3s_init_hv(void)
2675 {
2676 	int r;
2677 	/*
2678 	 * FIXME!! Do we need to check on all cpus ?
2679 	 */
2680 	r = kvmppc_core_check_processor_compat_hv();
2681 	if (r < 0)
2682 		return -ENODEV;
2683 
2684 	kvm_ops_hv.owner = THIS_MODULE;
2685 	kvmppc_hv_ops = &kvm_ops_hv;
2686 
2687 	init_default_hcalls();
2688 
2689 	r = kvmppc_mmu_hv_init();
2690 	return r;
2691 }
2692 
2693 static void kvmppc_book3s_exit_hv(void)
2694 {
2695 	kvmppc_hv_ops = NULL;
2696 }
2697 
2698 module_init(kvmppc_book3s_init_hv);
2699 module_exit(kvmppc_book3s_exit_hv);
2700 MODULE_LICENSE("GPL");
2701 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2702 MODULE_ALIAS("devname:kvm");
2703